ARROW-10385: [C++][Gandiva] Add support for LLVM 11

.env

@@ -24,9 +24,9 @@ ARCH=amd64
 CUDA=9.1
 DEBIAN=10
 UBUNTU=18.04
-FEDORA=32
+FEDORA=33
 PYTHON=3.6
-LLVM=10
+LLVM=11
 CLANG_TOOLS=8
 RUST=nightly-2020-04-22
 GO=1.12

.github/workflows/cpp.yml

@@ -338,7 +338,6 @@ jobs:
  CMAKE_ARGS: >-
  -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
  -DBoost_NO_BOOST_CMAKE=ON
- CMAKE_GENERATOR: MSYS Makefiles
  CMAKE_UNITY_BUILD: ON
  steps:
  - name: Disable Crash Dialogs

.github/workflows/cpp_cron.yml

@@ -46,7 +46,7 @@ jobs:
  matrix:
  name:
  - amd64-debian-10-cpp
- - amd64-fedora-32-cpp
+ - amd64-fedora-33-cpp
  - amd64-ubuntu-16.04-cpp
  - amd64-ubuntu-18.04-cpp
  - amd64-ubuntu-18.04-cpp-cmake32
@@ -55,10 +55,10 @@ jobs:
  image: debian-cpp
  title: AMD64 Debian 10 C++
  debian: 10
- - name: amd64-fedora-32-cpp
+ - name: amd64-fedora-33-cpp
  image: fedora-cpp
- title: AMD64 Fedora 32 C++
- fedora: 32
+ title: AMD64 Fedora 33 C++
+ fedora: 33
  - name: amd64-ubuntu-16.04-cpp
  image: ubuntu-cpp
  title: AMD64 Ubuntu 16.04 C++
@@ -75,7 +75,7 @@ jobs:
  # the defaults here should correspond to the values in .env
  ARCH: 'amd64'
  DEBIAN: ${{ matrix.debian || 10 }}
- FEDORA: ${{ matrix.fedora || 32 }}
+ FEDORA: ${{ matrix.fedora || 33 }}
  UBUNTU: ${{ matrix.ubuntu || 18.04 }}
  steps:
  - name: Checkout Arrow

.github/workflows/python_cron.yml

@@ -45,7 +45,7 @@ jobs:
  matrix:
  name:
  - debian-10-python-3
- - fedora-32-python-3
+ - fedora-33-python-3
  - ubuntu-18.04-python-3
  - conda-python-3.7-dask-latest
  - conda-python-3.7-turbodbc-latest
@@ -59,11 +59,11 @@ jobs:
  image: debian-python
  title: AMD64 Debian 10 Python 3
  debian: 10
- - name: fedora-32-python-3
- cache: fedora-32-python-3
+ - name: fedora-33-python-3
+ cache: fedora-33-python-3
  image: fedora-python
- title: AMD64 Fedora 32 Python 3
- fedora: 32
+ title: AMD64 Fedora 33 Python 3
+ fedora: 33
  - name: ubuntu-18.04-python-3
  cache: ubuntu-18.04-python-3
  image: ubuntu-python
@@ -102,7 +102,7 @@ jobs:
  env:
  # the defaults here should correspond to the values in .env
  DEBIAN: ${{ matrix.debian || 10 }}
- FEDORA: ${{ matrix.fedora || 32 }}
+ FEDORA: ${{ matrix.fedora || 33 }}
  UBUNTU: ${{ matrix.ubuntu || 18.04 }}
  PYTHON: ${{ matrix.python || 3.7 }}
  HDFS: ${{ matrix.hdfs || '2.9.2' }}

.github/workflows/ruby.yml

@@ -188,7 +188,6 @@ jobs:
  CMAKE_ARGS: >-
  -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
  -DBoost_NO_BOOST_CMAKE=ON
- CMAKE_GENERATOR: MSYS Makefiles
  CMAKE_UNITY_BUILD: ON
  steps:
  - name: Disable Crash Dialogs

.travis.yml

@@ -65,6 +65,9 @@ jobs:
  -e ARROW_GANDIVA=OFF
  -e ARROW_PARQUET=OFF
  "
+ # The LLVM's APT repository provides only arm64 binaries.
+ # We should use LLVM provided by Ubuntu.
+ LLVM: "10"
  UBUNTU: "20.04"
 
  - name: "C++ on s390x"
@@ -90,6 +93,9 @@ jobs:
  -e cares_SOURCE=BUNDLED
  -e gRPC_SOURCE=BUNDLED
  "
+ # The LLVM's APT repository provides only arm64 binaries.
+ # We should use LLVM provided by Ubuntu.
+ LLVM: "10"
  UBUNTU: "20.04"
 
  - name: "Go on s390x"

ci/conda_env_gandiva.yml

@@ -15,6 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-clangdev=10
-llvmdev=10
+clangdev=11
+llvmdev=11
 re2

ci/docker/fedora-32-cpp.dockerfile → ci/docker/fedora-33-cpp.dockerfile

@@ -16,7 +16,7 @@
 # under the License.
 
 ARG arch
-FROM ${arch}/fedora:32
+FROM ${arch}/fedora:33
 ARG arch
 
 # install dependencies
@@ -32,24 +32,26 @@ RUN dnf update -y && \
  cmake \
  curl-devel \
  flatbuffers-devel \
- java-1.8.0-openjdk-devel \
- java-1.8.0-openjdk-headless \
  gcc \
  gcc-c++ \
- glog-devel \
  gflags-devel \
+ git \
+ glog-devel \
  gmock-devel \
  google-benchmark-devel \
- protobuf-devel \
+ grpc-devel \
+ grpc-plugins \
  gtest-devel \
- git \
+ java-latest-openjdk-devel \
+ java-latest-openjdk-headless \
  libzstd-devel \
  llvm-devel \
  llvm-static \
  lz4-devel \
  make \
  ninja-build \
  openssl-devel \
+ protobuf-devel \
  python \
  rapidjson-devel \
  re2-devel \
@@ -64,7 +66,6 @@ COPY ci/scripts/install_minio.sh \
  /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
 
-# * gRPC 1.26 in Fedora 32 may have a problem. arrow-flight-test is stuck.
 ENV ARROW_BUILD_TESTS=ON \
  ARROW_DEPENDENCY_SOURCE=SYSTEM \
  ARROW_DATASET=ON \
@@ -85,7 +86,6 @@ ENV ARROW_BUILD_TESTS=ON \
  AWSSDK_SOURCE=BUNDLED \
  CC=gcc \
  CXX=g++ \
- gRPC_SOURCE=BUNDLED \
  ORC_SOURCE=BUNDLED \
  PARQUET_BUILD_EXECUTABLES=ON \
  PARQUET_BUILD_EXAMPLES=ON \

ci/docker/linux-apt-c-glib.dockerfile

@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
  gtk-doc-tools \
  libgirepository1.0-dev \
  libglib2.0-doc \
+ lsb-release \
  luarocks \
  pkg-config \
  ruby-dev && \

ci/docker/ubuntu-20.04-cpp.dockerfile

@@ -30,7 +30,22 @@ RUN echo "debconf debconf/frontend select Noninteractive" | \
 # while debugging package list with docker build.
 ARG clang_tools
 ARG llvm
-RUN apt-get update -y -q && \
+RUN if [ "${llvm}" -gt "10" ]; then \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ apt-transport-https \
+ ca-certificates \
+ gnupg \
+ wget && \
+ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+ echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main" > \
+ /etc/apt/sources.list.d/llvm.list && \
+ if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \
+ echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \
+ /etc/apt/sources.list.d/clang-tools.list; \
+ fi \
+ fi && \
+ apt-get update -y -q && \
  apt-get install -y -q --no-install-recommends \
  clang-${clang_tools} \
  clang-${llvm} \

ci/scripts/msys2_setup.sh

@@ -24,26 +24,26 @@ target=$1
 packages=()
 case "${target}" in
  cpp|c_glib|ruby)
- packages+=(make)
  packages+=(${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp)
- packages+=(${MINGW_PACKAGE_PREFIX}-ccache)
  packages+=(${MINGW_PACKAGE_PREFIX}-boost)
  packages+=(${MINGW_PACKAGE_PREFIX}-brotli)
+ packages+=(${MINGW_PACKAGE_PREFIX}-ccache)
  packages+=(${MINGW_PACKAGE_PREFIX}-clang)
  packages+=(${MINGW_PACKAGE_PREFIX}-cmake)
  packages+=(${MINGW_PACKAGE_PREFIX}-gcc)
  packages+=(${MINGW_PACKAGE_PREFIX}-gflags)
  packages+=(${MINGW_PACKAGE_PREFIX}-grpc)
  packages+=(${MINGW_PACKAGE_PREFIX}-gtest)
+ packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc)
  packages+=(${MINGW_PACKAGE_PREFIX}-llvm)
  packages+=(${MINGW_PACKAGE_PREFIX}-lz4)
+ packages+=(${MINGW_PACKAGE_PREFIX}-ninja)
  packages+=(${MINGW_PACKAGE_PREFIX}-polly)
  packages+=(${MINGW_PACKAGE_PREFIX}-protobuf)
  packages+=(${MINGW_PACKAGE_PREFIX}-python3-numpy)
  packages+=(${MINGW_PACKAGE_PREFIX}-rapidjson)
  packages+=(${MINGW_PACKAGE_PREFIX}-snappy)
  packages+=(${MINGW_PACKAGE_PREFIX}-thrift)
- packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc)
  packages+=(${MINGW_PACKAGE_PREFIX}-zlib)
  packages+=(${MINGW_PACKAGE_PREFIX}-zstd)
  ;;

cpp/CMakeLists.txt

@@ -109,7 +109,12 @@ set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(ARROW_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
 set(ARROW_DOC_DIR "share/doc/${PROJECT_NAME}")
 
-set(ARROW_LLVM_VERSIONS "10" "9" "8" "7")
+set(ARROW_LLVM_VERSIONS
+ "11"
+ "10"
+ "9"
+ "8"
+ "7")
 list(GET ARROW_LLVM_VERSIONS 0 ARROW_LLVM_VERSION_PRIMARY)
 string(REGEX
  REPLACE "^([0-9]+)(\\..+)?" "\\1" ARROW_LLVM_VERSION_PRIMARY_MAJOR

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -1824,6 +1824,7 @@ macro(build_rapidjson)
  set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_PREFIX}/include")
 
  add_dependencies(toolchain rapidjson_ep)
+ add_dependencies(toolchain-tests rapidjson_ep)
  add_dependencies(rapidjson rapidjson_ep)
 
  set(RAPIDJSON_VENDORED TRUE)

cpp/src/gandiva/engine.cc

@@ -53,6 +53,7 @@
 #include <llvm/Linker/Linker.h>
 #include <llvm/MC/SubtargetFeature.h>
 #include <llvm/Support/DynamicLibrary.h>
+#include <llvm/Support/Host.h>
 #include <llvm/Support/TargetRegistry.h>
 #include <llvm/Support/TargetSelect.h>
 #include <llvm/Support/raw_ostream.h>

dev/tasks/conda-recipes/arrow-cpp/meta.yaml

@@ -68,8 +68,8 @@ outputs:
  - glog
  - grpc-cpp
  - libprotobuf
- - clangdev 10
- - llvmdev 10
+ - clangdev 11
+ - llvmdev 11
  - libutf8proc
  - lz4-c
  - numpy

dev/tasks/gandiva-jars/build-cpp-linux.sh

@@ -64,7 +64,7 @@ cmake -DCMAKE_BUILD_TYPE=Release \
  -DBOOST_ROOT=/arrow_boost_dist \
  -GNinja /arrow/cpp
 ninja install
-ninja test
+CTEST_OUTPUT_ON_FAILURE=1 ninja test
 popd
 
 

dev/tasks/homebrew-formulae/apache-arrow.rb

@@ -16,7 +16,7 @@ class ApacheArrow < Formula
  depends_on "numpy"
  depends_on "[email protected]"
  depends_on "protobuf"
- depends_on "python@3.8"
+ depends_on "python@3.9"
  depends_on "rapidjson"
  depends_on "snappy"
  depends_on "thrift"
@@ -42,7 +42,7 @@ def install
  -DARROW_WITH_SNAPPY=ON
  -DARROW_WITH_BROTLI=ON
  -DARROW_INSTALL_NAME_RPATH=OFF
- -DPYTHON_EXECUTABLE=#{Formula["python@3.8"].bin/"python3"}
+ -DPython3_EXECUTABLE=#{Formula["python@3.9"].bin/"python3"}
  ]
  # Re-enable -DARROW_S3=ON and add back aws-sdk-cpp to depends_on in ARROW-6437
 

dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile

@@ -33,6 +33,15 @@ ARG LLVM
 RUN \
  quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
  apt update ${quiet} && \
+ apt install -y -V ${quiet} \
+ apt-transport-https \
+ ca-certificates \
+ gnupg \
+ wget && \
+ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+ echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM} main" > \
+ /etc/apt/sources.list.d/llvm.list && \
+ apt update ${quiet} && \
  apt install -y -V ${quiet} \
  build-essential \
  cmake \

dev/tasks/linux-packages/package-task.rb

@@ -54,7 +54,7 @@ def initialize(package, version, release_time, options={})
  @rpm_version = @version
  @rpm_release = "1"
  end
- @deb_release = "1"
+ @deb_release = ENV["DEB_RELEASE"] || "1"
  end
 
  def define

dev/tasks/nuget-packages/github.linux.yml

@@ -40,7 +40,9 @@ jobs:
  git -C arrow checkout FETCH_HEAD
  git -C arrow submodule update --init --recursive
  - name: Setup Python
- uses: actions/setup-python@v1
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.8
  - name: Setup Archery
  run: pip install -e arrow/dev/archery[docker]
  - name: Prepare version

dev/tasks/tasks.yml

@@ -1599,12 +1599,12 @@ tasks:
  UBUNTU: 18.04
  run: ubuntu-cpp
 
- test-fedora-32-cpp:
+ test-fedora-33-cpp:
  ci: circle
  template: docker-tests/circle.linux.yml
  params:
  env:
- FEDORA: 32
+ FEDORA: 33
  run: fedora-cpp
 
  test-ubuntu-18.04-cpp-release:
@@ -1719,12 +1719,12 @@ tasks:
  UBUNTU: 18.04
  run: ubuntu-python
 
- test-fedora-32-python-3:
+ test-fedora-33-python-3:
  ci: azure
  template: docker-tests/azure.linux.yml
  params:
  env:
- FEDORA: 32
+ FEDORA: 33
  run: fedora-python
 
  test-r-linux-as-cran:

docker-compose.yml

@@ -332,7 +332,7 @@ services:
  # docker-compose run --rm fedora-cpp
  # Parameters:
  # ARCH: amd64, arm64v8, ...
- # FEDORA: 32
+ # FEDORA: 33
  image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp
  build:
  context: .
@@ -572,7 +572,7 @@ services:
  # docker-compose run --rm fedora-python
  # Parameters:
  # ARCH: amd64, arm64v8, ...
- # FEDORA: 32
+ # FEDORA: 33
  image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3
  build:
  context: .