From 6c9a5fb49ac02617ed930e7c5e02aa79c1f8c10a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 04:10:44 +0900 Subject: [PATCH 01/14] [C++][Gandiva] Add support for LLVM 11 --- .env | 4 ++-- .github/workflows/cpp_cron.yml | 10 +++++----- .github/workflows/python_cron.yml | 12 ++++++------ ci/conda_env_gandiva.yml | 4 ++-- ...2-cpp.dockerfile => fedora-33-cpp.dockerfile} | 16 ++++++++-------- cpp/CMakeLists.txt | 7 ++++++- cpp/src/gandiva/engine.cc | 1 + dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 4 ++-- .../apache-arrow/apt/ubuntu-focal/Dockerfile | 9 +++++++++ dev/tasks/linux-packages/package-task.rb | 2 +- dev/tasks/tasks.yml | 8 ++++---- docker-compose.yml | 4 ++-- 12 files changed, 48 insertions(+), 33 deletions(-) rename ci/docker/{fedora-32-cpp.dockerfile => fedora-33-cpp.dockerfile} (92%) diff --git a/.env b/.env index c266213d53c2d..5c6aaa5182af7 100644 --- a/.env +++ b/.env @@ -24,9 +24,9 @@ ARCH=amd64 CUDA=9.1 DEBIAN=10 UBUNTU=18.04 -FEDORA=32 +FEDORA=33 PYTHON=3.6 -LLVM=10 +LLVM=11 CLANG_TOOLS=8 RUST=nightly-2020-04-22 GO=1.12 diff --git a/.github/workflows/cpp_cron.yml b/.github/workflows/cpp_cron.yml index 95c5f4b7cbb3a..5f404b22fe5b9 100644 --- a/.github/workflows/cpp_cron.yml +++ b/.github/workflows/cpp_cron.yml @@ -46,7 +46,7 @@ jobs: matrix: name: - amd64-debian-10-cpp - - amd64-fedora-32-cpp + - amd64-fedora-33-cpp - amd64-ubuntu-16.04-cpp - amd64-ubuntu-18.04-cpp - amd64-ubuntu-18.04-cpp-cmake32 @@ -55,10 +55,10 @@ jobs: image: debian-cpp title: AMD64 Debian 10 C++ debian: 10 - - name: amd64-fedora-32-cpp + - name: amd64-fedora-33-cpp image: fedora-cpp - title: AMD64 Fedora 32 C++ - fedora: 32 + title: AMD64 Fedora 33 C++ + fedora: 33 - name: amd64-ubuntu-16.04-cpp image: ubuntu-cpp title: AMD64 Ubuntu 16.04 C++ @@ -75,7 +75,7 @@ jobs: # the defaults here should correspond to the values in .env ARCH: 'amd64' DEBIAN: ${{ matrix.debian || 10 }} - FEDORA: ${{ matrix.fedora || 32 }} + FEDORA: ${{ matrix.fedora || 33 }} UBUNTU: ${{ matrix.ubuntu || 18.04 }} steps: - name: Checkout Arrow diff --git a/.github/workflows/python_cron.yml b/.github/workflows/python_cron.yml index 133a73b59bf84..256856bd19720 100644 --- a/.github/workflows/python_cron.yml +++ b/.github/workflows/python_cron.yml @@ -45,7 +45,7 @@ jobs: matrix: name: - debian-10-python-3 - - fedora-32-python-3 + - fedora-33-python-3 - ubuntu-18.04-python-3 - conda-python-3.7-dask-latest - conda-python-3.7-turbodbc-latest @@ -59,11 +59,11 @@ jobs: image: debian-python title: AMD64 Debian 10 Python 3 debian: 10 - - name: fedora-32-python-3 - cache: fedora-32-python-3 + - name: fedora-33-python-3 + cache: fedora-33-python-3 image: fedora-python - title: AMD64 Fedora 32 Python 3 - fedora: 32 + title: AMD64 Fedora 33 Python 3 + fedora: 33 - name: ubuntu-18.04-python-3 cache: ubuntu-18.04-python-3 image: ubuntu-python @@ -102,7 +102,7 @@ jobs: env: # the defaults here should correspond to the values in .env DEBIAN: ${{ matrix.debian || 10 }} - FEDORA: ${{ matrix.fedora || 32 }} + FEDORA: ${{ matrix.fedora || 33 }} UBUNTU: ${{ matrix.ubuntu || 18.04 }} PYTHON: ${{ matrix.python || 3.7 }} HDFS: ${{ matrix.hdfs || '2.9.2' }} diff --git a/ci/conda_env_gandiva.yml b/ci/conda_env_gandiva.yml index fa15d77df8384..5056456fc666f 100644 --- a/ci/conda_env_gandiva.yml +++ b/ci/conda_env_gandiva.yml @@ -15,6 +15,6 @@ # specific language governing permissions and limitations # under the License. -clangdev=10 -llvmdev=10 +clangdev=11 +llvmdev=11 re2 diff --git a/ci/docker/fedora-32-cpp.dockerfile b/ci/docker/fedora-33-cpp.dockerfile similarity index 92% rename from ci/docker/fedora-32-cpp.dockerfile rename to ci/docker/fedora-33-cpp.dockerfile index 40fe4617b12c2..9dde6999510ac 100644 --- a/ci/docker/fedora-32-cpp.dockerfile +++ b/ci/docker/fedora-33-cpp.dockerfile @@ -16,7 +16,7 @@ # under the License. ARG arch -FROM ${arch}/fedora:32 +FROM ${arch}/fedora:33 ARG arch # install dependencies @@ -32,17 +32,18 @@ RUN dnf update -y && \ cmake \ curl-devel \ flatbuffers-devel \ - java-1.8.0-openjdk-devel \ - java-1.8.0-openjdk-headless \ gcc \ gcc-c++ \ - glog-devel \ gflags-devel \ + git \ + glog-devel \ gmock-devel \ google-benchmark-devel \ - protobuf-devel \ + grpc-devel \ + grpc-plugins \ gtest-devel \ - git \ + java-latest-openjdk-devel \ + java-latest-openjdk-headless \ libzstd-devel \ llvm-devel \ llvm-static \ @@ -50,6 +51,7 @@ RUN dnf update -y && \ make \ ninja-build \ openssl-devel \ + protobuf-devel \ python \ rapidjson-devel \ re2-devel \ @@ -64,7 +66,6 @@ COPY ci/scripts/install_minio.sh \ /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local -# * gRPC 1.26 in Fedora 32 may have a problem. arrow-flight-test is stuck. ENV ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_DATASET=ON \ @@ -85,7 +86,6 @@ ENV ARROW_BUILD_TESTS=ON \ AWSSDK_SOURCE=BUNDLED \ CC=gcc \ CXX=g++ \ - gRPC_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ PARQUET_BUILD_EXECUTABLES=ON \ PARQUET_BUILD_EXAMPLES=ON \ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d3c5b2297447d..f9ab1548fbd9d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -109,7 +109,12 @@ set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") set(ARROW_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") set(ARROW_DOC_DIR "share/doc/${PROJECT_NAME}") -set(ARROW_LLVM_VERSIONS "10" "9" "8" "7") +set(ARROW_LLVM_VERSIONS + "11" + "10" + "9" + "8" + "7") list(GET ARROW_LLVM_VERSIONS 0 ARROW_LLVM_VERSION_PRIMARY) string(REGEX REPLACE "^([0-9]+)(\\..+)?" "\\1" ARROW_LLVM_VERSION_PRIMARY_MAJOR diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index ad73751ecdc32..cd4edff89ec17 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index b5a2bec769ab9..272ed8c5382a5 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -68,8 +68,8 @@ outputs: - glog - grpc-cpp - libprotobuf - - clangdev 10 - - llvmdev 10 + - clangdev 11 + - llvmdev 11 - libutf8proc - lz4-c - numpy diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile index 8715dd8897b8a..a861c41e48e49 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile @@ -33,6 +33,15 @@ ARG LLVM RUN \ quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ apt update ${quiet} && \ + apt install -y -V ${quiet} \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-focal-${LLVM} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + apt update ${quiet} && \ apt install -y -V ${quiet} \ build-essential \ cmake \ diff --git a/dev/tasks/linux-packages/package-task.rb b/dev/tasks/linux-packages/package-task.rb index 1762e9fd39bb5..7fa7af10b3e7e 100644 --- a/dev/tasks/linux-packages/package-task.rb +++ b/dev/tasks/linux-packages/package-task.rb @@ -54,7 +54,7 @@ def initialize(package, version, release_time, options={}) @rpm_version = @version @rpm_release = "1" end - @deb_release = "1" + @deb_release = ENV["DEB_RELEASE"] || "1" end def define diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 581e174fefbfe..9cda368d8654d 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1599,12 +1599,12 @@ tasks: UBUNTU: 18.04 run: ubuntu-cpp - test-fedora-32-cpp: + test-fedora-33-cpp: ci: circle template: docker-tests/circle.linux.yml params: env: - FEDORA: 32 + FEDORA: 33 run: fedora-cpp test-ubuntu-18.04-cpp-release: @@ -1719,12 +1719,12 @@ tasks: UBUNTU: 18.04 run: ubuntu-python - test-fedora-32-python-3: + test-fedora-33-python-3: ci: azure template: docker-tests/azure.linux.yml params: env: - FEDORA: 32 + FEDORA: 33 run: fedora-python test-r-linux-as-cran: diff --git a/docker-compose.yml b/docker-compose.yml index 90fdb970c2d04..d130ebcf42ee5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -332,7 +332,7 @@ services: # docker-compose run --rm fedora-cpp # Parameters: # ARCH: amd64, arm64v8, ... - # FEDORA: 32 + # FEDORA: 33 image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp build: context: . @@ -572,7 +572,7 @@ services: # docker-compose run --rm fedora-python # Parameters: # ARCH: amd64, arm64v8, ... - # FEDORA: 32 + # FEDORA: 33 image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3 build: context: . From f6e8b68f8ea4d762810590c6b1bd9e577bff1b96 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 04:19:16 +0900 Subject: [PATCH 02/14] Add apt-line for LLVM on Ubuntu 20.04 --- ci/docker/ubuntu-20.04-cpp.dockerfile | 12 ++++++++++++ .../apache-arrow/apt/ubuntu-focal/Dockerfile | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index d2a3fab34b985..7e83ea20b38e7 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -31,6 +31,18 @@ RUN echo "debconf debconf/frontend select Noninteractive" | \ ARG clang_tools ARG llvm RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -ge 10 ]; then \ + echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \ + /etc/apt/sources.list.d/clang-tools.list; \ + fi && \ apt-get install -y -q --no-install-recommends \ clang-${clang_tools} \ clang-${llvm} \ diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile index a861c41e48e49..97f4412be7230 100644 --- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile +++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile @@ -39,7 +39,7 @@ RUN \ gnupg \ wget && \ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-focal-${LLVM} main" > \ + echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM} main" > \ /etc/apt/sources.list.d/llvm.list && \ apt update ${quiet} && \ apt install -y -V ${quiet} \ From c5219bcd8371e2f8f05ddca80690025b865c34e0 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 04:33:23 +0900 Subject: [PATCH 03/14] Add missing "apt-get update" --- ci/docker/ubuntu-20.04-cpp.dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 7e83ea20b38e7..e6a6c46987015 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -43,6 +43,7 @@ RUN apt-get update -y -q && \ echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \ /etc/apt/sources.list.d/clang-tools.list; \ fi && \ + apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ clang-${clang_tools} \ clang-${llvm} \ From 2afb64e06811a3e7eb4e60f3ae29b26a4e72d270 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 04:36:01 +0900 Subject: [PATCH 04/14] Enable output-on-failure --- dev/tasks/gandiva-jars/build-cpp-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/gandiva-jars/build-cpp-linux.sh b/dev/tasks/gandiva-jars/build-cpp-linux.sh index 24402fc494560..20f9d32682ce3 100755 --- a/dev/tasks/gandiva-jars/build-cpp-linux.sh +++ b/dev/tasks/gandiva-jars/build-cpp-linux.sh @@ -64,7 +64,7 @@ cmake -DCMAKE_BUILD_TYPE=Release \ -DBOOST_ROOT=/arrow_boost_dist \ -GNinja /arrow/cpp ninja install -ninja test +CTEST_OUTPUT_ON_FAILURE=1 ninja test popd From 0890cf1fa11a20e412d8f7df1e12717636e86eb3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 04:52:52 +0900 Subject: [PATCH 05/14] Use system LLVM on not amd64 --- .travis.yml | 2 ++ ci/docker/ubuntu-20.04-cpp.dockerfile | 26 ++++++++++++++------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72eed9bd41092..d69ffc915a7a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,6 +65,7 @@ jobs: -e ARROW_GANDIVA=OFF -e ARROW_PARQUET=OFF " + LLVM: "10" UBUNTU: "20.04" - name: "C++ on s390x" @@ -90,6 +91,7 @@ jobs: -e cares_SOURCE=BUNDLED -e gRPC_SOURCE=BUNDLED " + LLVM: "10" UBUNTU: "20.04" - name: "Go on s390x" diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index e6a6c46987015..fbcda444915fd 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -30,18 +30,20 @@ RUN echo "debconf debconf/frontend select Noninteractive" | \ # while debugging package list with docker build. ARG clang_tools ARG llvm -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - ca-certificates \ - gnupg \ - wget && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main" > \ - /etc/apt/sources.list.d/llvm.list && \ - if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -ge 10 ]; then \ - echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \ - /etc/apt/sources.list.d/clang-tools.list; \ +RUN if [ "${llvm}" -gt "10" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ + echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \ + /etc/apt/sources.list.d/clang-tools.list; \ + fi \ fi && \ apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ From 8fce607fda36a6360d8d2a29a7822fd20d836c06 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 04:55:10 +0900 Subject: [PATCH 06/14] Add comment for LLVM=10 on not amd64 --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index d69ffc915a7a7..84bbbce23dbfe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,6 +65,8 @@ jobs: -e ARROW_GANDIVA=OFF -e ARROW_PARQUET=OFF " + # The LLVM's APT repository provides only arm64 binaries. + # We should use LLVM provided by Ubuntu. LLVM: "10" UBUNTU: "20.04" @@ -91,6 +93,8 @@ jobs: -e cares_SOURCE=BUNDLED -e gRPC_SOURCE=BUNDLED " + # The LLVM's APT repository provides only arm64 binaries. + # We should use LLVM provided by Ubuntu. LLVM: "10" UBUNTU: "20.04" From c507e25dc29816d76f784c9ad7bae6c014ce3274 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 05:04:23 +0900 Subject: [PATCH 07/14] RapidJSON is used by arrow/testing/json_internal.cc --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index cc37a3c2dfe19..d084f346803b5 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1824,6 +1824,7 @@ macro(build_rapidjson) set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_PREFIX}/include") add_dependencies(toolchain rapidjson_ep) + add_dependencies(toolchain-tests rapidjson_ep) add_dependencies(rapidjson rapidjson_ep) set(RAPIDJSON_VENDORED TRUE) From 6275cc49145b3b18a9041eacfe24fbd43cbf452b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 05:10:54 +0900 Subject: [PATCH 08/14] Use Ninja in MinGW build --- .github/workflows/cpp.yml | 1 - .github/workflows/ruby.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 5ed1605a2a872..4e3e27016c18d 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -338,7 +338,6 @@ jobs: CMAKE_ARGS: >- -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} -DBoost_NO_BOOST_CMAKE=ON - CMAKE_GENERATOR: MSYS Makefiles CMAKE_UNITY_BUILD: ON steps: - name: Disable Crash Dialogs diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index d7c81bc762654..0558ac8a2cf34 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -188,7 +188,6 @@ jobs: CMAKE_ARGS: >- -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} -DBoost_NO_BOOST_CMAKE=ON - CMAKE_GENERATOR: MSYS Makefiles CMAKE_UNITY_BUILD: ON steps: - name: Disable Crash Dialogs From 790413710ce2e9a416078e1ea14d85bda74a6aa2 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 05:14:17 +0900 Subject: [PATCH 09/14] Ensure installing lsb-release --- ci/docker/linux-apt-c-glib.dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/linux-apt-c-glib.dockerfile b/ci/docker/linux-apt-c-glib.dockerfile index dbd13045521e1..12c6e23a00d8e 100644 --- a/ci/docker/linux-apt-c-glib.dockerfile +++ b/ci/docker/linux-apt-c-glib.dockerfile @@ -25,6 +25,7 @@ RUN apt-get update -y -q && \ gtk-doc-tools \ libgirepository1.0-dev \ libglib2.0-doc \ + lsb-release \ luarocks \ pkg-config \ ruby-dev && \ From b02ec0f643abc7cae47e123d23bc0e820bbc09cb Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 05:17:37 +0900 Subject: [PATCH 10/14] Use Python 3.8 for building NuGet package --- dev/tasks/nuget-packages/github.linux.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/tasks/nuget-packages/github.linux.yml b/dev/tasks/nuget-packages/github.linux.yml index c8143e432c1e3..95c4464355e33 100644 --- a/dev/tasks/nuget-packages/github.linux.yml +++ b/dev/tasks/nuget-packages/github.linux.yml @@ -40,7 +40,9 @@ jobs: git -C arrow checkout FETCH_HEAD git -C arrow submodule update --init --recursive - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 + with: + python-version: 3.8 - name: Setup Archery run: pip install -e arrow/dev/archery[docker] - name: Prepare version From 660ec5b9b404eeac31255b7e79850bd8d3c31d10 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 05:25:16 +0900 Subject: [PATCH 11/14] Install Ninja --- ci/scripts/msys2_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh index 110b03bae6cc1..45b71ca087b43 100755 --- a/ci/scripts/msys2_setup.sh +++ b/ci/scripts/msys2_setup.sh @@ -24,7 +24,6 @@ target=$1 packages=() case "${target}" in cpp|c_glib|ruby) - packages+=(make) packages+=(${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp) packages+=(${MINGW_PACKAGE_PREFIX}-ccache) packages+=(${MINGW_PACKAGE_PREFIX}-boost) @@ -37,6 +36,7 @@ case "${target}" in packages+=(${MINGW_PACKAGE_PREFIX}-gtest) packages+=(${MINGW_PACKAGE_PREFIX}-llvm) packages+=(${MINGW_PACKAGE_PREFIX}-lz4) + packages+=(${MINGW_PACKAGE_PREFIX}-ninja) packages+=(${MINGW_PACKAGE_PREFIX}-polly) packages+=(${MINGW_PACKAGE_PREFIX}-protobuf) packages+=(${MINGW_PACKAGE_PREFIX}-python3-numpy) From fd073199faa1505db19224f039e8ba35a690364d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 06:38:17 +0900 Subject: [PATCH 12/14] Use Python 3.9 --- dev/tasks/homebrew-formulae/apache-arrow.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index 72d4cfa75d272..888e2a357a878 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -16,7 +16,7 @@ class ApacheArrow < Formula depends_on "numpy" depends_on "openssl@1.1" depends_on "protobuf" - depends_on "python@3.8" + depends_on "python@3.9" depends_on "rapidjson" depends_on "snappy" depends_on "thrift" @@ -42,7 +42,7 @@ def install -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_INSTALL_NAME_RPATH=OFF - -DPYTHON_EXECUTABLE=#{Formula["python@3.8"].bin/"python3"} + -DPython3_EXECUTABLE=#{Formula["python@3.9"].bin/"python3"} ] # Re-enable -DARROW_S3=ON and add back aws-sdk-cpp to depends_on in ARROW-6437 From 88441d55c1604f49c2cff978de8148bc3773c26b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 25 Oct 2020 06:50:14 +0900 Subject: [PATCH 13/14] Try adding libc++ for LLVM --- ci/scripts/msys2_setup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh index 45b71ca087b43..d776e03a99828 100755 --- a/ci/scripts/msys2_setup.sh +++ b/ci/scripts/msys2_setup.sh @@ -25,15 +25,17 @@ packages=() case "${target}" in cpp|c_glib|ruby) packages+=(${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp) - packages+=(${MINGW_PACKAGE_PREFIX}-ccache) packages+=(${MINGW_PACKAGE_PREFIX}-boost) packages+=(${MINGW_PACKAGE_PREFIX}-brotli) + packages+=(${MINGW_PACKAGE_PREFIX}-ccache) packages+=(${MINGW_PACKAGE_PREFIX}-clang) packages+=(${MINGW_PACKAGE_PREFIX}-cmake) packages+=(${MINGW_PACKAGE_PREFIX}-gcc) packages+=(${MINGW_PACKAGE_PREFIX}-gflags) packages+=(${MINGW_PACKAGE_PREFIX}-grpc) packages+=(${MINGW_PACKAGE_PREFIX}-gtest) + packages+=(${MINGW_PACKAGE_PREFIX}-libc++) + packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc) packages+=(${MINGW_PACKAGE_PREFIX}-llvm) packages+=(${MINGW_PACKAGE_PREFIX}-lz4) packages+=(${MINGW_PACKAGE_PREFIX}-ninja) @@ -43,7 +45,6 @@ case "${target}" in packages+=(${MINGW_PACKAGE_PREFIX}-rapidjson) packages+=(${MINGW_PACKAGE_PREFIX}-snappy) packages+=(${MINGW_PACKAGE_PREFIX}-thrift) - packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc) packages+=(${MINGW_PACKAGE_PREFIX}-zlib) packages+=(${MINGW_PACKAGE_PREFIX}-zstd) ;; From f136c6d016bbea7ff0651bce3391ff1e1a52c5cd Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 26 Oct 2020 05:49:54 +0900 Subject: [PATCH 14/14] Remove needless libc++ --- ci/scripts/msys2_setup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh index d776e03a99828..51dc45fc3a215 100755 --- a/ci/scripts/msys2_setup.sh +++ b/ci/scripts/msys2_setup.sh @@ -34,7 +34,6 @@ case "${target}" in packages+=(${MINGW_PACKAGE_PREFIX}-gflags) packages+=(${MINGW_PACKAGE_PREFIX}-grpc) packages+=(${MINGW_PACKAGE_PREFIX}-gtest) - packages+=(${MINGW_PACKAGE_PREFIX}-libc++) packages+=(${MINGW_PACKAGE_PREFIX}-libutf8proc) packages+=(${MINGW_PACKAGE_PREFIX}-llvm) packages+=(${MINGW_PACKAGE_PREFIX}-lz4)