diff --git a/.env b/.env
index aa9e9c7667078..e14e998f83afc 100644
--- a/.env
+++ b/.env
@@ -41,7 +41,7 @@ DEBIAN=10
 UBUNTU=18.04
 FEDORA=33
 PYTHON=3.6
-LLVM=11
+LLVM=12
 CLANG_TOOLS=8
 RUST=nightly-2020-11-24
 GO=1.12
diff --git a/.travis.yml b/.travis.yml
index 2cf70cca982ff..26b4d78fd2da0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -68,7 +68,7 @@ jobs:
           -e CMAKE_UNITY_BUILD=ON
           -e CPP_MAKE_PARALLELISM=4
           "
-        # The LLVM's APT repository provides only arm64 binaries.
+        # The LLVM's APT repository doesn't provide arm64 binaries.
         # We should use LLVM provided by Ubuntu.
         LLVM: "10"
         UBUNTU: "20.04"
@@ -100,9 +100,6 @@ jobs:
           -e cares_SOURCE=BUNDLED
           -e gRPC_SOURCE=BUNDLED
           "
-        # The LLVM's APT repository provides only arm64 binaries.
-        # We should use LLVM provided by Ubuntu.
-        LLVM: "10"
         UBUNTU: "20.04"
 
     - name: "Go on s390x"
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f60469169c13a..32f6dc00f3043 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -103,6 +103,7 @@ set(ARROW_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}")
 set(ARROW_DOC_DIR "share/doc/${PROJECT_NAME}")
 
 set(ARROW_LLVM_VERSIONS
+    "12.0"
     "11.1"
     "11.0"
     "10"
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile
index 20ddef188a29f..77abaf69d4df8 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile
@@ -73,8 +73,8 @@ RUN \
     tzdata \
     zlib1g-dev && \
   apt install -y -V -t buster-backports ${quiet} \
-    clang-8 \
-    llvm-8-dev && \
+    clang-11 \
+    llvm-11-dev && \
   if apt list | grep '^nvidia-cuda-toolkit/'; then \
     apt install -y -V ${quiet} nvidia-cuda-toolkit; \
   fi && \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
index c1404da6e927e..8fe70edb6ad0e 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile
@@ -34,7 +34,8 @@ RUN \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     build-essential \
-    clang-10 \
+    ccache \
+    clang \
     cmake \
     debhelper \
     devscripts \
@@ -57,7 +58,7 @@ RUN \
     libthrift-dev \
     libutf8proc-dev \
     libzstd-dev \
-    llvm-10-dev \
+    llvm-dev \
     lsb-release \
     ninja-build \
     pkg-config \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/Dockerfile
index 7b9cf7e16e739..5209be29fb56a 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/Dockerfile
@@ -34,7 +34,8 @@ RUN \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     build-essential \
-    clang-11 \
+    ccache \
+    clang \
     cmake \
     debhelper \
     devscripts \
@@ -58,7 +59,7 @@ RUN \
     libthrift-dev \
     libutf8proc-dev \
     libzstd-dev \
-    llvm-11-dev \
+    llvm-dev \
     lsb-release \
     ninja-build \
     pkg-config \
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-xenial/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-xenial/Dockerfile
index 15ed0fee40704..354168843e411 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-xenial/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-xenial/Dockerfile
@@ -35,7 +35,8 @@ RUN \
   apt install -y -V ${quiet} \
     autoconf-archive \
     build-essential \
-    clang-8 \
+    ccache \
+    clang \
     cmake \
     devscripts \
     dh-autoreconf \
@@ -57,8 +58,13 @@ RUN \
     libsnappy-dev \
     libssl-dev \
     libutf8proc-dev \
+<<<<<<< HEAD:dev/tasks/linux-packages/apache-arrow/apt/ubuntu-xenial/Dockerfile
     libzstd1-dev \
     llvm-8-dev \
+=======
+    libzstd-dev \
+    llvm-dev \
+>>>>>>> c408048d1... ARROW-12467: [C++][Gandiva] Add support for LLVM12:dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile
     lsb-release \
     pkg-config \
     protobuf-compiler \