Gandiva C++ Merge.

cpp/src/gandiva/CMakeLists.txt

@@ -0,0 +1,42 @@
+# Copyright (C) 2017-2018 Dremio Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.10)
+
+project(gandiva)
+
+# LLVM/Clang is required by multiple subdirs.
+find_package(LLVM)
+
+# Set the path where the byte-code files will be installed.
+set(GANDIVA_BC_INSTALL_DIR
+ ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/gandiva)
+
+set(GANDIVA_BC_FILE_NAME irhelpers.bc)
+set(GANDIVA_BC_INSTALL_PATH ${GANDIVA_BC_INSTALL_DIR}/${GANDIVA_BC_FILE_NAME})
+set(GANDIVA_BC_OUTPUT_PATH ${CMAKE_BINARY_DIR}/${GANDIVA_BC_FILE_NAME})
+
+# Set the path where the so lib file will be installed.
+if (APPLE)
+ set(GANDIVA_HELPER_LIB_FILE_NAME libgandiva_helpers.dylib)
+else()
+ set(GANDIVA_HELPER_LIB_FILE_NAME libgandiva_helpers.so)
+endif(APPLE)
+
+set(GANDIVA_HELPER_LIB_INSTALL_PATH ${GANDIVA_BC_INSTALL_DIR}/${GANDIVA_HELPER_LIB_FILE_NAME})
+set(GANDIVA_HELPER_LIB_OUTPUT_PATH ${CMAKE_BINARY_DIR}/src/codegen/${GANDIVA_HELPER_LIB_FILE_NAME})
+
+add_subdirectory(codegen)
+add_subdirectory(jni)
+add_subdirectory(precompiled)

cpp/src/gandiva/README.md

@@ -0,0 +1,68 @@
+# Gandiva C++
+
+## System setup
+
+Gandiva uses CMake as a build configuration system. Currently, it supports
+out-of-source builds only.
+
+Build Gandiva requires:
+
+* A C++11-enabled compiler. On Linux, gcc 4.8 and higher should be sufficient.
+* CMake
+* LLVM
+* Arrow
+* Boost
+* Protobuf
+
+On macOS, you can use [Homebrew][1]:
+
+```shell
+brew install cmake llvm boost protobuf
+```
+
+To install arrow, follow the steps in the [arrow Readme][2].
+## Building Gandiva
+
+Debug build :
+
+```shell
+git clone https:/dremio/gandiva.git
+cd gandiva/cpp
+mkdir debug
+cd debug
+cmake ..
+make
+ctest
+```
+
+Release build :
+
+```shell
+git clone https:/dremio/gandiva.git
+cd gandiva/cpp
+mkdir release
+cd release
+cmake .. -DCMAKE_BUILD_TYPE=Release
+make
+ctest
+```
+
+## Validating code style
+
+We follow the [google cpp code style][3]. To validate compliance,
+
+```shell
+cd debug
+make stylecheck
+```
+
+## Fixing code style
+
+```shell
+cd debug
+make stylefix
+```
+
+[1]: https://brew.sh/
+[2]: https:/apache/arrow/tree/master/cpp
+[3]: https://google.github.io/styleguide/cppguide.html

cpp/src/gandiva/codegen/CMakeLists.txt

@@ -0,0 +1,125 @@
+# Copyright (C) 2017-2018 Dremio Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+project(gandiva)
+
+# Find arrow
+find_package(ARROW)
+
+find_package(Boost COMPONENTS system regex filesystem REQUIRED)
+
+set(BC_FILE_PATH_CC "${CMAKE_CURRENT_BINARY_DIR}/bc_file_path.cc")
+configure_file(bc_file_path.cc.in ${BC_FILE_PATH_CC})
+
+# helper files that are shared between libgandiva and libgandiva_helpers
+set(SHARED_HELPER_FILES
+ like_holder.cc
+ regex_util.cc)
+
+set(SRC_FILES annotator.cc
+ bitmap_accumulator.cc
+ configuration.cc
+ engine.cc
+ expr_decomposer.cc
+ expr_validator.cc
+ expression.cc
+ expression_registry.cc
+ filter.cc
+ function_registry.cc
+ function_signature.cc
+ llvm_generator.cc
+ llvm_types.cc
+ projector.cc
+ selection_vector.cc
+ tree_expr_builder.cc
+ ${SHARED_HELPER_FILES}
+ ${BC_FILE_PATH_CC})
+
+add_library(gandiva_obj_lib OBJECT ${SRC_FILES})
+
+# set PIC so that object library can be included in shared libs.
+set_target_properties(gandiva_obj_lib PROPERTIES POSITION_INDEPENDENT_CODE 1)
+
+# For users of gandiva library (including integ tests), include-dir is :
+# /usr/**/include dir after install,
+# cpp/include during build
+# For building gandiva library itself, include-dir (in addition to above) is :
+# cpp/src
+target_include_directories(gandiva_obj_lib
+ PUBLIC
+ $<INSTALL_INTERFACE:include>
+ $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include>
+ PRIVATE
+ ${CMAKE_SOURCE_DIR}/src
+ $<TARGET_PROPERTY:LLVM::LLVM_INTERFACE,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:ARROW::ARROW_SHARED,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
+ $<TARGET_PROPERTY:gtest_main,INCLUDE_DIRECTORIES>
+)
+
+build_gandiva_lib("shared")
+
+build_gandiva_lib("static")
+
+# install for gandiva
+include(GNUInstallDirs)
+
+# install libgandiva
+install(
+ TARGETS gandiva_shared gandiva_static
+ DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
+
+# install the header files.
+install(
+ DIRECTORY ${CMAKE_SOURCE_DIR}/include/gandiva
+ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+)
+
+# Pre-compiled .so library for function helpers.
+add_library(gandiva_helpers SHARED
+ ${SHARED_HELPER_FILES}
+ function_holder_stubs.cc)
+
+target_compile_definitions(gandiva_helpers
+ PRIVATE -DGDV_HELPERS
+)
+
+target_include_directories(gandiva_helpers
+ PRIVATE
+ ${CMAKE_SOURCE_DIR}/include
+ ${CMAKE_SOURCE_DIR}/src
+ $<TARGET_PROPERTY:ARROW::ARROW_SHARED,INTERFACE_INCLUDE_DIRECTORIES>
+)
+
+target_link_libraries(gandiva_helpers PRIVATE Boost::boost)
+if (NOT APPLE)
+ target_link_libraries(gandiva_helpers LINK_PRIVATE -static-libstdc++ -static-libgcc)
+endif()
+
+#args: label test-file src-files
+add_gandiva_unit_test(bitmap_accumulator_test.cc bitmap_accumulator.cc)
+add_gandiva_unit_test(engine_llvm_test.cc engine.cc llvm_types.cc configuration.cc ${BC_FILE_PATH_CC})
+add_gandiva_unit_test(function_signature_test.cc function_signature.cc)
+add_gandiva_unit_test(function_registry_test.cc function_registry.cc function_signature.cc)
+add_gandiva_unit_test(llvm_types_test.cc llvm_types.cc)
+add_gandiva_unit_test(llvm_generator_test.cc llvm_generator.cc regex_util.cc engine.cc llvm_types.cc expr_decomposer.cc function_registry.cc annotator.cc bitmap_accumulator.cc configuration.cc function_signature.cc like_holder.cc regex_util.cc ${BC_FILE_PATH_CC})
+add_gandiva_unit_test(annotator_test.cc annotator.cc function_signature.cc)
+add_gandiva_unit_test(tree_expr_test.cc tree_expr_builder.cc expr_decomposer.cc annotator.cc function_registry.cc function_signature.cc like_holder.cc regex_util.cc)
+add_gandiva_unit_test(expr_decomposer_test.cc expr_decomposer.cc tree_expr_builder.cc annotator.cc function_registry.cc function_signature.cc like_holder.cc regex_util.cc)
+add_gandiva_unit_test(status_test.cc)
+add_gandiva_unit_test(expression_registry_test.cc llvm_types.cc expression_registry.cc function_signature.cc function_registry.cc)
+add_gandiva_unit_test(selection_vector_test.cc selection_vector.cc)
+add_gandiva_unit_test(lru_cache_test.cc)
+add_gandiva_unit_test(like_holder_test.cc like_holder.cc regex_util.cc)

cpp/src/gandiva/codegen/annotator.cc

@@ -0,0 +1,105 @@
+// Copyright (C) 2017-2018 Dremio Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "codegen/annotator.h"
+
+#include <memory>
+#include <string>
+
+#include "codegen/field_descriptor.h"
+
+namespace gandiva {
+
+FieldDescriptorPtr Annotator::CheckAndAddInputFieldDescriptor(FieldPtr field) {
+ // If the field is already in the map, return the entry.
+ auto found = in_name_to_desc_.find(field->name());
+ if (found != in_name_to_desc_.end()) {
+ return found->second;
+ }
+
+ auto desc = MakeDesc(field);
+ in_name_to_desc_[field->name()] = desc;
+ return desc;
+}
+
+FieldDescriptorPtr Annotator::AddOutputFieldDescriptor(FieldPtr field) {
+ auto desc = MakeDesc(field);
+ out_descs_.push_back(desc);
+ return desc;
+}
+
+FieldDescriptorPtr Annotator::MakeDesc(FieldPtr field) {
+ // TODO:
+ // - validity is optional
+ int data_idx = buffer_count_++;
+ int validity_idx = buffer_count_++;
+ int offsets_idx = FieldDescriptor::kInvalidIdx;
+ if (arrow::is_binary_like(field->type()->id())) {
+ offsets_idx = buffer_count_++;
+ }
+ return std::make_shared<FieldDescriptor>(field, data_idx, validity_idx, offsets_idx);
+}
+
+void Annotator::PrepareBuffersForField(const FieldDescriptor &desc,
+ const arrow::ArrayData &array_data,
+ EvalBatch *eval_batch) {
+ int buffer_idx = 0;
+
+ // TODO:
+ // - validity is optional
+
+ uint8_t *validity_buf = const_cast<uint8_t *>(array_data.buffers[buffer_idx]->data());
+ eval_batch->SetBuffer(desc.validity_idx(), validity_buf);
+ ++buffer_idx;
+
+ if (desc.HasOffsetsIdx()) {
+ uint8_t *offsets_buf = const_cast<uint8_t *>(array_data.buffers[buffer_idx]->data());
+ eval_batch->SetBuffer(desc.offsets_idx(), offsets_buf);
+ ++buffer_idx;
+ }
+
+ uint8_t *data_buf = const_cast<uint8_t *>(array_data.buffers[buffer_idx]->data());
+ eval_batch->SetBuffer(desc.data_idx(), data_buf);
+ ++buffer_idx;
+}
+
+EvalBatchPtr Annotator::PrepareEvalBatch(const arrow::RecordBatch &record_batch,
+ const ArrayDataVector &out_vector) {
+ EvalBatchPtr eval_batch = std::make_shared<EvalBatch>(
+ record_batch.num_rows(), buffer_count_, local_bitmap_count_);
+
+ // Fill in the entries for the input fields.
+ for (int i = 0; i < record_batch.num_columns(); ++i) {
+ const std::string &name = record_batch.column_name(i);
+ auto found = in_name_to_desc_.find(name);
+ if (found == in_name_to_desc_.end()) {
+ // skip columns not involved in the expression.
+ continue;
+ }
+
+ PrepareBuffersForField(*(found->second), *(record_batch.column(i))->data(),
+ eval_batch.get());
+ }
+
+ // Fill in the entries for the output fields.
+ int idx = 0;
+ for (auto &arraydata : out_vector) {
+ const FieldDescriptorPtr &desc = out_descs_.at(idx);
+ PrepareBuffersForField(*desc, *arraydata, eval_batch.get());
+ ++idx;
+ }
+ return eval_batch;
+}
+
+} // namespace gandiva