Separated format helper

gchq · Oct 18, 2024 · 65c1bbb · 65c1bbb
1 parent c36fbbc
commit 65c1bbb
Show file tree

Hide file tree

Showing 14 changed files with 44 additions and 18 deletions.
diff --git a/cpp/include/cudf_compact/cudf_compact.hpp b/cpp/include/cudf_compact/cudf_compact.hpp
@@ -17,7 +17,7 @@
 
 #include "cudf_compact/common_types.hpp"
 #include "cudf_compact/cudf_utils.hpp"
-#include "cudf_compact/format_helper.hpp"
+#include "format_helper/format_helper.hpp"
 #include "cudf_compact/parquet_types.h"
 
 #include <algorithm>// std::ranges::equal

diff --git a/cpp/include/cudf_compact/cudf_utils.hpp b/cpp/include/cudf_compact/cudf_utils.hpp
@@ -7,7 +7,7 @@
 #include <fmt/core.h>
 #include <spdlog/spdlog.h>
 
-#include "cudf_compact/s3/s3_sink.hpp"
+#include "s3/s3_sink.hpp"
 
 #include <concepts>
 #include <exception>

diff --git a/cpp/include/cudf_compact/format_helper.hpp → cpp/include/format_helper/format_helper.hpp b/cpp/include/cudf_compact/format_helper.hpp → cpp/include/format_helper/format_helper.hpp
diff --git a/cpp/include/s3/s3_utils.hpp b/cpp/include/s3/s3_utils.hpp
@@ -6,7 +6,7 @@
 #include <aws/s3/S3Client.h>
 #include <spdlog/spdlog.h>
 
-#include "cudf_compact/format_helper.hpp"
+#include "format_helper/format_helper.hpp"
 
 #include <concepts>
 #include <exception>

diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_subdirectory(cmdline)
+add_subdirectory(format_helper)
 add_subdirectory(s3)
 add_subdirectory(cudf_compact)
-add_subdirectory(compact_grpc)
+add_subdirectory(compact_grpc)
diff --git a/cpp/src/cmdline/main.cpp b/cpp/src/cmdline/main.cpp
@@ -1,6 +1,6 @@
 #include "configure_logging.hpp"
 #include "cudf_compact/cudf_compact.hpp"
-#include "cudf_compact/format_helper.hpp"
+#include "format_helper/format_helper.hpp"
 
 #ifdef SPDLOG_ACTIVE_LEVEL
 #undef SPDLOG_ACTIVE_LEVEL

diff --git a/cpp/src/cudf_compact/CMakeLists.txt b/cpp/src/cudf_compact/CMakeLists.txt
@@ -5,21 +5,19 @@ add_library(
  cudf_compact.cpp
  cudf_utils.cpp
  filters.cpp
- format_helper.cpp
  parquet_types.cpp
  ranges.cu
  ../../include/cudf_compact/common_types.hpp
  ../../include/cudf_compact/cudf_compact.hpp
  ../../include/cudf_compact/cudf_utils.hpp
  ../../include/cudf_compact/filters.hpp
- ../../include/cudf_compact/format_helper.hpp
  ../../include/cudf_compact/metadata.hpp
  ../../include/cudf_compact/parquet_types.h
  ../../include/cudf_compact/ranges.hpp
 )
 
 add_library(gpu_compact::cudf_compact ALIAS cudf_compact)
-target_link_libraries(cudf_compact PRIVATE fmt::fmt spdlog::spdlog gpu_compact::s3)
+target_link_libraries(cudf_compact PRIVATE fmt::fmt spdlog::spdlog gpu_compact::s3 gpu_compact::format_helper)
 target_link_libraries(cudf_compact PUBLIC cudf::cudf thrift::thrift)
 target_link_libraries(cudf_compact PRIVATE gpu_compact_options gpu_compact_warnings)
 target_include_directories(cudf_compact ${WARNING_GUARD} PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>

diff --git a/cpp/src/cudf_compact/cudf_compact.cpp b/cpp/src/cudf_compact/cudf_compact.cpp
@@ -15,11 +15,11 @@
 
 #include "cudf_compact/common_types.hpp"
 #include "cudf_compact/filters.hpp"
-#include "cudf_compact/format_helper.hpp"
 #include "cudf_compact/metadata.hpp"
 #include "cudf_compact/parquet_types.h"
 #include "cudf_compact/ranges.hpp"
-#include "cudf_compact/s3/s3_utils.hpp"
+#include "format_helper/format_helper.hpp"
+#include "s3/s3_utils.hpp"
 
 #include <algorithm>// std::reduce
 #include <cstddef>
@@ -244,7 +244,6 @@ std::optional<size_t> page_size(col_schema const &schema,
 }
 
 CompactionResult mergeSortedS3Files(CompactionInput const &details, std::size_t gpuChunk) {
- imbueCout();
  auto s3client = gpu_compact::cudf_compact::s3::makeClient();
  // TODO this should be read from compaction input details, will consist of row keys and sort keys
  // get the column to use for ranges

diff --git a/cpp/src/cudf_compact/filters.cpp b/cpp/src/cudf_compact/filters.cpp
@@ -7,7 +7,7 @@
 #include <cudf/utilities/type_dispatcher.hpp>
 #include <spdlog/spdlog.h>
 
-#include "cudf_compact/format_helper.hpp"
+#include "format_helper/format_helper.hpp"
 
 #include <exception>
 

diff --git a/cpp/src/format_helper/CMakeLists.txt b/cpp/src/format_helper/CMakeLists.txt
@@ -0,0 +1,28 @@
+include(GenerateExportHeader)
+
+add_library(format_helper format_helper.cpp ../../include/format_helper/format_helper.hpp)
+
+add_library(gpu_compact::format_helper ALIAS format_helper)
+target_link_libraries(format_helper PRIVATE fmt::fmt)
+target_link_libraries(format_helper PRIVATE gpu_compact_options gpu_compact_warnings)
+target_include_directories(format_helper ${WARNING_GUARD} PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
+ $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/include>)
+target_compile_features(format_helper PUBLIC cxx_std_20)
+# target_link_options(
+# format_helper
+# BEFORE
+# INTERFACE
+# "LINKER:--copy-dt-needed-entries")
+
+set_target_properties(
+ format_helper
+ PROPERTIES VERSION ${PROJECT_VERSION}
+ CXX_VISIBILITY_PRESET hidden
+ VISIBILITY_INLINES_HIDDEN YES)
+
+generate_export_header(format_helper EXPORT_FILE_NAME
+ ${PROJECT_BINARY_DIR}/include/gpu_compact/format_helper_export.hpp)
+
+if(NOT BUILD_SHARED_LIBS)
+ target_compile_definitions(format_helper PUBLIC CUDF_COMPACT_STATIC_DEFINE)
+endif()
diff --git a/cpp/src/cudf_compact/format_helper.cpp → cpp/src/format_helper/format_helper.cpp b/cpp/src/cudf_compact/format_helper.cpp → cpp/src/format_helper/format_helper.cpp
@@ -1,5 +1,5 @@
 
-#include "cudf_compact/format_helper.hpp"
+#include "format_helper/format_helper.hpp"
 
 #include <iostream>
 #include <locale>

diff --git a/cpp/src/s3/s3_sink.cpp b/cpp/src/s3/s3_sink.cpp
@@ -5,9 +5,9 @@
 #include <aws/s3/model/UploadPartRequest.h>
 #include <spdlog/spdlog.h>
 
-#include "cudf_compact/format_helper.hpp"
-#include "cudf_compact/s3/s3_sink.hpp"
-#include "cudf_compact/s3/s3_utils.hpp"
+#include "format_helper/format_helper.hpp"
+#include "s3/s3_sink.hpp"
+#include "s3/s3_utils.hpp"
 
 #include <algorithm>
 #include <exception>

diff --git a/cpp/src/s3/s3_utils.cpp b/cpp/src/s3/s3_utils.cpp
@@ -1,4 +1,4 @@
-#include "cudf_compact/s3/s3_utils.hpp"
+#include "s3/s3_utils.hpp"
 
 #include <fmt/core.h>
 

diff --git a/java/compaction/compaction-gpu/docker/Dockerfile b/java/compaction/compaction-gpu/docker/Dockerfile
@@ -60,7 +60,7 @@ RUN wget -q "https:/Kitware/CMake/releases/download/v3.30.5/cmake-3.
 
 # Install include-what-you-use
 RUN git clone https:/include-what-you-use/include-what-you-use.git -b clang_17 \
- && /opt/cmake/bin/cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-18 -S include-what-you-use/ -B build \
+ && /opt/cmake/bin/cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-17 -S include-what-you-use/ -B build \
  && cd build \
  && make -j $(nproc) \
  && make install \