Update to RAPIDS 24.10 (#494)

Closes #478
Requires nv-morpheus/utilities#75

Authors:
  - Christopher Harris (https:/cwharris)
  - Michael Demoret (https:/mdemoret-nv)

Approvers:
  - Anuradha Karuppiah (https:/AnuradhaKaruppiah)
  - David Gardner (https:/dagardner-nv)

URL: #494

.devcontainer/opt/mrc/bin/post-attach-command.sh

@@ -28,6 +28,6 @@ sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;
 
 if conda_env_find "${ENV_NAME}" ; \
 
-then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml --prune; \
-else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml; \
+then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-125_arch-x86_64.yaml --prune; \
+else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-125_arch-x86_64.yaml; \
 fi

.github/workflows/pr.yaml

@@ -49,7 +49,7 @@ jobs:
  - prepare
  - ci_pipe
  secrets: inherit
- uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02
+ uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10
  prepare:
  name: Prepare
  runs-on: ubuntu-latest
@@ -72,7 +72,7 @@ jobs:
  needs: [prepare]
  if: ${{ !fromJSON(needs.prepare.outputs.has_skip_ci_label) && fromJSON(needs.prepare.outputs.is_pr )}}
  secrets: inherit
- uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02
+ uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10
  with:
  enable_check_generated_files: false
 
@@ -89,9 +89,9 @@ jobs:
  # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
  conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
  # Build container
- container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-240214
+ container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-241002
  # Test container
- test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-240214
+ test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-241002
  # Info about the PR. Empty for non PR branches. Useful for extracting PR number, title, etc.
  pr_info: ${{ needs.prepare.outputs.pr_info }}
  secrets:

CMakeLists.txt

@@ -37,7 +37,7 @@ option(MRC_USE_CONDA "Enables finding dependencies via conda. All dependencies m
  environment" ON)
 option(MRC_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
 
-set(MRC_RAPIDS_VERSION "24.02" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
+set(MRC_RAPIDS_VERSION "24.10" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
 
 set(MRC_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(MRC_CACHE_DIR)

CONTRIBUTING.md

@@ -85,7 +85,7 @@ cd $MRC_ROOT
 #### Create MRC Conda environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc --file $MRC_ROOT/conda/environments/all_cuda-121_arch-x86_64.yaml
+conda env create -n mrc --file $MRC_ROOT/conda/environments/all_cuda-125_arch-x86_64.yaml
 conda activate mrc
 ```
 #### Build MRC

Dockerfile

@@ -17,7 +17,7 @@
 
 
 ARG FROM_IMAGE="rapidsai/ci-conda"
-ARG CUDA_VER=12.1.1
+ARG CUDA_VER=12.5.1
 ARG LINUX_DISTRO=ubuntu
 ARG LINUX_VER=22.04
 ARG PYTHON_VER=3.10
@@ -45,13 +45,13 @@ RUN useradd --uid $USER_UID --gid $USER_GID -m $USERNAME && \
  echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \
  chmod 0440 /etc/sudoers.d/$USERNAME
 
-COPY ./conda/environments/all_cuda-121_arch-x86_64.yaml /opt/mrc/conda/environments/all_cuda-121_arch-x86_64.yaml
+COPY ./conda/environments/all_cuda-125_arch-x86_64.yaml /opt/mrc/conda/environments/all_cuda-125_arch-x86_64.yaml
 
 RUN --mount=type=cache,target=/opt/conda/pkgs,sharing=locked \
  echo "create env: ${PROJ_NAME}" && \
  sudo -g conda -u $USERNAME \
  CONDA_ALWAYS_YES=true \
- /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/all_cuda-121_arch-x86_64.yaml && \
+ /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/all_cuda-125_arch-x86_64.yaml && \
  chmod -R a+rwX /opt/conda && \
  rm -rf /tmp/conda
 

README.md

@@ -118,7 +118,7 @@ cd $MRC_ROOT
 #### Create MRC Conda Environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc-dev --file $MRC_ROOT/conda/environments/all_cuda-121_arch-x86_64.yaml
+conda env create -n mrc-dev --file $MRC_ROOT/conda/environments/all_cuda-125_arch-x86_64.yaml
 conda activate mrc-dev
 ```
 <!-- omit in toc -->

ci/conda/recipes/libmrc/conda_build_config.yaml

@@ -14,20 +14,20 @@
 # limitations under the License.
 
 c_compiler_version:
- - 11.2
+ - 12.1
 
 cxx_compiler_version:
- - 11.2
+ - 12.1
 
 cuda_compiler:
  - cuda-nvcc
 
 cuda_compiler_version:
- - 12.1
+ - 12.5
 
 python:
  - 3.10
 
 # Setup the dependencies to build with multiple versions of RAPIDS
 rapids_version: # Keep around compatibility with current version -2
- - 24.02
+ - 24.10

ci/conda/recipes/libmrc/meta.yaml

@@ -15,7 +15,7 @@
 
 {% set version = environ.get('GIT_VERSION', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
 {% set py_version = environ.get('CONDA_PY', '3.10') %}
-{% set cuda_version = '.'.join(environ.get('CUDA', '12.1').split('.')[:2]) %}
+{% set cuda_version = '.'.join(environ.get('CUDA', '12.5').split('.')[:2]) %}
 
 package:
  name: libmrc-split
@@ -42,9 +42,9 @@ requirements:
  - cmake =3.27
  - libtool
  - ninja =1.11
- - numactl-libs-cos7-x86_64
+ - numactl =2.0.18
  - pkg-config =0.29
- - sysroot_linux-64 >=2.17
+ - sysroot_linux-64 >=2.28
  host:
  # Libraries necessary to build. Keep sorted!
  - boost-cpp =1.84
@@ -53,8 +53,8 @@ requirements:
  - cuda-nvrtc-dev {{ cuda_version }}.*
  - cuda-version {{ cuda_version }}.*
  - doxygen 1.10.0
- - glog =0.6
- - libgrpc =1.59
+ - glog>=0.7.1,<0.8
+ - libgrpc =1.62.2
  - gtest =1.14
  - libhwloc =2.9.2
  - librmm {{ rapids_version }}
@@ -80,14 +80,14 @@ outputs:
  - {{ compiler("cuda") }}
  - {{ compiler("cxx") }}
  - cmake =3.27
- - numactl-libs-cos7-x86_64
- - sysroot_linux-64 =2.17
+ - numactl =2.0.18
+ - sysroot_linux-64 >=2.28
  host:
  # Any libraries with weak run_exports need to go here to be added to the run. Keep sorted!
  - boost-cpp =1.84
  - cuda-version # Needed to allow pin_compatible to work
- - glog =0.6
- - libgrpc =1.59
+ - glog>=0.7.1,<0.8
+ - libgrpc =1.62.2
  - libhwloc =2.9.2
  - librmm {{ rapids_version }}
  - nlohmann_json =3.11

ci/conda/recipes/run_conda_build.sh

@@ -95,7 +95,7 @@ fi
 # Choose default variants
 if hasArg quick; then
  # For quick build, just do most recent version of rapids
- CONDA_ARGS_ARRAY+=("--variants" "{rapids_version: 24.02}")
+ CONDA_ARGS_ARRAY+=("--variants" "{rapids_version: 24.10}")
 fi
 
 # And default channels (should match dependencies.yaml)

ci/scripts/github/common.sh

@@ -35,7 +35,7 @@ id
 export NUM_PROC=${PARALLEL_LEVEL:-$(nproc)}
 export BUILD_CC=${BUILD_CC:-"gcc"}
 
-export CONDA_ENV_YML="${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml"
+export CONDA_ENV_YML="${MRC_ROOT}/conda/environments/all_cuda-125_arch-x86_64.yaml"
 
 export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMRC_BUILD_BENCHMARKS=ON -DMRC_BUILD_EXAMPLES=ON -DMRC_BUILD_PYTHON=ON -DMRC_BUILD_TESTS=ON -DMRC_USE_CONDA=ON -DMRC_PYTHON_BUILD_STUBS=ON"
 export CMAKE_BUILD_WITH_CODECOV="-DCMAKE_BUILD_TYPE=Debug -DMRC_ENABLE_CODECOV=ON -DMRC_PYTHON_PERFORM_INSTALL:BOOL=ON -DMRC_PYTHON_INPLACE_BUILD:BOOL=ON"

ci/scripts/run_ci_local.sh

@@ -58,7 +58,7 @@ GIT_BRANCH=$(git branch --show-current)
 GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 BASE_LOCAL_CI_TMP=${BASE_LOCAL_CI_TMP:-${MRC_ROOT}/.tmp/local_ci_tmp}
-CONTAINER_VER=${CONTAINER_VER:-240214}
+CONTAINER_VER=${CONTAINER_VER:-241002}
 CUDA_VER=${CUDA_VER:-12.1}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 

cmake/dependencies.cmake

@@ -48,18 +48,8 @@ morpheus_utils_configure_cccl()
 # =================
 morpheus_utils_configure_rmm()
 
-# gflags
-# ======
-rapids_find_package(gflags REQUIRED
- GLOBAL_TARGETS gflags
- BUILD_EXPORT_SET ${PROJECT_NAME}-exports
- INSTALL_EXPORT_SET ${PROJECT_NAME}-exports
-)
-
 # glog
 # ====
-# - link against shared
-# - todo: compile with -DWITH_GFLAGS=OFF and remove gflags dependency
 morpheus_utils_configure_glog()
 
 # nvidia cub
@@ -73,9 +63,9 @@ find_path(CUB_INCLUDE_DIRS "cub/cub.cuh"
 # =========
 rapids_find_package(gRPC REQUIRED
  GLOBAL_TARGETS
- gRPC::address_sorting gRPC::gpr gRPC::grpc gRPC::grpc_unsecure gRPC::grpc++ gRPC::grpc++_alts gRPC::grpc++_error_details gRPC::grpc++_reflection
- gRPC::grpc++_unsecure gRPC::grpc_plugin_support gRPC::grpcpp_channelz gRPC::upb gRPC::grpc_cpp_plugin gRPC::grpc_csharp_plugin gRPC::grpc_node_plugin
- gRPC::grpc_objective_c_plugin gRPC::grpc_php_plugin gRPC::grpc_python_plugin gRPC::grpc_ruby_plugin
+  gRPC::address_sorting gRPC::gpr gRPC::grpc gRPC::grpc_unsecure gRPC::grpc++ gRPC::grpc++_alts gRPC::grpc++_error_details gRPC::grpc++_reflection
+  gRPC::grpc++_unsecure gRPC::grpc_plugin_support gRPC::grpcpp_channelz gRPC::upb gRPC::grpc_cpp_plugin gRPC::grpc_csharp_plugin gRPC::grpc_node_plugin
+  gRPC::grpc_objective_c_plugin gRPC::grpc_php_plugin gRPC::grpc_python_plugin gRPC::grpc_ruby_plugin
  BUILD_EXPORT_SET ${PROJECT_NAME}-exports
  INSTALL_EXPORT_SET ${PROJECT_NAME}-exports
 )
@@ -101,26 +91,18 @@ morpheus_utils_configure_prometheus_cpp()
 if(MRC_BUILD_BENCHMARKS)
  # google benchmark
  # ================
- rapids_find_package(benchmark REQUIRED
-  GLOBAL_TARGETS benchmark::benchmark
+ include(${rapids-cmake-dir}/cpm/gbench.cmake)
+ rapids_cpm_gbench(
  BUILD_EXPORT_SET ${PROJECT_NAME}-exports
-
- # No install set
- FIND_ARGS
- CONFIG
  )
 endif()
 
 if(MRC_BUILD_TESTS)
  # google test
  # ===========
- rapids_find_package(GTest REQUIRED
-  GLOBAL_TARGETS GTest::gtest GTest::gmock GTest::gtest_main GTest::gmock_main
+ include(${rapids-cmake-dir}/cpm/gtest.cmake)
+ rapids_cpm_gtest(
  BUILD_EXPORT_SET ${PROJECT_NAME}-exports
-
- # No install set
- FIND_ARGS
- CONFIG
  )
 endif()
 

conda/environments/all_cuda-121_arch-x86_64.yaml → conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -16,31 +16,30 @@ dependencies:
 - clangxx=16
 - cmake=3.27
 - codecov=2.1
-- cuda-cudart-dev=12.1
+- cuda-cudart-dev=12.5
 - cuda-nvcc
-- cuda-nvml-dev=12.1
-- cuda-nvrtc-dev=12.1
-- cuda-tools=12.1
-- cuda-version=12.1
+- cuda-nvml-dev=12.5
+- cuda-nvrtc-dev=12.5
+- cuda-version=12.5
 - cxx-compiler
 - doxygen=1.9.2
 - flake8
 - gcovr=5.2
 - gdb
-- glog=0.6
+- glog>=0.7.1,<0.8
 - gtest=1.14
-- gxx=11.2
+- gxx=12.1
 - include-what-you-use=0.20
 - libclang-cpp=16
 - libclang=16
-- libgrpc=1.59
+- libgrpc=1.62.2
 - libhwloc=2.9.2
-- librmm=24.02
+- librmm=24.10
 - libxml2=2.11.6
 - llvmdev=16
 - ninja=1.11
 - nlohmann_json=3.11
-- numactl-libs-cos7-x86_64
+- numactl=2.0.18
 - numpy=1.24
 - pkg-config=0.29
 - pre-commit
@@ -53,4 +52,4 @@ dependencies:
 - scikit-build=0.17
 - ucx=1.15
 - yapf
-name: all_cuda-121_arch-x86_64
+name: all_cuda-125_arch-x86_64

conda/environments/ci_cuda-121_arch-x86_64.yaml → conda/environments/ci_cuda-125_arch-x86_64.yaml

@@ -11,26 +11,25 @@ dependencies:
 - ccache
 - cmake=3.27
 - codecov=2.1
-- cuda-cudart-dev=12.1
+- cuda-cudart-dev=12.5
 - cuda-nvcc
-- cuda-nvml-dev=12.1
-- cuda-nvrtc-dev=12.1
-- cuda-tools=12.1
-- cuda-version=12.1
+- cuda-nvml-dev=12.5
+- cuda-nvrtc-dev=12.5
+- cuda-version=12.5
 - cxx-compiler
 - doxygen=1.9.2
 - gcovr=5.2
-- glog=0.6
+- glog>=0.7.1,<0.8
 - gtest=1.14
-- gxx=11.2
+- gxx=12.1
 - include-what-you-use=0.20
-- libgrpc=1.59
+- libgrpc=1.62.2
 - libhwloc=2.9.2
-- librmm=24.02
+- librmm=24.10
 - libxml2=2.11.6
 - ninja=1.11
 - nlohmann_json=3.11
-- numactl-libs-cos7-x86_64
+- numactl=2.0.18
 - pkg-config=0.29
 - pre-commit
 - pybind11-stubgen=0.10
@@ -41,4 +40,4 @@ dependencies:
 - python=3.10
 - scikit-build=0.17
 - ucx=1.15
-name: ci_cuda-121_arch-x86_64
+name: ci_cuda-125_arch-x86_64

cpp/mrc/CMakeLists.txt

@@ -163,19 +163,18 @@ add_library(${PROJECT_NAME}::libmrc ALIAS libmrc)
 
 target_link_libraries(libmrc
  PUBLIC
- mrc_protos
- mrc_architect_protos
- rmm::rmm
- CUDA::cudart
- rxcpp::rxcpp
- glog::glog
- libcudacxx::libcudacxx
- Boost::fiber
  Boost::context
+ Boost::fiber
+ CUDA::cudart
  glog::glog
- gRPC::grpc++
- gRPC::grpc
  gRPC::gpr
+ gRPC::grpc
+ gRPC::grpc++
+ libcudacxx::libcudacxx
+ mrc_architect_protos
+ mrc_protos
+ rmm::rmm
+ rxcpp::rxcpp
  PRIVATE
  hwloc::hwloc
  prometheus-cpp::core # private in MR !199

cpp/mrc/include/mrc/core/utils.hpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,6 +23,7 @@
 
 #include <algorithm>
 #include <exception>
+#include <functional>
 #include <map>
 #include <set>
 #include <string>