Merge branch 'main' into block-direct-copy

.cargo/config.toml

@@ -24,7 +24,6 @@ rustflags = [
 ]
 
 [target.aarch64-apple-darwin]
-runner = "scripts/coredump/sign-and-run"
 rustflags = [
  # neon is enabled by default
  "-Clink-arg=-fuse-ld=/opt/homebrew/opt/llvm/bin/ld64.lld",

.config/hakari.toml

@@ -35,4 +35,6 @@ third-party = [
  { name = "criterion" },
  { name = "console" },
  { name = "similar" },
+ # FYI: https:/risingwavelabs/risingwave/issues/12315
+ { name = "tikv-jemalloc-sys", git = "https:/risingwavelabs/jemallocator.git", rev = "64a2d9" },
 ]

Cargo.toml

@@ -112,6 +112,10 @@ arrow-schema = "46"
 arrow-buffer = "46"
 arrow-flight = "46"
 arrow-select = "46"
+tikv-jemallocator = { git = "https:/risingwavelabs/jemallocator.git", features = [
+ "profiling",
+ "stats",
+], rev = "64a2d9" }
 
 risingwave_backup = { path = "./src/storage/backup" }
 risingwave_batch = { path = "./src/batch" }

Makefile.toml

@@ -281,6 +281,23 @@ ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "$
 ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/standalone"
 '''
 
+[tasks.codesign-playground]
+private = true
+category = "RiseDev - Build"
+description = "Codesign playground binary to support coredump"
+# If core dump is enabled by RiseDev and we're on an Apple Silicon platform,
+# codesign the binary before running.
+# https://developer.apple.com/forums/thread/694233?answerId=695943022#695943022
+condition = { env_set = [
+ "ENABLE_COREDUMP",
+], env = { "SYSTEM" = "darwin-arm64" } }
+script = '''
+#!/usr/bin/env bash
+
+set -ex
+codesign -s - -f --entitlements scripts/coredump/coredump.entitlements "target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave"
+'''
+
 [tasks.codesign-binaries]
 private = true
 category = "RiseDev - Build"
@@ -303,6 +320,7 @@ else
  binaries=("meta-node" "compute-node" "frontend" "compactor")
 fi
 
+set -ex
 echo -n "${binaries[*]}" | parallel -d ' ' \
  "codesign -s - -f --entitlements scripts/coredump/coredump.entitlements \"target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/{}\""
 '''
@@ -393,6 +411,24 @@ ${BUILD_HUMMOCK_TRACE_CMD}\
  ${RISEDEV_CARGO_BUILD_EXTRA_ARGS}
 '''
 
+[tasks.build-risingwave-playground]
+category = "RiseDev - Build"
+description = "Build RisingWave playground"
+condition = { env_true = ["ENABLE_BUILD_RUST"] }
+script = '''
+#!/usr/bin/env bash
+
+set -e
+[[ -z "${RISEDEV_RUSTFLAGS}" ]] || export RUSTFLAGS="${RISEDEV_RUSTFLAGS}"
+echo + RUSTFLAGS="${RUSTFLAGS:-<not set>}"
+set -xe
+
+cargo build -p risingwave_cmd_all \
+ --profile "${RISINGWAVE_BUILD_PROFILE}" \
+ ${RISINGWAVE_FEATURE_FLAGS} \
+ ${RISEDEV_CARGO_BUILD_EXTRA_ARGS}
+'''
+
 [tasks.clean]
 private = true
 category = "RiseDev - Build"
@@ -508,7 +544,11 @@ dependencies = [
 [tasks.pre-start-playground]
 category = "RiseDev - Prepare"
 description = "Preparation steps for playground"
-dependencies = ["build-connector-node"]
+dependencies = [
+ "build-risingwave-playground",
+ "codesign-playground",
+ "build-connector-node",
+]
 
 [tasks.check-risedev-env-file]
 private = true
@@ -557,35 +597,31 @@ dependencies = ["pre-start-playground"]
 script = '''
 #!/usr/bin/env bash
 
-set -ex
-
+set -e
 if [[ $ENABLE_COREDUMP == "true" ]]; then
+ echo "+ ulimit -c unlimited"
  ulimit -c unlimited
 fi
 
-cargo run -p risingwave_cmd_all \
- --profile "${RISINGWAVE_BUILD_PROFILE}" \
- ${RISINGWAVE_FEATURE_FLAGS} \
- -- playground
+set -x
+target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave playground $@
 '''
 
 [tasks.standalone]
 category = "RiseDev - Start/Stop"
 description = "🌟 Start a RisingWave standalone instance"
-dependencies = ["build"]
+dependencies = ["pre-start-playground"]
 script = '''
 #!/usr/bin/env bash
 
-set -euo pipefail
-
+set -e
 if [[ $ENABLE_COREDUMP == "true" ]]; then
+ echo "+ ulimit -c unlimited"
  ulimit -c unlimited
 fi
 
-cargo run -p risingwave_cmd_all \
- --profile "${RISINGWAVE_BUILD_PROFILE}" \
- ${RISINGWAVE_FEATURE_FLAGS} \
- -- standalone $@
+set -x
+target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave standalone $@
 '''
 
 # TODO(kwannoel): Support `tasks.standalone-dev` as well.
@@ -612,12 +648,13 @@ description = "🌟 Start a full RisingWave dev cluster using risedev-dev"
 script = '''
 #!/usr/bin/env bash
 
-set -ex
-
+set -e
 if [[ $ENABLE_COREDUMP == "true" ]]; then
+ echo "+ ulimit -c unlimited"
  ulimit -c unlimited
 fi
 
+set -x
 target/${BUILD_MODE_DIR}/risedev-dev ${@}
 '''
 

ci/Dockerfile

@@ -7,7 +7,7 @@ RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://us-east-2.ec2.archive.ubun
 RUN apt-get update -yy && \
  DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel python3 python3-pip \
  openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld postgresql-client kafkacat netcat mysql-client \
- maven zstd libzstd-dev -yy \
+ maven zstd libzstd-dev locales -yy \
  && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 SHELL ["/bin/bash", "-c"]
@@ -43,7 +43,7 @@ RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/ca
 RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
  [email protected] \
  [email protected] \
- && cargo install sccache --locked \
+ && cargo install sccache \
  && cargo cache -a \
  && rm -rf "/root/.cargo/registry/index" \
  && rm -rf "/root/.cargo/registry/cache" \

ci/build-ci-image.sh

@@ -13,7 +13,7 @@ cat ../rust-toolchain
 # !!! CHANGE THIS WHEN YOU WANT TO BUMP CI IMAGE !!! #
 # AND ALSO docker-compose.yml #
 ######################################################
-export BUILD_ENV_VERSION=v20230909
+export BUILD_ENV_VERSION=v20230914
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
  retries: 5
 
  source-test-env:
- image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230909
+ image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230914
  depends_on:
  - mysql
  - db
@@ -81,7 +81,7 @@ services:
  - ..:/risingwave
 
  sink-test-env:
- image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230909
+ image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230914
  depends_on:
  - mysql
  - db
@@ -91,12 +91,12 @@ services:
  - ..:/risingwave
 
  rw-build-env:
- image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230909
+ image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230914
  volumes:
  - ..:/risingwave
 
  ci-flamegraph-env:
- image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230909
+ image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230914
  # NOTE(kwannoel): This is used in order to permit
  # syscalls for `nperf` (perf_event_open),
  # so it can do CPU profiling.
@@ -107,7 +107,7 @@ services:
  - ..:/risingwave
 
  regress-test-env:
- image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230909
+ image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230914
  depends_on:
  db:
  condition: service_healthy

ci/scripts/regress-test.sh

@@ -30,8 +30,6 @@ mv target/debug/risingwave_regress_test-"$profile" target/debug/risingwave_regre
 chmod +x ./target/debug/risingwave_regress_test
 
 echo "--- Postgres regress test"
-apt-get -y update
-apt-get -y install locales
 locale-gen C
 export LANGUAGE=C
 export LANG=C