Merge branch 'main' into xxh/clickhouse_sink_rust_new

Cargo.toml

@@ -50,7 +50,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.0.0-alpha"
+version = "1.1.0-alpha"
 edition = "2021"
 homepage = "https:/risingwavelabs/risingwave"
 keywords = ["sql", "database", "streaming"]

Makefile.toml

@@ -486,7 +486,7 @@ script = '''
 set -ex
 
 RUST_BACKTRACE=1 \
-cargo run --bin risingwave \
+cargo run -p risingwave_cmd_all \
  --profile "${RISINGWAVE_BUILD_PROFILE}" \
  ${RISINGWAVE_FEATURE_FLAGS} \
  -- playground
@@ -507,7 +507,7 @@ fi
 
 source "${RC_ENV_FILE}"
 
-cargo run --bin risectl --profile "${RISINGWAVE_BUILD_PROFILE}" -- "$@"
+cargo run -p risingwave_cmd --bin risectl --profile "${RISINGWAVE_BUILD_PROFILE}" -- "$@"
 test $? -eq 0 || exit 1
 '''
 
@@ -1059,9 +1059,30 @@ description = "Clean data and start a full RisingWave dev cluster using risedev-
 
 [tasks.ci-kill]
 category = "RiseDev - CI"
-dependencies = ["k", "l", "check-logs"]
+dependencies = ["k", "l", "check-logs", "wait-processes-exit"]
 description = "Kill cluster, dump logs and check logs"
 
+[tasks.wait-processes-exit]
+private = true
+category = "Misc"
+description = "Wait for RisingWave processes to exit"
+script = """
+wait_for_process() {
+ process_name="$1"
+
+ while pgrep -x "$process_name" > /dev/null; do
+ echo "Process $process_name is still running... Wait for 1 sec"
+ sleep 1
+ done
+}
+
+wait_for_process meta-node
+wait_for_process compute-node
+wait_for_process frontend
+wait_for_process compactor
+echo "All processes has exited."
+"""
+
 [tasks.slt]
 category = "RiseDev - SQLLogicTest"
 install_crate = { version = "0.15.1", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
@@ -1100,7 +1121,7 @@ description = "Extract SQL examples written in SQLLogicTest syntax from Rust doc
 script = '''
 #!/usr/bin/env bash
 set -e
-cargo run --bin risedev-docslt -- "$@"
+cargo run -p risedev --bin risedev-docslt -- "$@"
 '''
 
 [tasks.compose]
@@ -1111,7 +1132,7 @@ script = '''
 set -e
 rm -rf ${PREFIX_DOCKER}/*
 mkdir -p "${PREFIX_DOCKER}"
-cargo run --bin risedev-compose -- -d "${PREFIX_DOCKER}" "$@"
+cargo run -p risedev --bin risedev-compose -- -d "${PREFIX_DOCKER}" "$@"
 echo docker-compose file generated in $(tput setaf 4)${PREFIX_DOCKER}$(tput sgr0)
 '''
 
@@ -1123,7 +1144,7 @@ script = '''
 set -e
 rm -rf "${PREFIX_DOCKER}"/*
 mkdir -p "${PREFIX_DOCKER}"
-cargo run --bin risedev-compose -- -d "${PREFIX_DOCKER}" --deploy "$@"
+cargo run -p risedev --bin risedev-compose -- -d "${PREFIX_DOCKER}" --deploy "$@"
 echo docker-compose file and deploy.sh generated in $(tput setaf 4)${PREFIX_DOCKER}$(tput sgr0)
 echo If you are ready, run $(tput setaf 4)./risedev apply-compose-deploy$(tput sgr0) to do the deployment.
 '''
@@ -1162,5 +1183,24 @@ cat << EOF > src/config/example.toml
 # This file is generated by ./risedev generate-example-config
 # Check detailed comments in src/common/src/config.rs
 EOF
-cargo run --bin example-config >> src/config/example.toml
+cargo run -p risingwave_common --bin example-config >> src/config/example.toml
 '''
+
+[tasks.psql]
+category = "RiseDev - PSQL Client"
+description = "Run local psql client with default connection parameters. You can pass extra arguments to psql."
+script = '''
+#!/usr/bin/env bash
+set -euo pipefail
+
+RC_ENV_FILE="${PREFIX_CONFIG}/risectl-env"
+
+if [ ! -f "${RC_ENV_FILE}" ]; then
+ echo "risectl-env file not found. Did you start cluster using $(tput setaf 4)\`./risedev d\`$(tput sgr0)?"
+ exit 1
+fi
+
+source "${RC_ENV_FILE}"
+
+psql -h $RW_FRONTEND_LISTEN_ADDRESS -p $RW_FRONTEND_PORT -U root -d dev "$@"
+'''

ci/Dockerfile

@@ -40,7 +40,7 @@ RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/ca
 RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
  [email protected] \
  && cargo install sccache --locked \
- && cargo install [email protected].1 --locked \
+ && cargo install [email protected].2 --locked \
  && cargo cache -a \
  && rm -rf "/root/.cargo/registry/index" \
  && rm -rf "/root/.cargo/registry/cache" \

ci/build-ci-image.sh

@@ -13,7 +13,7 @@ cat ../rust-toolchain
 # !!! CHANGE THIS WHEN YOU WANT TO BUMP CI IMAGE !!! #
 # AND ALSO docker-compose.yml #
 ######################################################
-export BUILD_ENV_VERSION=v20230727
+export BUILD_ENV_VERSION=v20230810
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
  retries: 5
 
  source-test-env:
- image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230727
+ image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230810
  depends_on:
  - mysql
  - db
@@ -80,7 +80,7 @@ services:
  - ..:/risingwave
 
  sink-test-env:
- image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230727
+ image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230810
  depends_on:
  - mysql
  - db
@@ -89,12 +89,12 @@ services:
  - ..:/risingwave
 
  rw-build-env:
- image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230727
+ image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230810
  volumes:
  - ..:/risingwave
 
  ci-flamegraph-env:
- image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230727
+ image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230810
  # NOTE(kwannoel): This is used in order to permit
  # syscalls for `nperf` (perf_event_open),
  # so it can do CPU profiling.
@@ -105,7 +105,7 @@ services:
  - ..:/risingwave
 
  regress-test-env:
- image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230727
+ image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230810
  depends_on:
  db:
  condition: service_healthy

ci/scripts/check.sh

@@ -23,15 +23,31 @@ scripts/check/check-trailing-spaces.sh
 echo "--- Run clippy check (dev, all features)"
 cargo clippy --all-targets --all-features --locked -- -D warnings
 
+echo "--- Show sccache stats"
+sccache --show-stats
+sccache --zero-stats
+
 echo "--- Run clippy check (release)"
 cargo clippy --release --all-targets --features "rw-static-link" --locked -- -D warnings
 
+echo "--- Show sccache stats"
+sccache --show-stats
+sccache --zero-stats
+
 echo "--- Build documentation"
 cargo doc --document-private-items --no-deps
 
+echo "--- Show sccache stats"
+sccache --show-stats
+sccache --zero-stats
+
 echo "--- Run doctest"
 cargo test --doc
 
+echo "--- Show sccache stats"
+sccache --show-stats
+sccache --zero-stats
+
 echo "--- Run audit check"
 cargo audit
 

ci/scripts/e2e-iceberg-sink-test.sh

@@ -39,7 +39,7 @@ echo "--- preparing iceberg"
 .risingwave/bin/mcli -C .risingwave/config/mcli mb hummock-minio/iceberg
 wget https://ci-deps-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
 tar -xf spark-3.3.1-bin-hadoop3.tgz --no-same-owner
-DEPENDENCIES=org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.0.0,org.apache.hadoop:hadoop-aws:3.3.2
+DEPENDENCIES=org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.3.1,org.apache.hadoop:hadoop-aws:3.3.2
 spark-3.3.1-bin-hadoop3/bin/spark-sql --packages $DEPENDENCIES \
  --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \
  --conf spark.sql.catalog.demo.type=hadoop \

ci/scripts/e2e-iceberg-sink-v2-test.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+source ci/scripts/common.sh
+
+while getopts 'p:' opt; do
+ case ${opt} in
+ p )
+ profile=$OPTARG
+ ;;
+ \? )
+ echo "Invalid Option: -$OPTARG" 1>&2
+ exit 1
+ ;;
+ : )
+ echo "Invalid option: $OPTARG requires an argument" 1>&2
+ ;;
+ esac
+done
+shift $((OPTIND -1))
+
+download_and_prepare_rw "$profile" source
+
+echo "--- starting risingwave cluster"
+mkdir -p .risingwave/log
+cargo make ci-start ci-iceberg-test
+sleep 1
+
+# prepare minio iceberg sink
+echo "--- preparing iceberg"
+.risingwave/bin/mcli -C .risingwave/config/mcli mb hummock-minio/iceberg
+wget https://ci-deps-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
+tar -xf spark-3.3.1-bin-hadoop3.tgz --no-same-owner
+DEPENDENCIES=org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.3.1,org.apache.hadoop:hadoop-aws:3.3.2
+spark-3.3.1-bin-hadoop3/bin/spark-sql --packages $DEPENDENCIES \
+ --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \
+ --conf spark.sql.catalog.demo.type=hadoop \
+ --conf spark.sql.catalog.demo.warehouse=s3a://iceberg/ \
+ --conf spark.sql.catalog.demo.hadoop.fs.s3a.endpoint=http://127.0.0.1:9301 \
+ --conf spark.sql.catalog.demo.hadoop.fs.s3a.access.key=hummockadmin \
+ --conf spark.sql.catalog.demo.hadoop.fs.s3a.secret.key=hummockadmin \
+ --S --e "CREATE TABLE demo.demo_db.demo_table(v1 int, v2 bigint, v3 string) TBLPROPERTIES ('format-version'='2');"
+
+# Used to create a snapshot in iceberg.
+# For now iceberg v2 sink can't support to insert in a new table without snapshot.
+# Fix it later.
+spark-3.3.1-bin-hadoop3/bin/spark-sql --packages $DEPENDENCIES \
+ --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \
+ --conf spark.sql.catalog.demo.type=hadoop \
+ --conf spark.sql.catalog.demo.warehouse=s3a://iceberg/ \
+ --conf spark.sql.catalog.demo.hadoop.fs.s3a.endpoint=http://127.0.0.1:9301 \
+ --conf spark.sql.catalog.demo.hadoop.fs.s3a.access.key=hummockadmin \
+ --conf spark.sql.catalog.demo.hadoop.fs.s3a.secret.key=hummockadmin \
+ --S --e "INSERT INTO demo.demo_db.demo_table values (3,1,'b')"
+
+echo "--- testing sinks"
+sqllogictest -p 4566 -d dev './e2e_test/sink/iceberg_sink_v2.slt'
+sleep 1
+
+# check sink destination iceberg
+spark-3.3.1-bin-hadoop3/bin/spark-sql --packages $DEPENDENCIES \
+ --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \
+ --conf spark.sql.catalog.demo.type=hadoop \
+ --conf spark.sql.catalog.demo.warehouse=s3a://iceberg/ \
+ --conf spark.sql.catalog.demo.hadoop.fs.s3a.endpoint=http://127.0.0.1:9301 \
+ --conf spark.sql.catalog.demo.hadoop.fs.s3a.access.key=hummockadmin \
+ --conf spark.sql.catalog.demo.hadoop.fs.s3a.secret.key=hummockadmin \
+ --S --e "INSERT OVERWRITE DIRECTORY './spark-output' USING CSV SELECT * FROM demo.demo_db.demo_table;"
+
+# check sink destination using shell
+if cat ./spark-output/*.csv | sort | awk -F "," '{
+if ($1 == 1 && $2 == 50 && $3 == "1-50") c1++;
+ if ($1 == 13 && $2 == 2 && $3 == "13-2") c2++;
+ if ($1 == 21 && $2 == 2 && $3 == "21-2") c3++;
+ if ($1 == 2 && $2 == 2 && $3 == "2-2") c4++;
+ if ($1 == 3 && $2 == 2 && $3 == "3-2") c5++;
+ if ($1 == 5 && $2 == 2 && $3 == "5-2") c6++;
+ if ($1 == 8 && $2 == 2 && $3 == "8-2") c7++; }
+ END { exit !(c1 == 1 && c2 == 1 && c3 == 1 && c4 == 1 && c5 == 1 && c6 == 1 && c7 == 1); }'; then
+ echo "Iceberg sink check passed"
+else
+ echo "The output is not as expected."
+ exit 1
+fi
+
+echo "--- Kill cluster"
+cargo make ci-kill

ci/scripts/e2e-kafka-sink-test.sh

@@ -3,6 +3,10 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
+./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only --create > /dev/null 2>&1
+./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --create > /dev/null 2>&1
+./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --create > /dev/null 2>&1
+
 sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/create_sink.slt'
 sleep 2
 

ci/scripts/integration-tests.sh

@@ -35,7 +35,7 @@ echo "--- ghcr login"
 echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
 
 echo "--- install postgresql"
-sudo yum install -y postgresql
+sudo yum install -y postgresql15
 
 cd integration_tests/scripts
 
@@ -65,7 +65,8 @@ echo "--- check if the ingestion is successful"
 upstream=$(echo ${case} | cut -d'-' -f 1)
 if [ "${upstream}" == "mysql" ]; then
  echo "install mysql"
- sudo yum install -y mysql
+ sudo rpm -Uvh https://dev.mysql.com/get/mysql80-community-release-el9-1.noarch.rpm
+ sudo dnf -y install mysql-community-server
 fi
 
 export PGPASSWORD=123456

ci/scripts/run-e2e-test.sh

@@ -47,6 +47,9 @@ sqllogictest -p 4566 -d dev './e2e_test/visibility_mode/*.slt' --junit "batch-${
 sqllogictest -p 4566 -d dev './e2e_test/database/prepare.slt'
 sqllogictest -p 4566 -d test './e2e_test/database/test.slt'
 
+echo "--- e2e, ci-3streaming-2serving-3fe, Apache Superset"
+sqllogictest -p 4566 -d dev './e2e_test/superset/*.slt' --junit "batch-${profile}"
+
 echo "--- e2e, ci-3streaming-2serving-3fe, python udf"
 python3 e2e_test/udf/test.py &
 sleep 2

ci/scripts/run-fuzz-test.sh

@@ -11,6 +11,14 @@ if [[ $RUN_SQLSMITH_FRONTEND -eq "1" ]]; then
  NEXTEST_PROFILE=ci cargo nextest run --package risingwave_sqlsmith --features "enable_sqlsmith_unit_test" 2> >(tee);
 fi
 
+extract_error_sql() {
+ cat "$1" \
+ | grep -E "(\[EXECUTING|\[TEST)" \
+ | sed 's/.*\[EXECUTING .*\]: //' \
+ | sed 's/.*\[TEST.*\]: //' \
+ | sed 's/$/;/' > $LOGDIR/error.sql.log
+}
+
 if [[ "$RUN_SQLSMITH" -eq "1" ]]; then
  while getopts 'p:' opt; do
  case ${opt} in
@@ -45,7 +53,13 @@ if [[ "$RUN_SQLSMITH" -eq "1" ]]; then
  echo "--- e2e, ci-3cn-1fe, run fuzzing"
  ./target/debug/sqlsmith test \
  --count "$SQLSMITH_COUNT" \
- --testdata ./src/tests/sqlsmith/tests/testdata
+ --testdata ./src/tests/sqlsmith/tests/testdata > $LOGDIR/fuzzing.log 2>&1 && rm $LOGDIR/*
+
+ if [[ -e $LOGDIR/fuzzing.log ]]; then
+ echo "Fuzzing failed"
+ extract_error_sql $LOGDIR/fuzzing.log
+ exit 1
+ fi
 
  # Sqlsmith does not write to stdout, so we need this to ensure buildkite
  # shows the right timing.

ci/workflows/main.yml

@@ -317,6 +317,20 @@ steps:
  timeout_in_minutes: 5
  retry: *auto-retry
 
+ - label: "end-to-end iceberg sink v2 test (release mode)"
+ command: "ci/scripts/e2e-iceberg-sink-v2-test.sh -p ci-release"
+ depends_on:
+ - "build-release"
+ - "build-other"
+ plugins:
+ - docker-compose#v4.9.0:
+ run: rw-build-env
+ config: ci/docker-compose.yml
+ mount-buildkite-agent: true
+ - ./ci/plugins/upload-failure-logs
+ timeout_in_minutes: 5
+ retry: *auto-retry
+
  - label: "e2e java-binding test (at release)"
  command: "ci/scripts/java-binding-test.sh -p ci-release"
  depends_on: