Merge branch 'main' into xx/bench-compression

Cargo.toml

@@ -24,6 +24,7 @@ members = [
  "src/storage/compactor",
  "src/storage/hummock_sdk",
  "src/stream",
+ "src/test_runner",
  "src/tests/regress",
  "src/utils/logging",
  "src/utils/memcomparable",

README.md

@@ -1,7 +1,6 @@
 ![RisingWave Logo](./docs/images/logo-title.svg)
 
 [![Slack](https://badgen.net/badge/Slack/Join%20RisingWave/0abd59?icon=slack)](https://join.slack.com/t/risingwave-community/shared_invite/zt-120rft0mr-d8uGk3d~NZiZAQWPnElOfw)
-[![CI](https:/singularity-data/risingwave/actions/workflows/main.yml/badge.svg)](https:/singularity-data/risingwave/actions/workflows/main.yml)
 [![Build status](https://badge.buildkite.com/9394d2bca0f87e2e97aa78b25f765c92d4207c0b65e7f6648f.svg)](https://buildkite.com/singularity-data/main)
 [![codecov](https://codecov.io/gh/singularity-data/risingwave/branch/main/graph/badge.svg?token=EB44K9K38B)](https://codecov.io/gh/singularity-data/risingwave)
 

ci/scripts/release.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+echo "--- Install gh cli"
+curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | \
+dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | \
+tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+apt update -yy && apt install gh -yy
+
+echo "--- Release create"
+gh release create "${BUILDKITE_TAG}" --generate-notes -d -p
+
+echo "--- Download artifacts"
+mkdir -p target/debug && cd target/debug
+buildkite-agent artifact download risingwave-release .
+mv risingwave-release risingwave
+chmod +x risingwave
+tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz risingwave
+
+echo "--- Release upload asset"
+gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz

ci/workflows/main.yml

@@ -98,3 +98,20 @@ steps:
  - shellcheck#v1.2.0:
  files: ./**/*.sh
  timeout_in_minutes: 10
+
+ - label: "release"
+ command: "ci/scripts/release.sh"
+ if: build.tag != null
+ depends_on: "build"
+ plugins:
+ - seek-oss/aws-sm#v2.3.1:
+ env:
+ GITHUB_TOKEN: github-token
+ - docker-compose#v3.9.0:
+ run: rw-build-env
+ config: ci/docker-compose.yml
+ mount-buildkite-agent: true
+ environment:
+ - GITHUB_TOKEN
+ - BUILDKITE_TAG
+ timeout_in_minutes: 5

e2e_test/batch/range_scan.slt

@@ -0,0 +1,64 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+CREATE TABLE orders (
+ order_id BIGINT,
+ user_id INTEGER,
+ date INTEGER);
+
+statement ok 
+CREATE MATERIALIZED VIEW orders_count_by_user AS
+ SELECT user_id, date, count(*) AS orders_count FROM orders GROUP BY user_id, date;
+
+statement ok
+insert into orders values 
+ (0, 42, 1111), 
+ (1, 42, 2222),
+ (2, 42, 2222),
+ (3, 43, 1111),
+ (4, 43, 2222),
+ (5, 42, 3333);
+
+query III
+SELECT * FROM orders_count_by_user WHERE user_id = 42;
+----
+42 1111 1
+42 2222 2
+42 3333 1
+
+query III
+SELECT * FROM orders_count_by_user WHERE user_id < 43;
+----
+42 1111 1
+42 2222 2
+42 3333 1
+
+query III
+SELECT * FROM orders_count_by_user WHERE user_id > 42;
+----
+43 1111 1
+43 2222 1
+
+query III
+SELECT * FROM orders_count_by_user WHERE user_id >= 43;
+----
+43 1111 1
+43 2222 1
+
+query III
+SELECT * FROM orders_count_by_user WHERE user_id = 42 AND date = 2222;
+----
+42 2222 2
+
+query III
+SELECT * FROM orders_count_by_user WHERE date > 1111 AND user_id = 42 AND 5<6 AND date <= 3333
+----
+42 2222 2
+42 3333 1
+
+statement ok
+drop materialized view orders_count_by_user;
+
+statement ok
+drop table orders;

e2e_test/batch/types/interval.slt

@@ -2,3 +2,33 @@ query TTTTT
 select interval '500' year, interval '-50' month, interval '5' hour, interval '5' minute, interval '5' second;
 ----
 500 years 00:00:00 -4 years -2 mons 00:00:00 05:00:00 00:05:00 00:00:05
+
+query TTTTT
+SELECT interval '1 year', interval '1 y', interval '1 yr';
+----
+1 year 00:00:00 1 year 00:00:00 1 year 00:00:00
+
+query TTTTT
+SELECT interval '2 month', interval '2 mon';
+----
+2 mons 00:00:00 2 mons 00:00:00
+
+query TTTTT
+SELECT interval '3 day';
+----
+3 days 00:00:00
+
+query TTTTT
+SELECT interval '4 hour';
+----
+04:00:00
+
+query TTTTT
+SELECT interval '5 minute', interval '2 m';
+----
+00:05:00 00:02:00
+
+query TTTTT
+SELECT interval '6 second';
+----
+00:00:06

e2e_test/streaming/extreme_null.slt

@@ -1,3 +1,6 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
 statement ok
 create table t1 (v1 int, v2 int, v3 int);
 
@@ -24,9 +27,6 @@ select avg_v1, sum_v2, count_v3 from mv2;
 statement ok
 insert into t1 values (3,4,4), (4,3,5);
 
-statement ok
-flush;
-
 query RII
 select avg_v1, sum_v2, count_v3 from mv2;
 ----
@@ -50,3 +50,50 @@ drop table t1
 
 statement ok
 drop table t4
+
+statement ok
+create table t (v1 interval, v2 timestamp, v3 varchar, v4 date);
+
+statement ok
+create materialized view mv1 as select max(v1) as a1, max(v2) as a2, max(v3) as a3, max(v4) as a4 from t;
+
+statement ok
+create materialized view mv2 as select min(v1) as a1, min(v2) as a2, min(v3) as a3, min(v4) as a4 from t;
+
+statement ok
+create materialized view mv3 as select count(v1) as a1, count(v2) as a2, count(v3) as a3, count(v4) as a4 from t;
+
+statement ok
+insert into t values (interval '1' minute, timestamp '2001-02-16 20:38:40', 'abc', date '2001-02-16');
+
+query TTTTTT
+select max(v1), max(v2), max(v3), max(v4) from t;
+----
+00:01:00 2001-02-16 20:38:40 abc 2001-02-16
+
+query TTTTTT
+select * from mv1;
+----
+00:01:00 2001-02-16 20:38:40 abc 2001-02-16
+
+query TTTTTT
+select * from mv2;
+----
+00:01:00 2001-02-16 20:38:40 abc 2001-02-16
+
+query TTTT
+select * from mv3;
+----
+1 1 1 1
+
+statement ok
+drop materialized view mv1;
+
+statement ok
+drop materialized view mv2;
+
+statement ok
+drop materialized view mv3;
+
+statement ok
+drop table t;

e2e_test/streaming/struct_table.slt

@@ -0,0 +1,26 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+create table st (v1 int, v2 struct<v1 int, v2 struct<v1 int, v2 int>>);
+
+statement ok
+create materialized view mv1 as select (v2).v2 from st;
+
+statement ok
+insert into st values(1,(1,(1,2)));
+
+statement ok
+insert into st values(1,(1,(1,3)));
+
+query I
+select * from mv1;
+----
+(1,2)
+(1,3)
+
+statement ok
+drop materialized view mv1
+
+statement ok
+drop table st

grafana/risingwave-dashboard.py

@@ -337,9 +337,9 @@ def section_streaming_actors(outer_panels):
  panels = outer_panels.sub_panel()
  return [
  outer_panels.row_collapsed("Streaming Actors", [
- panels.timeseries_rowsps("Actor Throughput", [
+ panels.timeseries_rowsps("Executor Throughput", [
  panels.target(
- "rate(stream_actor_row_count[15s]) > 0", "{{actor_id}}"
+ "rate(stream_executor_row_count[15s]) > 0", "{{actor_id}}->{{executor_id}}"
  ),
  ]),
  panels.timeseries_latency("Actor Barrier Latency", [

proto/batch_plan.proto

@@ -12,6 +12,28 @@ option optimize_for = SPEED;
 message RowSeqScanNode {
  plan_common.CellBasedTableDesc table_desc = 1;
  repeated plan_common.ColumnDesc column_descs = 2;
+ ScanRange scan_range = 3;
+}
+
+// The range to scan, which specifies a consecutive range of the PK
+// and can represent: (Suppose there are N columns in the PK)
+// - full table scan: `eq_conds` is empty, and `lower_bound` & `upper_bound` are `None`
+// - index range scan: `eq_conds` includes i (between 0 and N-1, inclusive) values,
+// and `lower_bound` & `upper_bound` is the range for the (i+1)th column
+// - index point get: `eq_conds` includes N values, and `lower_bound` & `upper_bound` are `None`
+message ScanRange {
+ // The i-th element represents the value of the i-th PK column.
+ repeated expr.ExprNode eq_conds = 1;
+
+ // `None` represent unbounded.
+ message Bound {
+ expr.ExprNode value = 1;
+ bool inclusive = 2;
+ }
+ // The lower bound of the next PK column subsequent to those in `eq_conds`.
+ Bound lower_bound = 2;
+ // The upper bound of the next PK column subsequent to those in `eq_conds`.
+ Bound upper_bound = 3;
 }
 
 message SourceScanNode {

proto/data.proto

@@ -158,6 +158,7 @@ message UpdateMutation {
 
 message AddMutation {
  repeated DispatcherMutation mutations = 1;
+ repeated SourceChangeSplit splits = 2;
 }
 
 message SourceChangeSplitMutation {

proto/hummock.proto

@@ -140,6 +140,8 @@ message CompactTask {
  repeated common.ParallelUnitMapping vnode_mappings = 11;
  // compaction group the task belongs to
  uint64 compaction_group_id = 12;
+ // existing_table_ids for compaction drop key
+ repeated uint32 existing_table_ids = 13;
 }
 
 message LevelHandler {

proto/stream_plan.proto

@@ -24,15 +24,6 @@ message SourceNode {
  plan_common.TableRefId table_ref_id = 1;
  repeated int32 column_ids = 2;
  SourceType source_type = 3;
- // split allocation information,
- // and in the future will distinguish between `StreamSource` and `TableSource`
- // so that there is no need to put many fields that are not common into the same SourceNode structure
- StreamSourceState stream_source_state = 4;
-}
-
-message StreamSourceState {
- string split_type = 1;
- repeated bytes stream_source_splits = 2;
 }
 
 message ProjectNode {

risedev.yml

@@ -435,6 +435,9 @@ template:
  # Etcd used by this Prometheus instance
  provide-etcd: "etcd*"
 
+ # Redpanda used by this Prometheus instance
+ provide-redpanda: "redpanda*"
+
  frontend:
  # Advertise address of frontend
  address: "127.0.0.1"