Merge branch 'OpenAtomFoundation:unstable' into incr

.github/workflows/pika.yml

@@ -78,6 +78,72 @@ jobs:
  chmod +x integrate_test.sh
  sh integrate_test.sh
 
+ build_on_rocky:
+ runs-on: ubuntu-latest
+ container:
+ image: rockylinux:9
+
+ steps:
+ - name: Install deps
+ run: |
+ dnf update -y
+ dnf install -y bash cmake wget git autoconf gcc perl-Digest-SHA tcl which tar g++ tar epel-release gcc-c++ libstdc++-devel gcc-toolset-13
+
+ - name: Set up Go
+ uses: actions/setup-go@v3
+ with:
+ go-version: 1.19
+
+ - name: Checkout
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: Configure CMake
+ run: |
+ source /opt/rh/gcc-toolset-13/enable
+ cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address .
+
+ - uses: actions/cache@v3
+ with:
+ path: ${{ github.workspace }}/deps
+ key: ${{ runner.os }}-rocky-deps-${{ hashFiles('**/CMakeLists.txt') }}
+
+ - uses: actions/cache@v3
+ with:
+ path: ${{ github.workspace }}/buildtrees
+ key: ${{ runner.os }}-rocky-buildtrees-${{ hashFiles('**/CMakeLists.txt') }}
+
+ - name: Build
+ run: |
+ source /opt/rh/gcc-toolset-13/enable
+ cmake --build build --config ${{ env.BUILD_TYPE }}
+
+ - name: Test
+ working-directory: ${{ github.workspace }}/build
+ run: ctest -C ${{ env.BUILD_TYPE }}
+
+ - name: Unit Test
+ working-directory: ${{ github.workspace }}
+ run: ./pikatests.sh all
+
+ - name: Start codis, pika master and pika slave
+ working-directory: ${{ github.workspace }}/build
+ run: |
+ chmod +x ../tests/integration/start_master_and_slave.sh
+ ../tests/integration/start_master_and_slave.sh
+ chmod +x ../tests/integration/start_codis.sh
+ ../tests/integration/start_codis.sh
+
+ - name: Run Go E2E Tests
+ working-directory: ${{ github.workspace }}/build
+ run: |
+ cd ../tools/pika_keys_analysis/
+ go test -v ./...
+ cd ../../tests/integration/
+ chmod +x integrate_test.sh
+ sh integrate_test.sh
+
  build_on_macos:
  runs-on: macos-12
 

.github/workflows/release.yml

@@ -47,9 +47,47 @@ jobs:
  name: ${{ matrix.name }}.sha256sum
  path: build/${{ matrix.name }}.sha256sum
 
+ rocky:
+ runs-on: ubuntu-latest
+ container:
+ image: rockylinux:9
+ env:
+ name: ${{ github.event.repository.name }}-${{ github.ref_name }}-rocky-amd64.tar.gz
+ steps:
+ - name: Install deps
+ run: |
+ dnf update -y
+ dnf install -y bash cmake wget git autoconf gcc perl-Digest-SHA tcl which tar g++ tar epel-release gcc-c++ libstdc++-devel gcc-toolset-13
+
+ - name: Checkout sources
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: Release build os - rocky
+ run: |
+ chmod +x ci/release-build.sh
+ ./ci/release-build.sh install rocky ${{ env.BUILD_TYPE }} -xe
+
+ - name: Calculate checksum and rename binary
+ shell: bash
+ run: ./ci/release-build.sh checksum ${{ github.event.repository.name }} ${{ env.name }}
+
+ - name: Upload artifacts
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ env.name }}
+ path: build/${{ env.name }}
+
+ - name: Upload checksum of artifacts
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ env.name }}.sha256sum
+ path: build/${{ env.name }}.sha256sum
+
  release:
  name: Release artifacts
- needs: [ build ]
+ needs: [ build, rocky ]
  runs-on: ubuntu-latest
  steps:
  - name: Download artifacts

CMakeLists.txt

@@ -22,6 +22,7 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
  endif()
 endif()
 
+link_directories("/opt/rh/gcc-toolset-13/root/lib/gcc/x86_64-redhat-linux/13")
 
 ############# You should enable sanitizer if you are developing pika #############
 # Uncomment the following two lines to enable AddressSanitizer to detect memory leaks and other memory-related bugs.
@@ -159,7 +160,7 @@ ExternalProject_Add(gtest
  make -j${CPU_CORE}
 )
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
  set(GTEST_LIBRARY ${INSTALL_LIBDIR_64}/libgtest.a)
  set(GTEST_MAIN_LIBRARY ${INSTALL_LIBDIR_64}/libgtest_main.a)
  set(GMOCK_LIBRARY ${INSTALL_LIBDIR_64}/libgmock.a)
@@ -282,7 +283,7 @@ else()
  set(LIB_GLOG libglog.a)
 endif()
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
  set(GLOG_LIBRARY ${INSTALL_LIBDIR_64}/${LIB_GLOG})
 else()
  set(GLOG_LIBRARY ${INSTALL_LIBDIR}/${LIB_GLOG})
@@ -317,7 +318,7 @@ ExternalProject_Add(snappy
  make -j${CPU_CORE}
 )
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
  set(SNAPPY_LIBRARY ${INSTALL_LIBDIR_64}/libsnappy.a)
 else()
  set(SNAPPY_LIBRARY ${INSTALL_LIBDIR}/libsnappy.a)
@@ -355,7 +356,7 @@ ExternalProject_Add(zstd
  make -j${CPU_CORE}
 )
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
  set(ZSTD_LIBRARY ${INSTALL_LIBDIR_64}/libzstd.a)
 else()
  set(ZSTD_LIBRARY ${INSTALL_LIBDIR}/libzstd.a)
@@ -394,7 +395,7 @@ else()
  set(LIB_FMT libfmt.a)
 endif()
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
  set(FMT_LIBRARY ${INSTALL_LIBDIR_64}/${LIB_FMT})
 else()
  set(FMT_LIBRARY ${INSTALL_LIBDIR}/${LIB_FMT})
@@ -432,7 +433,7 @@ ExternalProject_Add(lz4
  make -j${CPU_CORE}
 )
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
  set(LZ4_LIBRARY ${INSTALL_LIBDIR_64}/liblz4.a)
 else()
  set(LZ4_LIBRARY ${INSTALL_LIBDIR}/liblz4.a)
@@ -718,7 +719,7 @@ if (USE_PIKA_TOOLS)
  set(BZ2_LIBRARY ${INSTALL_LIBDIR}/libbz2.a)
 endif()
 
-if(${OS_VERSION} MATCHES "CentOS")
+if(${OS_VERSION} MATCHES "Rocky")
  set(ROCKSDB_LIBRARY ${INSTALL_LIBDIR_64}/librocksdb.a)
 else()
  set(ROCKSDB_LIBRARY ${INSTALL_LIBDIR}/librocksdb.a)

ci/release-build.sh

@@ -9,6 +9,12 @@ function install_deps() {
  elif [[ $OS == *"ubuntu"* ]]; then
  sudo apt-get install -y autoconf libprotobuf-dev protobuf-compiler
  sudo apt-get install -y clang-tidy-12
+ elif [[ $OS == *"rocky"* ]]; then
+ sudo dnf update -y
+ sudo dnf install -y bash cmake
+ sudo dnf install -y wget git autoconf gcc perl-Digest-SHA
+ sudo dnf install -y tcl which tar g++ tar epel-release gcc-c++ libstdc++-devel
+ sudo dnf install -y gcc-toolset-13
  else
  echo "not support $OS"
  fi
@@ -22,6 +28,9 @@ function configure_cmake() {
  cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=$BUILD_TYPE
  elif [[ $OS == *"ubuntu"* ]]; then
  cmake -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS="-s" -DCMAKE_EXE_LINKER_FLAGS="-s"
+ elif [[ $OS == *"rocky"* ]]; then
+ source /opt/rh/gcc-toolset-13/enable
+ cmake -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
  fi
  echo "configure cmake after ..."
 }

codis/Makefile

@@ -12,25 +12,25 @@ codis-deps:
 
 codis-dashboard: codis-deps
  $(info build codis-dashboard)
- @cd ${PRJ_ROOT}/cmd/dashboard && go mod tidy && go build -o ${PRJ_ROOT}/bin/codis-dashboard .
+ @cd ${PRJ_ROOT}/cmd/dashboard && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-dashboard .
  @${PRJ_ROOT}/bin/codis-dashboard --default-config > ${PRJ_ROOT}/config/dashboard.toml
 
 codis-proxy: codis-deps
  $(info build codis-proxy)
- @cd ${PRJ_ROOT}/cmd/proxy && go mod tidy && go build -o ${PRJ_ROOT}/bin/codis-proxy .
+ @cd ${PRJ_ROOT}/cmd/proxy && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-proxy .
  @${PRJ_ROOT}/bin/codis-proxy --default-config > ${PRJ_ROOT}/config/proxy.toml
 
 codis-admin: codis-deps
  $(info build codis-admin)
- @cd ${PRJ_ROOT}/cmd/admin && go mod tidy && go build -o ${PRJ_ROOT}/bin/codis-admin .
+ @cd ${PRJ_ROOT}/cmd/admin && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-admin .
 
 codis-ha: codis-deps
  $(info build codis-ha)
- @cd ${PRJ_ROOT}/cmd/ha && go mod tidy && go build -o ${PRJ_ROOT}/bin/codis-ha .
+ @cd ${PRJ_ROOT}/cmd/ha && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-ha .
 
 codis-fe: codis-deps
  $(info build codis-fe)
- @cd ${PRJ_ROOT}/cmd/fe && go mod tidy && go build -o ${PRJ_ROOT}/bin/codis-fe .
+ @cd ${PRJ_ROOT}/cmd/fe && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-fe .
  @rm -rf ${PRJ_ROOT}/bin/assets && cp -rf ${PRJ_ROOT}/cmd/fe/assets ./bin/
 
 clean:

include/pika_hash.h

@@ -148,12 +148,9 @@ class HIncrbyCmd : public Cmd {
 
  private:
  std::string key_, field_;
- int64_t new_value_ = 0;
  int64_t by_ = 0;
- int64_t ttl_ = 0;
  void DoInitial() override;
  rocksdb::Status s_;
- std::string ToRedisProtocol() override;
 };
 
 class HIncrbyfloatCmd : public Cmd {

include/pika_kv.h

@@ -115,7 +115,7 @@ class IncrCmd : public Cmd {
  int64_t new_value_ = 0;
  void DoInitial() override;
  rocksdb::Status s_;
- uint64_t ttl_ = 0;
+ int64_t ttl_ = 0;
  std::string ToRedisProtocol() override;
 };
 
@@ -140,7 +140,8 @@ class IncrbyCmd : public Cmd {
  int64_t by_ = 0, new_value_ = 0;
  void DoInitial() override;
  rocksdb::Status s_;
- uint64_t ttl_ = 0;
+ int64_t ttl_ = 0;
+ std::string ToRedisProtocol() override;
 };
 
 class IncrbyfloatCmd : public Cmd {
@@ -164,7 +165,8 @@ class IncrbyfloatCmd : public Cmd {
  double by_ = 0;
  void DoInitial() override;
  rocksdb::Status s_;
- uint64_t ttl_ = 0;
+ int64_t ttl_ = 0;
+ std::string ToRedisProtocol() override;
 };
 
 class DecrCmd : public Cmd {
@@ -257,6 +259,8 @@ class AppendCmd : public Cmd {
  std::string value_;
  void DoInitial() override;
  rocksdb::Status s_;
+ int64_t ttl_ = 0;
+ std::string ToRedisProtocol() override;
 };
 
 class MgetCmd : public Cmd {

src/pika_hash.cc

@@ -262,9 +262,10 @@ void HIncrbyCmd::DoInitial() {
 }
 
 void HIncrbyCmd::Do() {
- s_ = db_->storage()->HIncrby(key_, field_, by_, &new_value_, &ttl_);
+ int64_t new_value = 0;
+ s_ = db_->storage()->HIncrby(key_, field_, by_, &new_value);
  if (s_.ok() || s_.IsNotFound()) {
- res_.AppendContent(":" + std::to_string(new_value_));
+ res_.AppendContent(":" + std::to_string(new_value));
  AddSlotKey("h", key_, db_);
  } else if (s_.IsInvalidArgument() && s_.ToString().substr(0, std::char_traits<char>::length(ErrTypeMessage)) == ErrTypeMessage) {
  res_.SetRes(CmdRes::kMultiKey);
@@ -287,32 +288,6 @@ void HIncrbyCmd::DoUpdateCache() {
  }
 }
 
-std::string HIncrbyCmd::ToRedisProtocol() {
- std::string content;
- content.reserve(RAW_ARGS_LEN);
- RedisAppendLen(content, 4, "*");
-
- // to pksetexat cmd
- std::string pksetexat_cmd("pksetexat");
- RedisAppendLenUint64(content, pksetexat_cmd.size(), "$");
- RedisAppendContent(content, pksetexat_cmd);
- // key
- RedisAppendLenUint64(content, key_.size(), "$");
- RedisAppendContent(content, key_);
- // time_stamp
- char buf[100];
- auto time_stamp = time(nullptr) + ttl_;
- pstd::ll2string(buf, 100, time_stamp);
- std::string at(buf);
- RedisAppendLenUint64(content, at.size(), "$");
- RedisAppendContent(content, at);
- // value
- std::string new_value_str = std::to_string(new_value_);
- RedisAppendLenUint64(content, new_value_str.size(), "$");
- RedisAppendContent(content, new_value_str);
- return content;
-}
-
 void HIncrbyfloatCmd::DoInitial() {
  if (!CheckArg(argv_.size())) {
  res_.SetRes(CmdRes::kWrongNum, kCmdNameHIncrbyfloat);