fix:Synchronize the latest changes to the import-braft branch (#180)

fix:Synchronize the latest changes to the import-braft branch

CMakeLists.txt

@@ -1,3 +1,8 @@
+# Copyright (c) 2023-present, Qihoo, Inc. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+
 CMAKE_MINIMUM_REQUIRED(VERSION 3.14)
 PROJECT(PikiwiDB)
 
@@ -124,7 +129,6 @@ SET(EXTERNAL_PROJECT_LOG_ARGS
  LOG_INSTALL 0)
 
 INCLUDE(FetchContent)
-#include(cmake/CmakeLists.txt)
 
 include(findTools)
 include(leveldb)
@@ -138,7 +142,6 @@ include(rocksdb)
 include(zlib)
 include(protobuf)
 include(brpc)
-# ExternalProject_Add_StepTargets(extern_brpc build)
 include(braft)
 
 enable_testing()

build.sh

@@ -12,7 +12,7 @@ BUILD_TIME=${BUILD_TIME: 0: 10}
 COMMIT_ID=$(git rev-parse HEAD)
 SHORT_COMMIT_ID=${COMMIT_ID: 0: 8}
 
-BUILD_TYPE=release
+BUILD_TYPE=Release
 VERBOSE=0
 CMAKE_FLAGS=""
 MAKE_FLAGS=""

cmake/gflags.cmake

@@ -7,6 +7,9 @@ include_guard()
 
 include(cmake/utils.cmake)
 
+set(MY_BUILD_TYPE ${CMAKE_BUILD_TYPE})
+set(CMAKE_BUILD_TYPE ${THIRD_PARTY_BUILD_TYPE})
+
 FetchContent_DeclareGitHubWithMirror(gflags
  gflags/gflags v2.2.2
  SHA256=19713a36c9f32b33df59d1c79b4958434cb005b5b47dc5400a7a4b078111d9b5
@@ -28,3 +31,5 @@ target_link_libraries(gflags_static Threads::Threads)
 SET(GFLAGS_INCLUDE_PATH ${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build/include)
 SET(GFLAGS_LIBRARY ${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build/libgflags.a)
 SET(GFLAGS_LIB ${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build/libgflags.a)
+
+set(CMAKE_BUILD_TYPE ${MY_BUILD_TYPE})

cmake/glog.cmake

@@ -22,4 +22,8 @@ FetchContent_MakeAvailableWithArgs(glog
 )
 
 SET(GLOG_INCLUDE_PATH ${CMAKE_CURRENT_BINARY_DIR}/_deps/glog-src/src CACHE BOOL "" FORCE)
-SET(GLOG_LIB ${CMAKE_CURRENT_BINARY_DIR}/_deps/glog-build/libglog.a CACHE BOOL "" FORCE)
+if(CMAKE_BUILD_TYPE STREQUAL "Release")
+ SET(GLOG_LIB ${CMAKE_CURRENT_BINARY_DIR}/_deps/glog-build/libglog.a CACHE BOOL "" FORCE)
+elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
+ SET(GLOG_LIB ${CMAKE_CURRENT_BINARY_DIR}/_deps/glog-build/libglogd.a CACHE BOOL "" FORCE)
+endif()

cmake/protobuf.cmake

@@ -113,7 +113,7 @@ FUNCTION(build_protobuf TARGET_NAME)
  # Make sure zlib's two headers are in your /Path/to/install/include path,
  # and delete libz.so which we don't need
  FILE(WRITE ${PROTOBUF_SOURCES_DIR}/src/config.sh
- "rm -f ${THIRD_PARTY_PATH}/install/zlib/lib/libz.so* && mkdir -p ${THIRD_PARTY_PATH}/install/protobuf/include && cp ${THIRD_PARTY_PATH}/install/zlib/include/* ${THIRD_PARTY_PATH}/install/protobuf/include/ && ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_C_FLAGS='${CMAKE_C_FLAGS}' -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} -DCMAKE_C_FLAGS_RELEASE='${CMAKE_C_FLAGS_RELEASE}' -DCMAKE_CXX_FLAGS='${CMAKE_CXX_FLAGS}' -DCMAKE_CXX_FLAGS_RELEASE='${CMAKE_CXX_FLAGS_RELEASE}' -DCMAKE_CXX_FLAGS_DEBUG='${CMAKE_CXX_FLAGS_DEBUG}' -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_SKIP_RPATH=ON -Dprotobuf_WITH_ZLIB=ON -DZLIB_INCLUDE_DIR=${THIRD_PARTY_PATH}/install/zlib/include -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE='${THIRD_PARTY_BUILD_TYPE}' -DCMAKE_PREFIX_PATH=${prefix_path} ${EXTERNAL_OPTIONAL_ARGS}"
+ "rm ${THIRD_PARTY_PATH}/install/zlib/lib/libz.so* -f && mkdir -p ${THIRD_PARTY_PATH}/install/protobuf/include && cp ${THIRD_PARTY_PATH}/install/zlib/include/* ${THIRD_PARTY_PATH}/install/protobuf/include/ && ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_C_FLAGS='${CMAKE_C_FLAGS}' -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} -DCMAKE_C_FLAGS_RELEASE='${CMAKE_C_FLAGS_RELEASE}' -DCMAKE_CXX_FLAGS='${CMAKE_CXX_FLAGS}' -DCMAKE_CXX_FLAGS_RELEASE='${CMAKE_CXX_FLAGS_RELEASE}' -DCMAKE_CXX_FLAGS_DEBUG='${CMAKE_CXX_FLAGS_DEBUG}' -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_SKIP_RPATH=ON -Dprotobuf_WITH_ZLIB=ON -DZLIB_INCLUDE_DIR=${THIRD_PARTY_PATH}/install/zlib/include -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE='${THIRD_PARTY_BUILD_TYPE}' -DCMAKE_PREFIX_PATH=${prefix_path} ${EXTERNAL_OPTIONAL_ARGS}"
  )
 
  ExternalProject_Add(

cmake/rocksdb.cmake

@@ -24,7 +24,7 @@ FetchContent_MakeAvailableWithArgs(rocksdb
  WITH_LIBURING=OFF
  WITH_LZ4=OFF
  WITH_SNAPPY=OFF
- WITH_ZLIB=OFF
+ WITH_ZLIB=ON
  WITH_ZSTD=OFF
  WITH_GFLAGS=OFF
  USE_RTTI=ON

pikiwidb.conf

@@ -17,7 +17,7 @@ port 9221
 timeout 0
 
 # Directory to store the data of PikiwiDB.
-db-path ./db
+db-path ./db/
 
 # Specify the server verbosity level.
 # This can be one of:
@@ -35,7 +35,7 @@ logfile stdout
 # Set the number of databases. The default database is DB 0, you can select
 # a different one on a per-connection basis using SELECT <dbid> where
 # dbid is a number between 0 and 'databases'-1
-databases 16
+databases 2
 
 ################################ SNAPSHOTTING #################################
 #
@@ -158,6 +158,10 @@ slave-read-only yes # PikiwiDB always set slave read only
 #
 # repl-ping-slave-period 10
 
+# Limit the maximum number of bytes returned to the client, currently only the hgetall command will be restricted
+# By default the size is 1073741824.
+# max-client-response-size 1073741824
+
 # The following option sets a timeout for both Bulk transfer I/O timeout and
 # master data or ping response timeout. The default value is 60 seconds.
 #
@@ -339,3 +343,9 @@ backend 1
 backendpath dump
 # the frequency of dump to backend per second
 backendhz 10
+# the rocksdb number per db
+db-instance-num 5
+# default 86400 * 7
+rocksdb-ttl-second 604800
+# default 86400 * 3
+rocksdb-periodic-second 259200;

src/CMakeLists.txt

@@ -1,3 +1,7 @@
+# Copyright (c) 2023-present, Qihoo, Inc. All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
 
 AUX_SOURCE_DIRECTORY(. PIKIWIDB_SRC)
 

src/base_cmd.h

@@ -44,6 +44,7 @@ const std::string kCmdNameBitOp = "bitop";
 const std::string kCmdNameGetBit = "getbit";
 const std::string kCmdNameBitCount = "bitcount";
 const std::string kCmdNameGetRange = "getrange";
+const std::string kCmdNameSetRange = "setrange";
 const std::string kCmdNameDecr = "decr";
 const std::string kCmdNameIncr = "incr";
 
@@ -59,6 +60,7 @@ const std::string kCmdNameConfig = "config";
 const std::string kCmdNameFlushdb = "flushdb";
 const std::string kCmdNameFlushall = "flushall";
 const std::string kCmdNameAuth = "auth";
+const std::string kCmdNameSelect = "select";
 
 // hash cmd
 const std::string kCmdNameHSet = "hset";
@@ -72,13 +74,26 @@ const std::string kCmdNameHLen = "hlen";
 const std::string kCmdNameHStrLen = "hstrlen";
 const std::string kCmdNameHScan = "hscan";
 const std::string kCmdNameHVals = "hvals";
+const std::string kCmdNameHIncrbyFloat = "hincrbyfloat";
+const std::string kCmdNameHRandField = "hrandfield";
 
 // set cmd
 const std::string kCmdNameSIsMember = "sismember";
 const std::string kCmdNameSAdd = "sadd";
 const std::string kCmdNameSUnionStore = "sunionstore";
+const std::string kCmdNameSInter = "sinter";
 const std::string kCmdNameSRem = "srem";
+const std::string kCmdNameSInterStore = "sinterstore";
 const std::string kCmdNameSUnion = "sunion";
+const std::string kCmdNameSCard = "scard";
+const std::string kCmdNameSMove = "smove";
+
+// list cmd
+const std::string kCmdNameLPush = "lpush";
+const std::string kCmdNameRPush = "rpush";
+const std::string kCmdNameRPop = "rpop";
+const std::string kCmdNameLRem = "lrem";
+const std::string kCmdNameLRange = "lrange";
 
 enum CmdFlags {
  kCmdFlagsWrite = (1 << 0), // May modify the dataset

src/client.cc

@@ -326,7 +326,6 @@ int PClient::handlePacket(const char* start, int bytes) {
 
  DEBUG("client {}, cmd {}", conn->GetUniqueId(), cmdName_);
 
- PSTORE.SelectDB(db_);
  FeedMonitors(params_);
 
  // const PCommandInfo* info = PCommandTable::GetCommandInfo(cmdName_);
@@ -403,12 +402,11 @@ PClient* PClient::Current() { return s_current; }
 
 PClient::PClient(TcpConnection* obj)
  : tcp_connection_(std::static_pointer_cast<TcpConnection>(obj->shared_from_this())),
- db_(0),
+ dbno_(0),
  flag_(0),
  name_("clientxxx"),
  parser_(params_) {
  auth_ = false;
- SelectDB(0);
  reset();
 }
 
@@ -503,15 +501,6 @@ void PClient::Close() {
  }
 }
 
-bool PClient::SelectDB(int db) {
- if (PSTORE.SelectDB(db) >= 0) {
- db_ = db;
- return true;
- }
-
- return false;
-}
-
 void PClient::reset() {
  s_current = nullptr;
  parser_.Reset();
@@ -625,10 +614,17 @@ void PClient::TransferToSlaveThreads() {
  auto slave_loop = tcp_connection->SelectSlaveEventLoop();
  auto id = tcp_connection->GetUniqueId();
  auto event_object = loop->GetEventObject(id);
- loop->Unregister(event_object);
- event_object->SetUniqueId(-1);
- slave_loop->Register(event_object, 0);
- tcp_connection->ResetEventLoop(slave_loop);
+ auto del_conn = [loop, slave_loop, event_object]() {
+ loop->Unregister(event_object);
+ event_object->SetUniqueId(-1);
+ auto tcp_connection = std::dynamic_pointer_cast<TcpConnection>(event_object);
+ assert(tcp_connection);
+ tcp_connection->ResetEventLoop(slave_loop);
+
+ auto add_conn = [slave_loop, event_object]() { slave_loop->Register(event_object, 0); };
+ slave_loop->Execute(std::move(add_conn));
+ };
+ loop->Execute(std::move(del_conn));
  }
 }
 

src/client.h

@@ -82,7 +82,6 @@ class CmdRes {
 
  void SetRes(CmdRet _ret, const std::string& content = "");
 
- protected:
  inline void RedisAppendContent(std::string& str, const std::string& value) {
  str.append(value.data(), value.size());
  str.append(CRLF);
@@ -124,7 +123,11 @@ class PClient : public std::enable_shared_from_this<PClient>, public CmdRes {
 
  void Close();
 
- bool SelectDB(int db);
+ // dbno
+ void SetCurrentDB(int dbno) { dbno_ = dbno; }
+
+ int GetCurrentDB() { return dbno_; }
+
  static PClient* Current();
 
  // multi
@@ -211,7 +214,7 @@ class PClient : public std::enable_shared_from_this<PClient>, public CmdRes {
 
  PProtoParser parser_;
 
- int db_ = -1;
+ int dbno_;
 
  std::unordered_set<std::string> channels_;
  std::unordered_set<std::string> pattern_channels_;

src/cmd_admin.cc

@@ -63,4 +63,19 @@ void FlushallCmd::DoCmd(PClient* client) {
  client->SetRes(CmdRes::kOK);
 }
 
+SelectCmd::SelectCmd(const std::string& name, int16_t arity)
+ : BaseCmd(name, arity, kCmdFlagsAdmin | kCmdFlagsReadonly, kAclCategoryAdmin) {}
+
+bool SelectCmd::DoInitial(PClient* client) { return true; }
+
+void SelectCmd::DoCmd(PClient* client) {
+ int index = atoi(client->argv_[1].c_str());
+ if (index < 0 || index >= g_config.databases) {
+ client->SetRes(CmdRes::kInvalidIndex, kCmdNameSelect + " DB index is out of range");
+ return;
+ }
+ client->SetCurrentDB(index);
+ client->SetRes(CmdRes::kOK);
+}
+
 } // namespace pikiwidb

src/cmd_admin.h

@@ -8,9 +8,12 @@
 #pragma once
 
 #include "base_cmd.h"
+#include "config.h"
 
 namespace pikiwidb {
 
+extern PConfig g_config;
+
 class CmdConfig : public BaseCmdGroup {
  public:
  CmdConfig(const std::string& name, int arity);
@@ -70,4 +73,15 @@ class FlushallCmd : public BaseCmd {
  void DoCmd(PClient* client) override;
 };
 
+class SelectCmd : public BaseCmd {
+ public:
+ SelectCmd(const std::string& name, int16_t arity);
+
+ protected:
+ bool DoInitial(PClient* client) override;
+
+ private:
+ void DoCmd(PClient* client) override;
+};
+
 } // namespace pikiwidb