Skip to content

Commit

Permalink
feat: Merge branch import_braft (#300)
Browse files Browse the repository at this point in the history
* feat:import braft (#130)

* feat:  Implementing Redis-Raft commands to build a cluster. (#136)

Co-authored-by: panlei-coder <[email protected]>
Co-authored-by: century <[email protected]>
Co-authored-by: panlei-coder <[email protected]>
Co-authored-by: alexstocks <[email protected]>
Co-authored-by: dingxiaoshuai123 <[email protected]>
Co-authored-by: longfar <[email protected]>
  • Loading branch information
7 people authored May 6, 2024
1 parent 184f54b commit e9e535b
Show file tree
Hide file tree
Showing 77 changed files with 4,626 additions and 239 deletions.
60 changes: 60 additions & 0 deletions .github/workflows/import_braft.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
name: Import BRaft Actions (Temporary)

on:
push:
pull_request:
branches: [ "import-braft" ]

jobs:
check_format:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4

- name: Build
run: bash ci/build.sh

- name: Check Format
working-directory: ${{ github.workspace }}/build
run: make check-format

build_on_macos:
runs-on: macos-latest
needs: check_format

steps:
- uses: actions/checkout@v4

- name: Build
env:
CPLUS_INCLUDE_PATH: /opt/homebrew/include
run: |
brew install autoconf
brew install go
sh build.sh
- name: Run Go E2E Tests
working-directory: ${{ github.workspace }}/build
run: |
cd ../tests
go mod tidy
go test ./pikiwidb_suite_test.go ./consistency_test.go -v
build_on_ubuntu:
runs-on: ubuntu-latest
needs: check_format

steps:
- uses: actions/checkout@v4

- name: Build
run: |
bash build.sh
- name: Run Go E2E Tests
working-directory: ${{ github.workspace }}/build
run: |
cd ../tests
go mod tidy
go test ./pikiwidb_suite_test.go ./consistency_test.go -v
2 changes: 2 additions & 0 deletions .github/workflows/pikiwidb.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ jobs:
- uses: actions/checkout@v4

- name: Build
env:
CPLUS_INCLUDE_PATH: /opt/homebrew/include
run: |
brew install autoconf
brew install go
Expand Down
82 changes: 69 additions & 13 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
# Copyright (c) 2023-present, Qihoo, Inc. All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

CMAKE_MINIMUM_REQUIRED(VERSION 3.14)
PROJECT(PikiwiDB)

SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated -g -D'GIT_COMMIT_ID=\"${GIT_COMMIT_ID}\"'")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=return-type -D'GIT_COMMIT_ID=\"${GIT_COMMIT_ID}\"'")

# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24:
IF (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
Expand Down Expand Up @@ -35,7 +40,7 @@ EXECUTE_PROCESS(COMMAND git rev-parse HEAD OUTPUT_VARIABLE GIT_COMMIT_ID OUTPUT_
ADD_DEFINITIONS(-DKPIKIWIDB_GIT_COMMIT_ID="${GIT_COMMIT_ID}")
MESSAGE(STATUS "Git commit id: ${GIT_COMMIT_ID}")

############# You should enable sanitizer if you are developing pika #############
############# You should enable sanitizer if you are developing pikiwidb #############
# Uncomment the following two lines to enable AddressSanitizer to detect memory leaks and other memory-related bugs.
# SET(CMAKE_BUILD_TYPE "Debug")
# SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address -O0 -fno-omit-frame-pointer -fno-optimize-sibling-calls")
Expand Down Expand Up @@ -78,7 +83,7 @@ ELSEIF (CMAKE_SYSTEM_NAME MATCHES "Linux")
set(CMAKE_THREAD_LIBS_INIT "-lpthread")
ELSEIF (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
SET(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++")
SET(CMAKE_CXX_FLAGS "-pthread -Wl,--no-as-needed -ldl -Wno-restrict")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wl,--no-as-needed -ldl")
ENDIF ()
ADD_DEFINITIONS(-DOS_LINUX)
ELSE ()
Expand All @@ -101,28 +106,80 @@ SET(INSTALL_LIBDIR ${STAGED_INSTALL_PREFIX}/lib)
SET(INSTALL_LIBDIR_64 ${STAGED_INSTALL_PREFIX}/lib64)
SET(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${STAGED_INSTALL_PREFIX})
SET(BUILD_SUPPORT_DIR ${PROJECT_SOURCE_DIR}/build_support)
# make sure we use the same compiler for all dependencies
SET(CMAKE_POSITION_INDEPENDENT_CODE ON)

MESSAGE(STATUS "${PROJECT_NAME} staged install: ${STAGED_INSTALL_PREFIX}")
MESSAGE(STATUS "Current platform: ${OS_VERSION} ")
CMAKE_HOST_SYSTEM_INFORMATION(RESULT CPU_CORE QUERY NUMBER_OF_LOGICAL_CORES)
MESSAGE(STATUS "CPU core ${CPU_CORE}")

#openssl
FIND_PACKAGE(OpenSSL REQUIRED)

MESSAGE(STATUS "ssl:" ${OPENSSL_SSL_LIBRARY})
MESSAGE(STATUS "crypto:" ${OPENSSL_CRYPTO_LIBRARY})

ADD_LIBRARY(ssl SHARED IMPORTED GLOBAL)
SET_PROPERTY(TARGET ssl PROPERTY IMPORTED_LOCATION ${OPENSSL_SSL_LIBRARY})

ADD_LIBRARY(crypto SHARED IMPORTED GLOBAL)
SET_PROPERTY(TARGET crypto PROPERTY IMPORTED_LOCATION ${OPENSSL_CRYPTO_LIBRARY})

SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")

SET(THIRD_PARTY_PATH ${CMAKE_CURRENT_BINARY_DIR}/third-party)
SET(THIRD_PARTY_BUILD_TYPE ${CMAKE_BUILD_TYPE})
SET(EXTERNAL_PROJECT_LOG_ARGS
LOG_DOWNLOAD 0
LOG_UPDATE 1
LOG_CONFIGURE 0
LOG_BUILD 0
LOG_TEST 1
LOG_INSTALL 0)

IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
SET(LIB rt)
ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
SET(LIB
pthread
"-framework CoreFoundation"
"-framework CoreGraphics"
"-framework CoreData"
"-framework CoreText"
"-framework Security"
"-framework Foundation"
"-Wl,-U,_MallocExtension_ReleaseFreeMemory"
"-Wl,-U,_ProfilerStart"
"-Wl,-U,_ProfilerStop"
"-Wl,-U,__Z13GetStackTracePPvii")
ENDIF()
SET(LIB ${LIB} CACHE INTERNAL "libs which should be linked for executable target")

INCLUDE(FetchContent)
#include(cmake/CmakeLists.txt)

include(cmake/findTools.cmake)
include(cmake/libevent.cmake)
include(cmake/llhttp.cmake)
include(cmake/fmt.cmake)
include(cmake/spdlog.cmake)
include(cmake/gtest.cmake)
include(cmake/rocksdb.cmake)
INCLUDE(cmake/gflags.cmake)
INCLUDE(cmake/findTools.cmake)
INCLUDE(cmake/leveldb.cmake)
INCLUDE(cmake/libevent.cmake)
INCLUDE(cmake/llhttp.cmake)
INCLUDE(cmake/fmt.cmake)
INCLUDE(cmake/spdlog.cmake)
INCLUDE(cmake/gtest.cmake)
INCLUDE(cmake/rocksdb.cmake)
INCLUDE(cmake/zlib.cmake)
INCLUDE(cmake/protobuf.cmake)
INCLUDE(cmake/brpc.cmake)
INCLUDE(cmake/braft.cmake)

ENABLE_TESTING()

enable_testing()
SET(PROTO_OUTPUT_DIR "${CMAKE_BINARY_DIR}/generated_pb")
FILE(MAKE_DIRECTORY "${PROTO_OUTPUT_DIR}")

ADD_SUBDIRECTORY(src/pstd)
ADD_SUBDIRECTORY(src/net)
ADD_SUBDIRECTORY(src/praft)
ADD_SUBDIRECTORY(src/storage)
ADD_SUBDIRECTORY(src)

Expand Down Expand Up @@ -177,4 +234,3 @@ ADD_CUSTOM_TARGET(cpplint echo '${LINT_FILES}' | xargs -n12 -P8
--linelength=120
--filter=-legal/copyright,-build/header_guard,-runtime/references
)

1 change: 0 additions & 1 deletion README_CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ PikiwiDB 可以和 Redis 之间进行复制,可以读取 Redis 的 rdb 文件
这些特性 PikiwiDB 都有:-)

## 持久化:内存不再是上限

RocksDB 可以配置为 PikiwiDB 的持久化存储引擎,可以存储更多的数据。

## 命令列表
Expand Down
8 changes: 7 additions & 1 deletion build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,13 @@ C_GREEN="\033[32m"

C_END="\033[0m"

BUILD_TYPE=release
BUILD_TIME=$(git log -1 --format=%ai)
BUILD_TIME=${BUILD_TIME: 0: 10}

COMMIT_ID=$(git rev-parse HEAD)
SHORT_COMMIT_ID=${COMMIT_ID: 0: 8}

BUILD_TYPE=Release
VERBOSE=0
CMAKE_FLAGS=""
MAKE_FLAGS=""
Expand Down
46 changes: 46 additions & 0 deletions cmake/braft.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Copyright (c) 2023-present, Qihoo, Inc. All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

INCLUDE(ExternalProject)

SET(BRAFT_SOURCES_DIR ${THIRD_PARTY_PATH}/braft)
SET(BRAFT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/braft)
SET(BRAFT_INCLUDE_DIR "${BRAFT_INSTALL_DIR}/include" CACHE PATH "braft include directory." FORCE)
SET(BRAFT_LIBRARIES "${BRAFT_INSTALL_DIR}/lib/libbraft.a" CACHE FILEPATH "braft library." FORCE)

SET(prefix_path "${THIRD_PARTY_PATH}/install/brpc|${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-build|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-src")

ExternalProject_Add(
extern_braft
${EXTERNAL_PROJECT_LOG_ARGS}
DEPENDS brpc
GIT_REPOSITORY "https:/pikiwidb/braft.git"
GIT_TAG master
PREFIX ${BRAFT_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-DCMAKE_INSTALL_PREFIX=${BRAFT_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=${BRAFT_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_PREFIX_PATH=${prefix_path}
-DBRPC_WITH_GLOG=OFF
${EXTERNAL_OPTIONAL_ARGS}
LIST_SEPARATOR |
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BRAFT_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR:PATH=${BRAFT_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
BUILD_IN_SOURCE 1
BUILD_COMMAND $(MAKE) -j ${CPU_CORE} braft-static
INSTALL_COMMAND mkdir -p ${BRAFT_INSTALL_DIR}/lib/ COMMAND cp ${BRAFT_SOURCES_DIR}/src/extern_braft/output/lib/libbraft.a ${BRAFT_LIBRARIES} COMMAND cp -r ${BRAFT_SOURCES_DIR}/src/extern_braft/output/include ${BRAFT_INCLUDE_DIR}/
)
ADD_DEPENDENCIES(extern_braft brpc)
ADD_LIBRARY(braft STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET braft PROPERTY IMPORTED_LOCATION ${BRAFT_LIBRARIES})
ADD_DEPENDENCIES(braft extern_braft)
49 changes: 49 additions & 0 deletions cmake/brpc.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# Copyright (c) 2023-present, Qihoo, Inc. All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

INCLUDE(ExternalProject)

SET(BRPC_SOURCES_DIR ${THIRD_PARTY_PATH}/brpc)
SET(BRPC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/brpc)
SET(BRPC_INCLUDE_DIR "${BRPC_INSTALL_DIR}/include" CACHE PATH "brpc include directory." FORCE)
SET(BRPC_LIBRARIES "${BRPC_INSTALL_DIR}/lib/libbrpc.a" CACHE FILEPATH "brpc library." FORCE)

# Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
SET(prefix_path "${CMAKE_CURRENT_BINARY_DIR}/_deps/gflags-build|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-build|${CMAKE_CURRENT_BINARY_DIR}/_deps/leveldb-src")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations")
# If minimal .a is need, you can set WITH_DEBUG_SYMBOLS=OFF
EXTERNALPROJECT_ADD(
extern_brpc
${EXTERNAL_PROJECT_LOG_ARGS}
DEPENDS ssl crypto zlib protobuf leveldb gflags
URL https:/apache/brpc/archive/refs/tags/1.8.0.tar.gz
URL_HASH SHA256=13ffb2f1f57c679379a20367c744b3e597614a793ec036cd7580aae90798019d
PREFIX ${BRPC_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-DCMAKE_INSTALL_PREFIX=${BRPC_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=${BRPC_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_PREFIX_PATH=${prefix_path}
-DWITH_GLOG=OFF
-DDOWNLOAD_GTEST=OFF
${EXTERNAL_OPTIONAL_ARGS}
LIST_SEPARATOR |
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BRPC_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR:PATH=${BRPC_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
BUILD_IN_SOURCE 1
BUILD_COMMAND $(MAKE) -j ${CPU_CORE} brpc-static
INSTALL_COMMAND mkdir -p ${BRPC_INSTALL_DIR}/lib/ COMMAND cp ${BRPC_SOURCES_DIR}/src/extern_brpc/output/lib/libbrpc.a ${BRPC_LIBRARIES} COMMAND cp -r ${BRPC_SOURCES_DIR}/src/extern_brpc/output/include ${BRPC_INCLUDE_DIR}/
)
ADD_DEPENDENCIES(extern_brpc ssl crypto zlib protobuf leveldb gflags)
ADD_LIBRARY(brpc STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET brpc PROPERTY IMPORTED_LOCATION ${BRPC_LIBRARIES})
ADD_DEPENDENCIES(brpc extern_brpc)
Loading

0 comments on commit e9e535b

Please sign in to comment.