From e6816f8d1fd9720a6fe66ace36a15821c1525757 Mon Sep 17 00:00:00 2001 From: Michael Legleux Date: Sat, 12 Aug 2023 18:20:54 -0400 Subject: [PATCH] Update Ubuntu build image --- Builds/CMake/RippledRelease.cmake | 2 +- Builds/containers/gitlab-ci/pkgbuild.yml | 2 +- .../dpkg/debian/rippled-reporting.install | 4 +- Builds/containers/packaging/dpkg/debian/rules | 97 ++++++++----------- Builds/containers/packaging/rpm/rippled.spec | 75 +++++++------- Builds/containers/ubuntu-builder/Dockerfile | 7 ++ .../containers/ubuntu-builder/ubuntu_setup.sh | 49 ++-------- .../ubuntu-builder/ubuntu_setup2.sh | 49 ++++++++++ .../ubuntu-builder/ubuntu_setup3.sh | 7 ++ 9 files changed, 158 insertions(+), 134 deletions(-) create mode 100755 Builds/containers/ubuntu-builder/ubuntu_setup2.sh create mode 100755 Builds/containers/ubuntu-builder/ubuntu_setup3.sh diff --git a/Builds/CMake/RippledRelease.cmake b/Builds/CMake/RippledRelease.cmake index 1c60ea80d7b..a0ad3696572 100644 --- a/Builds/CMake/RippledRelease.cmake +++ b/Builds/CMake/RippledRelease.cmake @@ -49,7 +49,7 @@ if (is_root_project) docker run -v ${CMAKE_CURRENT_SOURCE_DIR}:/opt/rippled_bld/pkg/rippled -v ${CMAKE_CURRENT_BINARY_DIR}/packages:/opt/rippled_bld/pkg/out - -t rippleci/rippled-rpm-builder:${container_label} + -t rippled-rpm-builder:${container_label} /bin/bash -c "cp -fpu rippled/Builds/containers/packaging/rpm/build_rpm.sh . && ./build_rpm.sh" VERBATIM USES_TERMINAL diff --git a/Builds/containers/gitlab-ci/pkgbuild.yml b/Builds/containers/gitlab-ci/pkgbuild.yml index f35186acb8e..e2326852867 100644 --- a/Builds/containers/gitlab-ci/pkgbuild.yml +++ b/Builds/containers/gitlab-ci/pkgbuild.yml @@ -14,7 +14,7 @@ variables: RPM_CONTAINER_TAG: "2023-02-13" RPM_CONTAINER_NAME: "rippled-rpm-builder" RPM_CONTAINER_FULLNAME: "${RPM_CONTAINER_NAME}:${RPM_CONTAINER_TAG}" - DPKG_CONTAINER_TAG: "2023-03-20" + DPKG_CONTAINER_TAG: "2023-07-31" DPKG_CONTAINER_NAME: "rippled-dpkg-builder" DPKG_CONTAINER_FULLNAME: "${DPKG_CONTAINER_NAME}:${DPKG_CONTAINER_TAG}" ARTIFACTORY_HOST: "artifactory.ops.ripple.com" diff --git a/Builds/containers/packaging/dpkg/debian/rippled-reporting.install b/Builds/containers/packaging/dpkg/debian/rippled-reporting.install index 255c7b0b5c4..0cee940f59e 100644 --- a/Builds/containers/packaging/dpkg/debian/rippled-reporting.install +++ b/Builds/containers/packaging/dpkg/debian/rippled-reporting.install @@ -1,8 +1,8 @@ -bld/rippled-reporting/rippled-reporting opt/rippled-reporting/bin +build.rippled-reporting/rippled-reporting opt/rippled-reporting/bin cfg/rippled-reporting.cfg opt/rippled-reporting/etc debian/tmp/opt/rippled-reporting/etc/validators.txt opt/rippled-reporting/etc opt/rippled-reporting/bin/update-rippled-reporting.sh opt/rippled-reporting/bin/getRippledReportingInfo opt/rippled-reporting/etc/update-rippled-reporting-cron -etc/logrotate.d/rippled-reporting \ No newline at end of file +etc/logrotate.d/rippled-reporting diff --git a/Builds/containers/packaging/dpkg/debian/rules b/Builds/containers/packaging/dpkg/debian/rules index c7234c847dd..08b180e104c 100755 --- a/Builds/containers/packaging/dpkg/debian/rules +++ b/Builds/containers/packaging/dpkg/debian/rules @@ -15,69 +15,56 @@ override_dh_systemd_start: dh_systemd_start --no-restart-on-upgrade override_dh_auto_configure: - apt install --yes gcc-11 g++-11 - update-alternatives --install \ - /usr/bin/gcc gcc /usr/bin/gcc-11 100 \ - --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ - --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ - --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-11 \ - --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 \ - --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ - --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-11 \ - --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-11 - update-alternatives --set gcc /usr/bin/gcc-11 - env - rm -rf bld - conan profile update settings.compiler.cppstd=20 gcc - conan profile update settings.compiler.version=11 gcc - conan export external/snappy snappy/1.1.9@ + /root/.pyenv/shims/conan export external/snappy snappy/1.1.10@ + /root/.pyenv/shims/conan export external/soci soci/4.0.3@ - conan install . \ - --profile gcc \ - --install-folder bld/rippled \ - --build missing \ - --build boost \ - --build sqlite3 \ - --settings build_type=Release + mkdir build.rippled - cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/ripple \ - -Dstatic=ON \ - -Dunity=OFF \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -Dvalidator_keys=ON \ - -B bld/rippled + cd build.rippled && \ + /root/.pyenv/shims/conan install .. \ + --profile gcc \ + --output-folder . \ + --build missing \ + --settings build_type=Release - conan install . \ - --profile gcc \ - --install-folder bld/rippled-reporting \ - --build missing \ - --build boost \ - --build sqlite3 \ - --build libuv \ - --settings build_type=Release \ - --settings compiler.cppstd=17 \ - --options reporting=True + cd build.rippled && \ + cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -Dvalidator_keys=ON \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake - cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/rippled-reporting \ - -Dstatic=ON \ - -Dunity=OFF \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -Dreporting=ON \ - -B bld/rippled-reporting + mkdir build.rippled-reporting + + cd build.rippled-reporting && \ + /root/.pyenv/shims/conan install .. \ + --profile gcc \ + --output-folder . \ + --settings compiler.cppstd=17 \ + --settings build_type=Release \ + --build missing \ + --build boost \ + --build sqlite3 \ + --build libuv \ + --options reporting=True + + cd build.rippled-reporting && \ + cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -Dvalidator_keys=ON \ + -Dstatic=ON \ + -Dunity=OFF \ + -Dreporting=ON \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake override_dh_auto_build: - cmake --build bld/rippled --target rippled --target validator-keys -j${nproc} - cmake --build bld/rippled-reporting --target rippled -j${nproc} + cmake --build build.rippled --target rippled --target validator-keys --parallel 8 + cmake --build build.rippled-reporting --target rippled --parallel 8 override_dh_auto_install: - cmake --install bld/rippled --prefix debian/tmp/opt/ripple - install -D bld/rippled/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys + cmake --install build.rippled --prefix debian/tmp/opt/ripple + install -D build.rippled/validator-keys/validator-keys debian/tmp/opt/ripple/bin/validator-keys install -D Builds/containers/shared/update-rippled.sh debian/tmp/opt/ripple/bin/update-rippled.sh install -D bin/getRippledInfo debian/tmp/opt/ripple/bin/getRippledInfo install -D Builds/containers/shared/update-rippled-cron debian/tmp/opt/ripple/etc/update-rippled-cron diff --git a/Builds/containers/packaging/rpm/rippled.spec b/Builds/containers/packaging/rpm/rippled.spec index fe451d645d4..0c2f454fa42 100644 --- a/Builds/containers/packaging/rpm/rippled.spec +++ b/Builds/containers/packaging/rpm/rippled.spec @@ -36,41 +36,44 @@ History server for XRP Ledger %setup -c -n rippled %build -rm -rf ~/.conan/profiles/default - -cp /opt/libcstd/libstdc++.so.6.0.22 /usr/lib64 -cp /opt/libcstd/libstdc++.so.6.0.22 /lib64 -ln -sf /usr/lib64/libstdc++.so.6.0.22 /usr/lib64/libstdc++.so.6 -ln -sf /lib64/libstdc++.so.6.0.22 /usr/lib64/libstdc++.so.6 +source /opt/rh/devtoolset-11/enable source /opt/rh/rh-python38/enable + pip install "conan<2" + conan profile new default --detect -conan profile update settings.compiler.libcxx=libstdc++11 default conan profile update settings.compiler.cppstd=20 default +conan profile update settings.compiler.libcxx=libstdc++11 default cd rippled -mkdir -p bld.rippled -conan export external/snappy snappy/1.1.9@ +conan export external/snappy snappy/1.1.10@ +conan export external/soci soci/4.0.3@ +mkdir -p bld.rippled pushd bld.rippled + +cp /opt/libcstd/libstdc++.so.6.0.22 /usr/lib64 +cp /opt/libcstd/libstdc++.so.6.0.22 /lib64 +ln -sf /usr/lib64/libstdc++.so.6.0.22 /usr/lib64/libstdc++.so.6 +ln -sf /lib64/libstdc++.so.6.0.22 /usr/lib64/libstdc++.so.6 + conan install .. \ - --settings build_type=Release \ - --output-folder . \ - --build missing - -cmake -G Ninja \ - -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ - -DCMAKE_INSTALL_PREFIX=%{_prefix} \ - -DCMAKE_BUILD_TYPE=Release \ - -Dunity=OFF \ - -Dstatic=ON \ - -Dvalidator_keys=ON \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - .. + --profile default \ + --output-folder . \ + --build missing \ + --settings build_type=Release + +cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -Dvalidator_keys=ON \ + -DCMAKE_INSTALL_PREFIX=%{_prefix} \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake cmake --build . --parallel $(nproc) --target rippled --target validator-keys + popd mkdir -p bld.rippled-reporting @@ -83,16 +86,16 @@ conan install .. \ --settings compiler.cppstd=17 \ --options reporting=True -cmake -G Ninja \ - -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ - -DCMAKE_INSTALL_PREFIX=%{_prefix} \ - -DCMAKE_BUILD_TYPE=Release \ - -Dunity=OFF \ - -Dstatic=ON \ - -Dvalidator_keys=ON \ - -Dreporting=ON \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - .. +cmake .. \ + -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ + -DCMAKE_INSTALL_PREFIX=%{_prefix} \ + -DCMAKE_BUILD_TYPE=Release \ + -Dunity=OFF \ + -Dstatic=ON \ + -Dvalidator_keys=ON \ + -Dreporting=ON \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ cmake --build . --parallel $(nproc) --target rippled @@ -103,7 +106,7 @@ test -e /etc/pki/tls || { mkdir -p /etc/pki; ln -s /usr/lib/ssl /etc/pki/tls; } rm -rf $RPM_BUILD_ROOT DESTDIR=$RPM_BUILD_ROOT cmake --build rippled/bld.rippled --target install #-- -v mkdir -p $RPM_BUILD_ROOT -rm -rf ${RPM_BUILD_ROOT}/%{_prefix}/lib64/ +rm -rf ${RPM_BUILD_ROOT}%{_prefix}/lib64/ install -d ${RPM_BUILD_ROOT}/etc/opt/ripple install -d ${RPM_BUILD_ROOT}/usr/local/bin @@ -130,9 +133,9 @@ install -D rippled/bld.rippled-reporting/rippled-reporting ${RPM_BUILD_ROOT}%{_b install -D ./rippled/cfg/rippled-reporting.cfg ${RPM_BUILD_ROOT}%{_prefix}/etc/rippled-reporting.cfg install -D ./rippled/cfg/validators-example.txt ${RPM_BUILD_ROOT}%{_prefix}/etc/validators.txt install -D ./rippled/Builds/containers/packaging/rpm/50-rippled-reporting.preset ${RPM_BUILD_ROOT}/usr/lib/systemd/system-preset/50-rippled-reporting.preset -ln -s %{_prefix}/bin/rippled-reporting ${RPM_BUILD_ROOT}/usr/local/bin/rippled-reporting -ln -s %{_prefix}/etc/rippled-reporting.cfg ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/rippled-reporting.cfg -ln -s %{_prefix}/etc/validators.txt ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/validators.txt +ln -sf %{_prefix}/bin/rippled-reporting ${RPM_BUILD_ROOT}/usr/local/bin/rippled-reporting +ln -sf %{_prefix}/etc/rippled-reporting.cfg ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/rippled-reporting.cfg +ln -sf %{_prefix}/etc/validators.txt ${RPM_BUILD_ROOT}/etc/opt/rippled-reporting/validators.txt install -d $RPM_BUILD_ROOT/var/log/rippled-reporting install -d $RPM_BUILD_ROOT/var/lib/rippled-reporting install -D ./rippled/Builds/containers/shared/rippled-reporting.service ${RPM_BUILD_ROOT}/usr/lib/systemd/system/rippled-reporting.service diff --git a/Builds/containers/ubuntu-builder/Dockerfile b/Builds/containers/ubuntu-builder/Dockerfile index 23723967fc2..e2604d3a1ca 100644 --- a/Builds/containers/ubuntu-builder/Dockerfile +++ b/Builds/containers/ubuntu-builder/Dockerfile @@ -6,8 +6,15 @@ LABEL git-commit=$GIT_COMMIT WORKDIR /root COPY ubuntu-builder/ubuntu_setup.sh . +COPY ubuntu-builder/ubuntu_setup2.sh . + RUN ./ubuntu_setup.sh && rm ubuntu_setup.sh +RUN ./ubuntu_setup2.sh && rm ubuntu_setup2.sh + +COPY ubuntu-builder/ubuntu_setup3.sh . +RUN ./ubuntu_setup3.sh && rm ubuntu_setup3.sh + RUN mkdir -m 777 -p /opt/rippled_bld/pkg/ WORKDIR /opt/rippled_bld/pkg diff --git a/Builds/containers/ubuntu-builder/ubuntu_setup.sh b/Builds/containers/ubuntu-builder/ubuntu_setup.sh index cd8db75153d..778bcf53c0b 100755 --- a/Builds/containers/ubuntu-builder/ubuntu_setup.sh +++ b/Builds/containers/ubuntu-builder/ubuntu_setup.sh @@ -6,37 +6,33 @@ set -o xtrace # Parameters -gcc_version=${GCC_VERSION:-10} -cmake_version=${CMAKE_VERSION:-3.25.1} -conan_version=${CONAN_VERSION:-1.59} +gcc_version=${GCC_VERSION:-11} + +export DEBIAN_FRONTEND=noninteractive apt update # Iteratively build the list of packages to install so that we can interleave # the lines with comments explaining their inclusion. dependencies='' -# - to identify the Ubuntu version -dependencies+=' lsb-release' # - for add-apt-repository dependencies+=' software-properties-common' # - to download CMake dependencies+=' curl' # - to build CMake dependencies+=' libssl-dev' -# - Python headers for Boost.Python -dependencies+=' python3-dev' -# - to install Conan -dependencies+=' python3-pip' +# - for Python +dependencies+=' libbz2-dev liblzma-dev libsqlite3-dev' # - to download rippled dependencies+=' git' # - CMake generators (but not CMake itself) dependencies+=' make ninja-build' -apt install --yes ${dependencies} +apt-get install --yes ${dependencies} add-apt-repository --yes ppa:ubuntu-toolchain-r/test -apt install --yes gcc-${gcc_version} g++-${gcc_version} \ - debhelper debmake debsums gnupg dh-buildinfo dh-make dh-systemd cmake \ - ninja-build zlib1g-dev make cmake ninja-build autoconf automake \ - pkg-config apt-transport-https +apt-get install --yes gcc-${gcc_version} g++-${gcc_version} +apt-get install --yes build-essential libssl-dev zlib1g-dev \ +libbz2-dev libreadline-dev libsqlite3-dev curl \ +libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # Give us nice unversioned aliases for gcc and company. update-alternatives --install \ @@ -49,28 +45,3 @@ update-alternatives --install \ --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-dump-${gcc_version} \ --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-tool-${gcc_version} update-alternatives --auto gcc - -# Download and unpack CMake. -cmake_slug="cmake-${cmake_version}" -curl --location --remote-name \ - "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_slug}.tar.gz" -tar xzf ${cmake_slug}.tar.gz -rm ${cmake_slug}.tar.gz - -# Build and install CMake. -cd ${cmake_slug} -./bootstrap --parallel=$(nproc) -make --jobs $(nproc) -make install -cd .. -rm --recursive --force ${cmake_slug} - -# Install Conan. -pip3 install conan==${conan_version} - -conan profile new --detect gcc -conan profile update settings.compiler=gcc gcc -conan profile update settings.compiler.version=${gcc_version} gcc -conan profile update settings.compiler.libcxx=libstdc++11 gcc -conan profile update env.CC=/usr/bin/gcc gcc -conan profile update env.CXX=/usr/bin/g++ gcc diff --git a/Builds/containers/ubuntu-builder/ubuntu_setup2.sh b/Builds/containers/ubuntu-builder/ubuntu_setup2.sh new file mode 100755 index 00000000000..07ea577c68a --- /dev/null +++ b/Builds/containers/ubuntu-builder/ubuntu_setup2.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o xtrace + +# Parameters + +gcc_version=${GCC_VERSION:-11} +cmake_version=${CMAKE_VERSION:-3.25.1} +cmake_sha256=1c511d09516af493694ed9baf13c55947a36389674d657a2d5e0ccedc6b291d8 +conan_version=${CONAN_VERSION:-1.60} + +curl https://pyenv.run | bash +export PYENV_ROOT="$HOME/.pyenv" +command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH" +eval "$(pyenv init -)" + +pyenv install 3.11.2 +pyenv global 3.11.2 + +# Download and unpack CMake. +cmake_slug="cmake-${cmake_version}" +cmake_archive="${cmake_slug}.tar.gz" +curl --location --remote-name \ + "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_archive}" +echo "${cmake_sha256} ${cmake_archive}" | sha256sum --check +tar -xzf ${cmake_archive} +rm ${cmake_archive} + +# Build and install CMake. +cd ${cmake_slug} +./bootstrap --parallel=$(nproc) +make --jobs $(nproc) +make install +cd .. +rm --recursive --force ${cmake_slug} + +# Install Conan. +pip install --upgrade pip +pip install conan==${conan_version} + +conan profile new --detect gcc +conan profile update settings.compiler=gcc gcc +conan profile update settings.compiler.version=${gcc_version} gcc +conan profile update settings.compiler.libcxx=libstdc++11 gcc +conan profile update settings.compiler.cppstd=20 gcc +conan profile update env.CC=/usr/bin/gcc gcc +conan profile update env.CXX=/usr/bin/g++ gcc diff --git a/Builds/containers/ubuntu-builder/ubuntu_setup3.sh b/Builds/containers/ubuntu-builder/ubuntu_setup3.sh new file mode 100755 index 00000000000..91c7ca2406a --- /dev/null +++ b/Builds/containers/ubuntu-builder/ubuntu_setup3.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o xtrace + +apt-get install --yes build-essential fakeroot devscripts cmake debhelper dh-systemd