From 3e4e8e170cb471a1ec7822b3139da02176b7700d Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Tue, 20 Oct 2026 05:38:06 +0700 Subject: [PATCH 1/8] feat: add zstd as an optional dependency Block files and potentially some records in evodb [such as snapshots of masternode lists] are easy to be compressed up to 20%. In the future zstd may be used as well to compress mn-list's snapshots in evodb database and other heavy compressable data. Add the depends package (static library, headers and pkg-config file), the configure check, the CPPFLAGS/LDADD wiring, the host-package lists for the non-depends CI jobs and the build docs. The library is optional like ZeroMQ and BerkeleyDB, since it only saves disk space: --with-zstd defaults to auto, --without-zstd and NO_ZSTD=1 for depends turn it off, and ENABLE_ZSTD tells the code whether it is linked. --- configure.ac | 22 ++++++++++++++++++++++ depends/Makefile | 5 ++++- depends/README.md | 1 + depends/config.site.in | 4 ++++ depends/packages/packages.mk | 2 ++ depends/packages/zstd.mk | 19 +++++++++++++++++++ doc/build-freebsd.md | 2 +- doc/build-netbsd.md | 2 +- doc/build-openbsd.md | 2 +- doc/build-osx.md | 2 +- doc/build-unix.md | 10 +++++++++- doc/dependencies.md | 1 + src/Makefile.am | 10 +++++----- src/Makefile.bench.include | 1 + src/Makefile.qt.include | 2 +- src/Makefile.qttest.include | 2 +- src/Makefile.test.include | 3 ++- 17 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 depends/packages/zstd.mk diff --git a/configure.ac b/configure.ac index a9e35495bca7..69d4334693a0 100644 --- a/configure.ac +++ b/configure.ac @@ -149,6 +149,12 @@ AC_ARG_WITH([bdb], [use_bdb=$withval], [use_bdb=auto]) +AC_ARG_WITH([zstd], + [AS_HELP_STRING([--without-zstd], + [disable zstd compression of block and undo files (default is enabled if libzstd is found)])], + [use_zstd=$withval], + [use_zstd=auto]) + AC_ARG_ENABLE([usdt], [AS_HELP_STRING([--enable-usdt], [enable tracepoints for Userspace, Statically Defined Tracing (default is yes if sys/sdt.h is found)])], @@ -1695,6 +1701,21 @@ AC_CHECK_LIB([gmp], [__gmpz_init],GMP_LIBS=-lgmp, AC_MSG_ERROR(libgmp missing)) CPPFLAGS="$TEMP_CPPFLAGS" LDFLAGS="$TEMP_LDFLAGS" +dnl zstd check +if test "$use_zstd" != "no"; then + PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.4.0], + [AC_DEFINE([ENABLE_ZSTD], [1], [Define this symbol to enable zstd compression]) + use_zstd=yes], + [if test "$use_zstd" = "yes"; then + AC_MSG_ERROR([libzstd version 1.4.0 or greater not found.]) + fi + AC_MSG_WARN([libzstd version 1.4.0 or greater not found, disabling zstd compression]) + use_zstd=no]) +fi +if test "$use_zstd" = "yes" && test "$suppress_external_warnings" != "no"; then + ZSTD_CFLAGS=SUPPRESS_WARNINGS($ZSTD_CFLAGS) +fi + AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) dnl libmultiprocess library check @@ -2104,6 +2125,7 @@ if test $bitcoin_enable_qt != "no"; then echo " with qr = $use_qr" fi echo " with zmq = $use_zmq" +echo " with zstd = $use_zstd" if test $enable_fuzz = "no"; then echo " with test = $use_tests" else diff --git a/depends/Makefile b/depends/Makefile index a50aabc0bb27..2bddb9067cf8 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -40,6 +40,7 @@ NO_BDB ?= NO_SQLITE ?= NO_WALLET ?= NO_ZMQ ?= +NO_ZSTD ?= NO_UPNP ?= NO_USDT ?= NO_NATPMP ?= @@ -170,10 +171,11 @@ upnp_packages_$(NO_UPNP) = $(upnp_packages) natpmp_packages_$(NO_NATPMP) = $(natpmp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) +zstd_packages_$(NO_ZSTD) = $(zstd_packages) multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) $(zstd_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(zmq_packages_),) @@ -246,6 +248,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@no_qt@|$(NO_QT)|' \ -e 's|@no_qr@|$(NO_QR)|' \ -e 's|@no_zmq@|$(NO_ZMQ)|' \ + -e 's|@no_zstd@|$(NO_ZSTD)|' \ -e 's|@no_wallet@|$(NO_WALLET)|' \ -e 's|@no_bdb@|$(NO_BDB)|' \ -e 's|@no_sqlite@|$(NO_SQLITE)|' \ diff --git a/depends/README.md b/depends/README.md index ef640a65998b..4b74fb6424a6 100644 --- a/depends/README.md +++ b/depends/README.md @@ -82,6 +82,7 @@ The following can be set when running make: `make FOO=bar` - `NO_QT`: Don't download/build/cache Qt and its dependencies - `NO_QR`: Don't download/build/cache packages needed for enabling qrencode - `NO_ZMQ`: Don't download/build/cache packages needed for enabling ZeroMQ +- `NO_ZSTD`: Don't download/build/cache zstd (block files are then stored uncompressed) - `NO_WALLET`: Don't download/build/cache libs needed to enable the wallet - `NO_BDB`: Don't download/build/cache BerkeleyDB - `NO_SQLITE`: Don't download/build/cache SQLite diff --git a/depends/config.site.in b/depends/config.site.in index 398a09b74c63..0d3ad525e8fc 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -70,6 +70,10 @@ if test -z "$enable_zmq" && test -n "@no_zmq@"; then enable_zmq=no fi +if test -z "$with_zstd" && test -n "@no_zstd@"; then + with_zstd=no +fi + if test -z "$enable_usdt" && test -n "@no_usdt@"; then enable_usdt=no fi diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 59232930eb38..930e4277a40a 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -18,6 +18,8 @@ sqlite_packages=sqlite zmq_packages=zeromq +zstd_packages=zstd + upnp_packages=miniupnpc natpmp_packages=libnatpmp diff --git a/depends/packages/zstd.mk b/depends/packages/zstd.mk new file mode 100644 index 000000000000..0be5db336712 --- /dev/null +++ b/depends/packages/zstd.mk @@ -0,0 +1,19 @@ +package=zstd +$(package)_version=1.5.7 +$(package)_download_path=https://github.com/facebook/zstd/releases/download/v$($(package)_version) +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 + +define $(package)_set_vars + $(package)_build_opts=CC="$($(package)_cc)" AR="$($(package)_ar)" + $(package)_build_opts+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" LDFLAGS="$($(package)_ldflags)" + $(package)_build_opts+=ZSTD_LEGACY_SUPPORT=0 ZSTD_LIB_DEPRECATED=0 ZSTD_LIB_DICTBUILDER=0 +endef + +define $(package)_build_cmds + $(MAKE) -C lib libzstd.a $($(package)_build_opts) +endef + +define $(package)_stage_cmds + $(MAKE) -C lib install-static install-includes install-pc PREFIX=$(host_prefix) DESTDIR=$($(package)_staging_dir) $($(package)_build_opts) +endef diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index d33629b551c8..768b9a06048f 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -10,7 +10,7 @@ This guide describes how to build dashd, command-line utilities, and GUI on Free Run the following as root to install the base dependencies for building. ```bash -pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf libbacktrace +pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf libbacktrace zstd ``` diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index 86f72e1096e4..e24fd7e49648 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -14,7 +14,7 @@ Install the required dependencies the usual way you [install software on NetBSD] The example commands below use `pkgin`. ```bash -pkgin install autoconf automake libtool pkg-config git gmake boost libevent gmp +pkgin install autoconf automake libtool pkg-config git gmake boost libevent gmp zstd ``` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 8b0ee196eb46..8ccc3e7e25e4 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -10,7 +10,7 @@ This guide describes how to build dashd, command-line utilities, and GUI on Open Run the following as root to install the base dependencies for building. ```bash -pkg_add bash git gmake gmp libevent libtool boost +pkg_add bash git gmake gmp libevent libtool boost zstd # Select the newest version of the following packages: pkg_add autoconf automake python diff --git a/doc/build-osx.md b/doc/build-osx.md index 87914d09affc..b7590ca99b44 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -47,7 +47,7 @@ To install, run the following from your terminal: See [dependencies.md](dependencies.md) for a complete overview. ``` bash -brew install automake libtool boost gmp pkg-config libevent +brew install automake libtool boost gmp pkg-config libevent zstd ``` ### 4. Clone Dash repository diff --git a/doc/build-unix.md b/doc/build-unix.md index 419df8488ab1..51ceeee7540e 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -64,6 +64,10 @@ executables, which are based on BerkeleyDB 4.8. If you do not care about wallet To build Dash Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) +zstd compresses block and undo files on disk (see: `--with-zstd`, package `libzstd-dev` +on Debian and Ubuntu, `libzstd-devel` on Fedora); a build without it cannot read block files written +with it. + Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`): ```sh @@ -153,6 +157,10 @@ pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley D To build Dash Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) +zstd compresses block and undo files on disk (see: `--with-zstd`, package `libzstd-dev` +on Debian and Ubuntu, `libzstd-devel` on Fedora); a build without it cannot read block files written +with it. + Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`): ```sh @@ -258,7 +266,7 @@ Setup and Build Example: Arch Linux This example lists the steps necessary to setup and build a command line only distribution of the latest changes on Arch Linux: ```sh -pacman --sync --needed autoconf automake base-devel boost cmake gcc git libbacktrace libevent libtool make patch pkgconf python sqlite +pacman --sync --needed autoconf automake base-devel boost cmake gcc git libbacktrace libevent libtool make patch pkgconf python sqlite zstd git clone https://github.com/dashpay/dash.git cd dash/ ./autogen.sh diff --git a/doc/dependencies.md b/doc/dependencies.md index eaf67114fd32..62602df2dca8 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -27,6 +27,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | | libgmp | [link](https://gmplib.org/download/gmp/)[ \* ](#note1) | 6.3.0 | [6.2.0](https://github.com/dashpay/bls-signatures/pull/92) | No | +| [zstd](../depends/packages/zstd.mk) (block file compression) | [link](https://github.com/facebook/zstd/releases) | 1.5.7 | 1.4.0 | No | ### GUI | Dependency | Releases | Version used | Minimum required | Runtime | diff --git a/src/Makefile.am b/src/Makefile.am index ad095c971a9f..6de3c526417e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -532,7 +532,7 @@ obj/build.h: FORCE libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # node # -libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(LEVELDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(LEVELDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) $(ZSTD_CFLAGS) libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_node_a_SOURCES = \ active/context.cpp \ @@ -1038,7 +1038,7 @@ libbitcoin_common_a_SOURCES = \ # # util # -libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(ZSTD_CFLAGS) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ bls/bls_batchverifier.h \ @@ -1126,7 +1126,7 @@ bitcoin_bin_ldadd = \ $(LIBMEMENV) \ $(LIBSECP256K1) -bitcoin_bin_ldadd += $(BACKTRACE_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(GMP_LIBS) +bitcoin_bin_ldadd += $(BACKTRACE_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(GMP_LIBS) $(ZSTD_LIBS) dashd_SOURCES = $(bitcoin_daemon_sources) init/bitcoind.cpp dashd_CPPFLAGS = $(bitcoin_bin_cppflags) @@ -1247,8 +1247,8 @@ if BUILD_BITCOIN_KERNEL_LIB lib_LTLIBRARIES += $(LIBBITCOINKERNEL) libdashkernel_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(PTHREAD_FLAGS) -libdashkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(LIBDASHBLS) $(GMP_LIBS) -libdashkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL -isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include -isystem$(srcdir)/immer $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +libdashkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(LIBDASHBLS) $(GMP_LIBS) $(ZSTD_LIBS) +libdashkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL -isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include -isystem$(srcdir)/immer $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(ZSTD_CFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) # libdashkernel requires default symbol visibility, explicitly specify that # here so that things still work even when user configures with diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index a9949c57b682..a872d082feb3 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -87,6 +87,7 @@ bench_bench_dash_LDADD = \ $(MINIUPNPC_LIBS) \ $(NATPMP_LIBS) \ $(GMP_LIBS) \ + $(ZSTD_LIBS) \ $(BACKTRACE_LIBS) if ENABLE_ZMQ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 9dc7d11ba75f..0b1831ae0090 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -489,7 +489,7 @@ bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBDASHBLS) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ $(BACKTRACE_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(LIBSECP256K1) \ - $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) + $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) $(ZSTD_LIBS) bitcoin_qt_ldflags = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 4690c07a5ea0..fcf7957e1590 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -90,7 +90,7 @@ endif qt_test_test_dash_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBDASHBLS) $(LIBUNIVALUE) $(LIBLEVELDB) \ $(LIBMEMENV) $(BACKTRACE_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) \ $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(LIBSECP256K1) \ - $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) + $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) $(ZSTD_LIBS) qt_test_test_dash_qt_LDFLAGS = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) qt_test_test_dash_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d59f65985291..be5836b8d35c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -60,6 +60,7 @@ FUZZ_SUITE_LD_COMMON = \ $(EVENT_LIBS) \ $(EVENT_PTHREADS_LIBS) \ $(GMP_LIBS) \ + $(ZSTD_LIBS) \ $(BACKTRACE_LIBS) if USE_UPNP @@ -283,7 +284,7 @@ test_test_dash_LDADD += $(LIBBITCOIN_NODE) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON $(LIBDASHBLS) $(LIBLEVELDB) $(LIBMEMENV) $(BACKTRACE_LIBS) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) test_test_dash_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_test_dash_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) +test_test_dash_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) $(ZSTD_LIBS) test_test_dash_LDFLAGS = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -static if ENABLE_ZMQ From e8e914a236f204ca7a97c0d15942ebbadf486b29 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Tue, 20 Oct 2026 05:38:23 +0700 Subject: [PATCH 2/8] feat: add zstd compress/decompress helpers Thin wrappers around the one-shot libzstd API so callers never include themselves. Decompression only accepts frames that record their content size and bounds that size by a caller-supplied maximum, which lets the block store reject malformed records without over-allocating. Builds without zstd keep the same interface: ZstdAvailable() reports the difference, compression returns nothing and decompression always fails, so callers store data uncompressed and report compressed data they cannot read. --- src/Makefile.am | 3 ++ src/test/util_tests.cpp | 34 +++++++++++++++++ src/util/zstd.cpp | 63 +++++++++++++++++++++++++++++++ src/util/zstd.h | 33 ++++++++++++++++ test/util/data/non-backported.txt | 1 + 5 files changed, 134 insertions(+) create mode 100644 src/util/zstd.cpp create mode 100644 src/util/zstd.h diff --git a/src/Makefile.am b/src/Makefile.am index 6de3c526417e..1025d215400d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -484,6 +484,7 @@ BITCOIN_CORE_H = \ util/vecdeque.h \ util/vector.h \ util/wpipe.h \ + util/zstd.h \ validation.h \ validationinterface.h \ versionbits.h \ @@ -1088,6 +1089,7 @@ libbitcoin_util_a_SOURCES = \ util/time.cpp \ util/tokenpipe.cpp \ util/wpipe.cpp \ + util/zstd.cpp \ $(BITCOIN_CORE_H) # @@ -1396,6 +1398,7 @@ libdashkernel_la_SOURCES = \ util/threadnames.cpp \ util/time.cpp \ util/tokenpipe.cpp \ + util/zstd.cpp \ validation.cpp \ validationinterface.cpp \ versionbits.cpp \ diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 20adaea2a732..1ae4a52fb566 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -1980,4 +1981,37 @@ BOOST_AUTO_TEST_CASE(padding_test) BOOST_CHECK_EQUAL(PadString("", 1), " "); } +BOOST_AUTO_TEST_CASE(util_zstd_roundtrip) +{ + std::vector data(10000); + for (size_t i{0}; i < data.size(); ++i) data[i] = std::byte(i % 7); + if (!util::ZstdAvailable()) { + BOOST_CHECK(util::ZstdCompress(data, 3).empty()); + BOOST_CHECK(!util::ZstdDecompress(data, data.size())); + BOOST_CHECK_EQUAL(util::ZstdMaxLevel(), 0); + return; + } + + const auto compressed{util::ZstdCompress(data, 3)}; + BOOST_CHECK(!compressed.empty()); + BOOST_CHECK(compressed.size() < data.size()); + + const auto restored{util::ZstdDecompress(compressed, data.size())}; + BOOST_REQUIRE(restored); + BOOST_CHECK(*restored == data); + + const auto empty{util::ZstdCompress({}, 3)}; + BOOST_CHECK(!empty.empty()); + const auto restored_empty{util::ZstdDecompress(empty, 0)}; + BOOST_REQUIRE(restored_empty); + BOOST_CHECK(restored_empty->empty()); + + // content size above the caller's bound, truncated frame, garbage + BOOST_CHECK(!util::ZstdDecompress(compressed, data.size() - 1)); + BOOST_CHECK(!util::ZstdDecompress(Span{compressed}.first(compressed.size() / 2), data.size())); + BOOST_CHECK(!util::ZstdDecompress(data, data.size())); + + BOOST_CHECK(util::ZstdMaxLevel() >= 19); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/zstd.cpp b/src/util/zstd.cpp new file mode 100644 index 000000000000..29ea452074a7 --- /dev/null +++ b/src/util/zstd.cpp @@ -0,0 +1,63 @@ +// Copyright (c) 2026 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include + +#ifdef ENABLE_ZSTD +#include +#endif + +namespace util { + +bool ZstdAvailable() +{ +#ifdef ENABLE_ZSTD + return true; +#else + return false; +#endif +} + +#ifdef ENABLE_ZSTD +int ZstdMaxLevel() +{ + return ZSTD_maxCLevel(); +} + +std::vector ZstdCompress(Span in, int level) +{ + std::vector out(ZSTD_compressBound(in.size())); + const size_t written{ZSTD_compress(out.data(), out.size(), in.data(), in.size(), level)}; + if (ZSTD_isError(written)) return {}; + out.resize(written); + return out; +} + +std::optional> ZstdDecompress(Span in, size_t max_out) +{ + const unsigned long long content_size{ZSTD_getFrameContentSize(in.data(), in.size())}; + if (content_size == ZSTD_CONTENTSIZE_UNKNOWN || content_size == ZSTD_CONTENTSIZE_ERROR || content_size > max_out) { + return std::nullopt; + } + std::vector out(content_size); + const size_t written{ZSTD_decompress(out.data(), out.size(), in.data(), in.size())}; + if (ZSTD_isError(written) || written != out.size()) return std::nullopt; + return out; +} + +#else + +int ZstdMaxLevel() { return 0; } + +std::vector ZstdCompress(Span, int) { return {}; } + +std::optional> ZstdDecompress(Span, size_t) { return std::nullopt; } + +#endif // ENABLE_ZSTD + +} // namespace util diff --git a/src/util/zstd.h b/src/util/zstd.h new file mode 100644 index 000000000000..559eb7e70c34 --- /dev/null +++ b/src/util/zstd.h @@ -0,0 +1,33 @@ +// Copyright (c) 2026 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_ZSTD_H +#define BITCOIN_UTIL_ZSTD_H + +#include + +#include +#include +#include + +namespace util { + +/** Whether this build links libzstd. Without it nothing is compressed and compressed data cannot be read. */ +bool ZstdAvailable(); + +/** Highest compression level accepted by the linked libzstd, 0 without zstd. */ +int ZstdMaxLevel(); + +/** Compress `in` into a single zstd frame. Returns an empty vector on library error. */ +std::vector ZstdCompress(Span in, int level); + +/** + * Decompress a single zstd frame whose content size is recorded in the frame header. + * Returns nullopt for a malformed frame, an unknown content size, or a content size above max_out. + */ +std::optional> ZstdDecompress(Span in, size_t max_out); + +} // namespace util + +#endif // BITCOIN_UTIL_ZSTD_H diff --git a/test/util/data/non-backported.txt b/test/util/data/non-backported.txt index 776d03b21d54..44dd29436ea9 100644 --- a/test/util/data/non-backported.txt +++ b/test/util/data/non-backported.txt @@ -103,6 +103,7 @@ src/util/helpers.h src/util/ranges_set.* src/util/std23.h src/util/wpipe.* +src/util/zstd.* src/wallet/bip39* src/wallet/coinjoin.* src/wallet/hdchain.* From 6374825dfa3922264a3e4a178627d36a48b1f54f Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Tue, 20 Oct 2026 05:42:03 +0700 Subject: [PATCH 3/8] feat: read block records through a decompressing record reader Every reader of a block record deserialized straight from the file at the indexed position and never looked at the 8-byte record header, and txindex seeked to a transaction offset inside the stored bytes. Neither can work once a stored record may be a zstd frame of the block. Add util::ZstdFileReader, a stream over an AutoFile that passes a plain record through and decompresses a flagged one chunk by chunk while the deserializer pulls bytes, so neither the record nor its contents are ever held in memory as a whole and no size limit is involved; the serializer's own limits stay the only limits, as before. OpenRecordPayload reads the record header, masks the new BLOCK_RECORD_COMPRESSED flag out of the size field and returns such a reader, and RecordHeaderPos maps an indexed payload position back to the header. ReadBlockFromDisk and TxIndex::FindTx keep their shape and read through that reader instead of the file; txindex skips to the transaction offset inside the decompressed stream, so the index format is unchanged and offsets stay relative to the serialized block. The reindex scanner reads the whole record from its buffered file, decompresses it when flagged and deserializes from memory, since it needs the record as a whole for out-of-order blocks. Nothing writes flagged records yet, so behaviour is unchanged. --- src/index/txindex.cpp | 15 ++++---- src/node/blockstorage.cpp | 27 ++++++++++++-- src/node/blockstorage.h | 11 ++++++ src/test/util_tests.cpp | 45 +++++++++++++++++++++++ src/util/zstd.cpp | 75 +++++++++++++++++++++++++++++++++++++++ src/util/zstd.h | 39 ++++++++++++++++++++ src/validation.cpp | 42 +++++++++++++++------- 7 files changed, 233 insertions(+), 21 deletions(-) diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 7cb5069f43e1..c7345bb9f1fa 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -7,9 +7,12 @@ #include #include #include +#include #include using node::OpenBlockFile; +using node::OpenRecordPayload; +using node::RecordHeaderPos; constexpr uint8_t DB_TXINDEX{'t'}; @@ -79,19 +82,17 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe return false; } - AutoFile file{OpenBlockFile(postx, true)}; + AutoFile file{OpenBlockFile(RecordHeaderPos(postx), true)}; if (file.IsNull()) { LogError("%s: OpenBlockFile failed\n", __func__); return false; } CBlockHeader header; try { - file >> header; - if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) { - LogError("%s: fseek(...) failed\n", __func__); - return false; - } - file >> tx; + util::ZstdFileReader block_in{OpenRecordPayload(file)}; + block_in >> header; + block_in.ignore(postx.nTxOffset); + block_in >> tx; } catch (const std::exception& e) { LogError("%s: Deserialize or I/O error - %s\n", __func__, e.what()); return false; diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 822e983573cb..a6a6256b72e7 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -832,12 +833,33 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid return true; } +FlatFilePos RecordHeaderPos(const FlatFilePos& pos) +{ + if (pos.nPos < BLOCK_SERIALIZATION_HEADER_SIZE) return {}; + return {pos.nFile, static_cast(pos.nPos - BLOCK_SERIALIZATION_HEADER_SIZE)}; +} + +/** Size field of the record the file is positioned at; the file is left at the payload */ +static uint32_t ReadRecordSizeField(AutoFile& filein) +{ + uint32_t size_field; + filein.ignore(CMessageHeader::MESSAGE_START_SIZE); + filein >> size_field; + return size_field; +} + +util::ZstdFileReader OpenRecordPayload(AutoFile& filein) +{ + const uint32_t size_field{ReadRecordSizeField(filein)}; + return {filein, size_field & BLOCK_RECORD_SIZE_MASK, (size_field & BLOCK_RECORD_COMPRESSED) != 0}; +} + std::optional ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); // Open history file to read - AutoFile filein{OpenBlockFile(pos, true)}; + AutoFile filein{OpenBlockFile(RecordHeaderPos(pos), true)}; if (filein.IsNull()) { LogError("ReadBlockFromDisk: OpenBlockFile failed for %s\n", pos.ToString()); return std::nullopt; @@ -845,7 +867,8 @@ std::optional ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, // Read block try { - filein >> block; + util::ZstdFileReader block_in{OpenRecordPayload(filein)}; + block_in >> block; } catch (const std::exception& e) { LogError("ReadBlockFromDisk: Deserialize or I/O error - %s at %s\n", e.what(), pos.ToString()); return std::nullopt; diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 66eb4bfc35dd..7232ba61f4b9 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -21,6 +21,7 @@ #include class ArgsManager; +class AutoFile; class BlockValidationState; class CBlock; class CBlockUndo; @@ -32,6 +33,9 @@ struct FlatFilePos; namespace Consensus { struct Params; } +namespace util { +class ZstdFileReader; +} namespace node { static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; @@ -45,6 +49,9 @@ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** Size of header written by WriteBlockToDisk before a serialized CBlock */ static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); +/** Set in the size field of a block record whose payload is one zstd frame of the serialized CBlock */ +static constexpr uint32_t BLOCK_RECORD_COMPRESSED{0x80000000}; +static constexpr uint32_t BLOCK_RECORD_SIZE_MASK{~BLOCK_RECORD_COMPRESSED}; extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; @@ -253,6 +260,10 @@ fs::path GetBlockPosFilename(const FlatFilePos& pos); void UnlinkPrunedFiles(const std::set& setFilesToPrune); /** Functions for disk access for blocks */ +/** Position of the 8-byte header of the record whose payload starts at pos, null when pos leaves no room for one */ +FlatFilePos RecordHeaderPos(const FlatFilePos& pos); +/** Read the header of the record the file is positioned at and return a reader for its payload, decompressing it when flagged */ +util::ZstdFileReader OpenRecordPayload(AutoFile& filein); std::optional ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 1ae4a52fb566..db09d98cb18d 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -5,6 +5,7 @@ #include #include // For Hash() #include // For CKey +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -2012,6 +2014,49 @@ BOOST_AUTO_TEST_CASE(util_zstd_roundtrip) BOOST_CHECK(!util::ZstdDecompress(data, data.size())); BOOST_CHECK(util::ZstdMaxLevel() >= 19); + + // streaming reader over a file holding the frame followed by other data + const fs::path path{m_args.GetDataDirBase() / "zstd_frame.dat"}; + { + AutoFile file{fsbridge::fopen(path, "wb")}; + file.write(compressed); + file << uint32_t{0xdeadbeef}; + } + { + AutoFile file{fsbridge::fopen(path, "rb")}; + util::ZstdFileReader reader{file, compressed.size(), /*compressed=*/true}; + std::vector streamed(data.size()); + for (size_t pos{0}; pos < streamed.size(); pos += 1000) reader.read(Span{streamed}.subspan(pos, 1000)); + BOOST_CHECK(streamed == data); + BOOST_CHECK_THROW(reader.read(Span{streamed}.first(1)), std::ios_base::failure); + uint32_t trailer; + file >> trailer; + BOOST_CHECK_EQUAL(trailer, 0xdeadbeef); + } + { + AutoFile file{fsbridge::fopen(path, "rb")}; + util::ZstdFileReader reader{file, compressed.size(), /*compressed=*/true}; + std::vector tail(data.size() - 1234); + reader.ignore(1234); + reader.read(tail); + BOOST_CHECK(std::ranges::equal(tail, Span{data}.subspan(1234))); + } + { + // the frame cut short by the recorded size + AutoFile file{fsbridge::fopen(path, "rb")}; + util::ZstdFileReader reader{file, compressed.size() / 2, /*compressed=*/true}; + std::vector streamed(data.size()); + BOOST_CHECK_THROW(reader.read(streamed), std::ios_base::failure); + } + { + // a plain record is passed through + AutoFile file{fsbridge::fopen(path, "rb")}; + util::ZstdFileReader reader{file, compressed.size() + sizeof(uint32_t), /*compressed=*/false}; + reader.ignore(compressed.size()); + uint32_t trailer; + reader >> trailer; + BOOST_CHECK_EQUAL(trailer, 0xdeadbeef); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/zstd.cpp b/src/util/zstd.cpp index 29ea452074a7..193b99ac603d 100644 --- a/src/util/zstd.cpp +++ b/src/util/zstd.cpp @@ -8,6 +8,13 @@ #include +#include +#include + +#include +#include +#include + #ifdef ENABLE_ZSTD #include #endif @@ -23,6 +30,26 @@ bool ZstdAvailable() #endif } +void ZstdFileReader::read(Span dst) +{ + if (m_compressed) return ReadCompressed(dst); + m_file.read(dst); +} + +void ZstdFileReader::ignore(size_t n) +{ + if (!m_compressed) { + if (std::fseek(m_file.Get(), static_cast(n), SEEK_CUR) != 0) throw std::ios_base::failure("ZstdFileReader::ignore: fseek failed"); + return; + } + std::byte buf[1024]; + while (n > 0) { + const size_t now{std::min(n, sizeof(buf))}; + read({buf, now}); + n -= now; + } +} + #ifdef ENABLE_ZSTD int ZstdMaxLevel() { @@ -50,6 +77,46 @@ std::optional> ZstdDecompress(Span in, s return out; } +ZstdFileReader::ZstdFileReader(AutoFile& file, size_t size, bool compressed) + : m_file{file}, + m_compressed{compressed}, + m_remaining{size}, + m_stream{compressed ? ZSTD_createDStream() : nullptr, [](ZSTD_DCtx_s* stream) { ZSTD_freeDStream(stream); }} +{ + if (!m_compressed) return; + if (!m_stream) throw std::ios_base::failure("ZstdFileReader: ZSTD_createDStream failed"); + m_in.resize(ZSTD_DStreamInSize()); + m_out.resize(ZSTD_DStreamOutSize()); +} + +void ZstdFileReader::ReadCompressed(Span dst) +{ + while (!dst.empty()) { + if (!m_unread.empty()) { + const size_t n{std::min(dst.size(), m_unread.size())}; + std::copy_n(m_unread.begin(), n, dst.begin()); + m_unread = m_unread.subspan(n); + dst = dst.subspan(n); + continue; + } + if (m_frame_done) throw std::ios_base::failure("ZstdFileReader::read: end of data"); + if (m_pending.empty()) { + if (m_remaining == 0) throw std::ios_base::failure("ZstdFileReader::read: truncated frame"); + const Span chunk{Span{m_in}.first(std::min(m_remaining, m_in.size()))}; + m_file.read(chunk); + m_remaining -= chunk.size(); + m_pending = chunk; + } + ZSTD_inBuffer in{m_pending.data(), m_pending.size(), 0}; + ZSTD_outBuffer out{m_out.data(), m_out.size(), 0}; + const size_t ret{ZSTD_decompressStream(m_stream.get(), &out, &in)}; + if (ZSTD_isError(ret)) throw std::ios_base::failure(strprintf("ZstdFileReader::read: %s", ZSTD_getErrorName(ret))); + m_pending = m_pending.subspan(in.pos); + m_unread = Span{m_out}.first(out.pos); + m_frame_done = ret == 0; + } +} + #else int ZstdMaxLevel() { return 0; } @@ -58,6 +125,14 @@ std::vector ZstdCompress(Span, int) { return {}; } std::optional> ZstdDecompress(Span, size_t) { return std::nullopt; } +ZstdFileReader::ZstdFileReader(AutoFile& file, size_t size, bool compressed) + : m_file{file}, m_compressed{compressed}, m_remaining{size}, m_stream{nullptr, [](ZSTD_DCtx_s*) {}} {} + +void ZstdFileReader::ReadCompressed(Span) +{ + throw std::ios_base::failure("ZstdFileReader::read: compressed record, but this build has no zstd support"); +} + #endif // ENABLE_ZSTD } // namespace util diff --git a/src/util/zstd.h b/src/util/zstd.h index 559eb7e70c34..c117ac2af7a8 100644 --- a/src/util/zstd.h +++ b/src/util/zstd.h @@ -5,12 +5,17 @@ #ifndef BITCOIN_UTIL_ZSTD_H #define BITCOIN_UTIL_ZSTD_H +#include #include #include +#include #include #include +class AutoFile; +struct ZSTD_DCtx_s; + namespace util { /** Whether this build links libzstd. Without it nothing is compressed and compressed data cannot be read. */ @@ -28,6 +33,40 @@ std::vector ZstdCompress(Span in, int level); */ std::optional> ZstdDecompress(Span in, size_t max_out); +/** + * Reads a record of size bytes from the current position of a file. A compressed record is one + * zstd frame, decompressed chunk by chunk while the caller reads, so neither the frame nor its + * contents are ever held in memory as a whole; a plain record is passed through. Throws + * std::ios_base::failure on a truncated or corrupt frame, when reading past its end, and for a + * compressed record in builds without zstd support. + */ +class ZstdFileReader +{ + AutoFile& m_file; + const bool m_compressed; + size_t m_remaining; //!< bytes of the record not read from the file yet + std::unique_ptr m_stream; + std::vector m_in; + std::vector m_out; + Span m_pending; //!< compressed bytes read from the file, not consumed by zstd yet + Span m_unread; //!< decompressed bytes not returned to the caller yet + bool m_frame_done{false}; + + void ReadCompressed(Span dst); + +public: + ZstdFileReader(AutoFile& file, size_t size, bool compressed); + void read(Span dst); + void ignore(size_t n); + + template + ZstdFileReader& operator>>(T&& obj) + { + ::Unserialize(*this, obj); + return *this; + } +}; + } // namespace util #endif // BITCOIN_UTIL_ZSTD_H diff --git a/src/validation.cpp b/src/validation.cpp index 470dc215b5dc..2414088747ea 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -57,6 +57,7 @@ #include #include #include +#include #include #include #include @@ -93,6 +94,8 @@ using kernel::ComputeUTXOStats; using kernel::LoadMempool; using fsbridge::FopenFn; +using node::BLOCK_RECORD_COMPRESSED; +using node::BLOCK_RECORD_SIZE_MASK; using node::BlockManager; using node::BlockMap; using node::CBlockIndexHeightOnlyComparator; @@ -5672,6 +5675,7 @@ void ChainstateManager::LoadExternalBlockFile( nRewind++; // start one byte further next time, in case of failure blkdat.SetLimit(); // remove former limit unsigned int nSize = 0; + bool compressed{false}; try { // locate a header unsigned char buf[CMessageHeader::MESSAGE_START_SIZE]; @@ -5682,8 +5686,11 @@ void ChainstateManager::LoadExternalBlockFile( continue; } // read size - blkdat >> nSize; - if (nSize < 80 || nSize > nMaxBlockSize) + uint32_t size_field{0}; + blkdat >> size_field; + compressed = (size_field & BLOCK_RECORD_COMPRESSED) != 0; + nSize = size_field & BLOCK_RECORD_SIZE_MASK; + if ((!compressed && nSize < 80) || nSize > nMaxBlockSize) continue; } catch (const std::exception&) { // no valid block header found; don't complain @@ -5691,18 +5698,31 @@ void ChainstateManager::LoadExternalBlockFile( break; } try { - // read block header + // read block record const uint64_t nBlockPos{blkdat.GetPos()}; if (dbp) dbp->nPos = nBlockPos; blkdat.SetLimit(nBlockPos + nSize); + std::vector block_data(nSize); + blkdat.read(block_data); + // Position to the marker before the next block, but it's still possible to rewind to the start of the + // current block (without a disk read). + nRewind = nBlockPos + nSize; + if (compressed) { + auto decompressed{util::ZstdDecompress(block_data, nMaxBlockSize)}; + if (!decompressed) { + if (util::ZstdAvailable()) { + LogPrint(BCLog::REINDEX, "%s: Could not decompress block at file offset 0x%x\n", __func__, nBlockPos); + } else { + LogWarning("%s: Skipping zstd-compressed block at file offset 0x%x, this build has no zstd support\n", __func__, nBlockPos); + } + continue; + } + block_data = std::move(*decompressed); + } CBlockHeader header; - blkdat >> header; + SpanReader{SER_DISK, CLIENT_VERSION, MakeUCharSpan(block_data)} >> header; const uint256 hash{header.GetHash()}; - // Skip the rest of this block (this may read from disk into memory); position to the marker before the - // next block, but it's still possible to rewind to the start of the current block (without a disk read). - nRewind = nBlockPos + nSize; - blkdat.SkipTo(nRewind); std::shared_ptr pblock{}; // needs to remain available after the cs_main lock is released to avoid duplicate reads from disk @@ -5721,11 +5741,9 @@ void ChainstateManager::LoadExternalBlockFile( // process in case the block isn't known yet const CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { - // This block can be processed immediately; rewind to its start, read and deserialize it. - blkdat.SetPos(nBlockPos); + // This block can be processed immediately; deserialize it. pblock = std::make_shared(); - blkdat >> *pblock; - nRewind = blkdat.GetPos(); + SpanReader{SER_DISK, CLIENT_VERSION, MakeUCharSpan(block_data)} >> *pblock; BlockValidationState state; if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr, &hash)) { From b18d8baca2cae9eefbf7fdc57484c74af6d51393 Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Tue, 20 Oct 2026 05:43:56 +0700 Subject: [PATCH 4/8] feat: write new block records zstd-compressed Block files are the bulk of a node's disk usage. zstd level 3 saves 19% per block at ~400 MB/s, and higher levels add nothing per block. Compress each newly written block into one zstd frame and flag the record through the high bit of its size field, so the record stays self-describing for reindex and old records are untouched. Blocks that would not shrink are stored as before, which keeps the stored payload within MaxBlockSize(). The level comes from the new -blockcompression option: 0 stores blocks uncompressed, levels above 5 produce a warning, and the default is set in a later commit. During reindex the block file accounting takes the stored record size rather than the serialized size, so pruning usage and the next write offset stay exact for compressed files. A build without zstd stores blocks uncompressed: its default level is 0 and any other value is an init error. It cannot read compressed records, so after the block index loads it checks the tip's record and refuses to start on block files written with compression, naming both ways out: a build with zstd, or -reindex, which skips the compressed records and re-downloads those blocks. --- doc/release-notes-7939.md | 16 +++++++ src/init.cpp | 17 +++++++ src/node/blockstorage.cpp | 50 ++++++++++++++++++-- src/node/blockstorage.h | 5 ++ src/test/blockmanager_tests.cpp | 84 +++++++++++++++++++++++++++++++-- src/test/util_tests.cpp | 1 + src/util/zstd.cpp | 19 ++++++++ src/util/zstd.h | 10 +++- 8 files changed, 192 insertions(+), 10 deletions(-) create mode 100644 doc/release-notes-7939.md diff --git a/doc/release-notes-7939.md b/doc/release-notes-7939.md new file mode 100644 index 000000000000..90ed9460f127 --- /dev/null +++ b/doc/release-notes-7939.md @@ -0,0 +1,16 @@ +Block storage +------------- + +New blocks are written to `blocks/blk*.dat` zstd-compressed, one frame per block, which saves +about a fifth of the space new block files take. Block files written by earlier versions are read +as before and are not rewritten. The new `-blockcompression=` option selects the zstd level +(default 0, stores blocks uncompressed for backwards compatibility). + +zstd is an optional build dependency (`--without-zstd`, `NO_ZSTD=1` for depends). A build without +it stores everything uncompressed, accepts only `-blockcompression=0`, and refuses to start on +block files written with compression until `-reindex` re-downloads those blocks. + +Downgrading to a version without this feature requires a `-reindex` if blockcompression > 0 had ever +been used, since older versions cannot read the compressed records. +`contrib/linearize/linearize-data.py` needs the `zstandard` Python module (or Python 3.14+) to read +block files containing compressed records. diff --git a/src/init.cpp b/src/init.cpp index 008ae13fdb33..b7f249c6875f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -93,6 +93,7 @@ #include #include #include +#include #include #include #include @@ -171,6 +172,7 @@ using node::CalculateCacheSizes; using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; using node::DEFAULT_STOPAFTERBLOCKIMPORT; +using node::MAX_RECOMMENDED_BLOCK_COMPRESSION_LEVEL; using node::g_indexes_ready_to_sync; using node::LoadChainstate; using node::MempoolPath; @@ -599,6 +601,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-alertnotify=", "Execute command when an alert is raised (%s in cmd is replaced by message)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #endif argsman.AddArg("-assumevalid=", strprintf("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s, devnet: %s)", defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex(), devnetChainParams->GetConsensus().defaultAssumeValid.GetHex()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-blockcompression=", strprintf("zstd compression level for newly written block files, 0 to store blocks uncompressed; builds without zstd support accept 0 only (default: %d)", util::DefaultZstdLevel()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-blocksdir=", "Specify directory to hold blocks subdirectory for *.dat files (default: )", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-fastprune", "Use smaller block files and lower minimum prune height for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-tinyblk", "Use smaller block files for testing purposes", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -1345,6 +1348,17 @@ bool AppInitParameterInteraction(const ArgsManager& args) fPruneMode = true; } + const int64_t block_compression{args.GetIntArg("-blockcompression", util::DefaultZstdLevel())}; + if (!util::ZstdAvailable() && block_compression != 0) { + return InitError(_("-blockcompression requires a build with zstd support.")); + } + if (block_compression < 0 || block_compression > util::ZstdMaxLevel()) { + return InitError(strprintf(_("-blockcompression must be between 0 and %d."), util::ZstdMaxLevel())); + } + if (block_compression > MAX_RECOMMENDED_BLOCK_COMPRESSION_LEVEL) { + InitWarning(strprintf(_("-blockcompression=%d: levels above %d cost much more CPU per block for no measurable gain."), block_compression, MAX_RECOMMENDED_BLOCK_COMPRESSION_LEVEL)); + } + nConnectTimeout = args.GetIntArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) { nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; @@ -2104,6 +2118,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return LoadChainstate(chainman, cache_sizes, options, *node.evodb, *node.dmnman, *node.special_tx); }); } + if (status == node::ChainstateLoadStatus::SUCCESS && !util::ZstdAvailable() && !options.reindex && chainman.m_blockman.m_have_compressed_records) { + return InitError(strprintf(_("The block files contain zstd-compressed records, but this build of %s has no zstd support. Use a build with zstd support, or restart with -reindex to re-download the blocks."), PACKAGE_NAME)); + } if (status == node::ChainstateLoadStatus::SUCCESS) { uiInterface.InitMessage(_("Verifying blocks…").translated); if (chainman.m_blockman.m_have_pruned && options.check_blocks > chainman.GetParams().MinBlocksToKeep()) { diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index a6a6256b72e7..650eeb2e15ec 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -22,6 +22,7 @@ #include #include +#include #include namespace node { @@ -348,6 +349,9 @@ bool BlockManager::WriteBlockIndexDB() vBlocks.push_back(*it); m_dirty_blockindex.erase(it++); } + if (m_have_compressed_records && !m_block_tree_db->WriteFlag("compressedrecords", true)) { + return false; + } if (!m_block_tree_db->WriteBatchSync(vFiles, m_last_blockfile, vBlocks)) { return false; } @@ -394,6 +398,7 @@ bool BlockManager::LoadBlockIndexDB() // Check whether we have ever pruned block & undo files m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned); + m_block_tree_db->ReadFlag("compressedrecords", m_have_compressed_records); if (m_have_pruned) { LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); } @@ -769,7 +774,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP return true; } -static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) +static bool WriteBlockToDisk(Span payload, uint32_t size_field, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append AutoFile fileout{OpenBlockFile(pos)}; @@ -779,8 +784,7 @@ static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessa } // Write index header - unsigned int nSize = GetSerializeSize(block, CLIENT_VERSION); - fileout << messageStart << nSize; + fileout << messageStart << size_field; // Write block long fileOutPos = ftell(fileout.Get()); @@ -789,11 +793,28 @@ static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessa return false; } pos.nPos = (unsigned int)fileOutPos; - fileout << block; + fileout.write(payload); return true; } +/** Serialize the block for storage, as a zstd frame when that is smaller. Returns the payload and its size field. */ +static std::pair BuildBlockRecordPayload(const CBlock& block) +{ + DataStream ss; + ss << block; + const int level{util::ZstdLevel()}; + if (level > 0) { + const std::vector compressed{util::ZstdCompress(MakeByteSpan(ss), level)}; + if (!compressed.empty() && compressed.size() < ss.size()) { + const uint32_t size_field{static_cast(compressed.size()) | BLOCK_RECORD_COMPRESSED}; + return {DataStream{compressed}, size_field}; + } + } + const uint32_t size_field{static_cast(ss.size())}; + return {std::move(ss), size_field}; +} + bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block) { AssertLockHeld(::cs_main); @@ -900,14 +921,32 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus return true; } +/** Size field of the record already stored at pos */ +static std::optional StoredBlockSizeField(const FlatFilePos& pos) +{ + AutoFile filein{OpenBlockFile(RecordHeaderPos(pos), true)}; + if (filein.IsNull()) return std::nullopt; + try { + return ReadRecordSizeField(filein); + } catch (const std::exception&) { + return std::nullopt; + } +} + FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; const auto position_known {dbp != nullptr}; + DataStream payload; + uint32_t size_field{0}; if (position_known) { blockPos = *dbp; + // the stored record may be smaller than the serialized block + if (const auto stored{StoredBlockSizeField(blockPos)}) nBlockSize = *stored & BLOCK_RECORD_SIZE_MASK; } else { + std::tie(payload, size_field) = BuildBlockRecordPayload(block); + nBlockSize = payload.size(); // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE). // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. @@ -918,7 +957,8 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, cons return FlatFilePos(); } if (!position_known) { - if (!WriteBlockToDisk(block, blockPos, GetParams().MessageStart())) { + if (size_field & BLOCK_RECORD_COMPRESSED) m_have_compressed_records = true; + if (!WriteBlockToDisk(MakeByteSpan(payload), size_field, blockPos, GetParams().MessageStart())) { AbortNode("Failed to write block"); return FlatFilePos(); } diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 7232ba61f4b9..97b40948ae76 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -39,6 +39,8 @@ class ZstdFileReader; namespace node { static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; +/** Higher levels cost several times more CPU per block for no measurable gain */ +static constexpr int MAX_RECOMMENDED_BLOCK_COMPRESSION_LEVEL{5}; /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB @@ -240,6 +242,9 @@ class BlockManager /** True if any block files have ever been pruned. */ bool m_have_pruned = false; + /** True if any zstd-compressed block or undo record has ever been written, which a build without zstd cannot read. */ + bool m_have_compressed_records = false; + //! Check whether the block associated with this index entry is pruned or not. bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index f70cbc2c2ba3..1bacd2ac9bff 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -3,18 +3,37 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include +#include #include #include +#include +#include