feat: optional zstd compression of block and undo files

configure.ac

@@ -149,6 +149,12 @@ AC_ARG_WITH([bdb],
   [use_bdb=$withval],
   [use_bdb=auto])
 
+AC_ARG_WITH([zstd],
+  [AS_HELP_STRING([--without-zstd],
+  [disable zstd compression of block files and masternode lists (default is enabled if libzstd is found)])],
+  [use_zstd=$withval],
+  [use_zstd=auto])
+
 AC_ARG_ENABLE([usdt],
   [AS_HELP_STRING([--enable-usdt],
   [enable tracepoints for Userspace, Statically Defined Tracing (default is yes if sys/sdt.h is found)])],
@@ -1694,7 +1700,23 @@ AC_CHECK_LIB([gmp],  [__gmpz_init],GMP_LIBS=-lgmp, AC_MSG_ERROR(libgmp missing))
 CPPFLAGS="$TEMP_CPPFLAGS"
 LDFLAGS="$TEMP_LDFLAGS"
 
+dnl zstd check
+if test "$use_zstd" != "no"; then
+  PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.4.0],
+    [AC_DEFINE([ENABLE_ZSTD], [1], [Define this symbol to enable zstd compression])
+     use_zstd=yes],
+    [if test "$use_zstd" = "yes"; then
+       AC_MSG_ERROR([libzstd version 1.4.0 or greater not found.])
+     fi
+     AC_MSG_WARN([libzstd version 1.4.0 or greater not found, disabling zstd compression])
+     use_zstd=no])
+fi
+if test "$use_zstd" = "yes" && test "$suppress_external_warnings" != "no"; then
+  ZSTD_CFLAGS=SUPPRESS_WARNINGS($ZSTD_CFLAGS)
+fi
+
 AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"])
+AM_CONDITIONAL([ENABLE_ZSTD], [test "$use_zstd" = "yes"])
 
 dnl libmultiprocess library check
 
@@ -2103,6 +2125,7 @@ if test $bitcoin_enable_qt != "no"; then
     echo "    with qr       = $use_qr"
 fi
 echo "  with zmq        = $use_zmq"
+echo "  with zstd       = $use_zstd"
 if test $enable_fuzz = "no"; then
     echo "  with test       = $use_tests"
 else

contrib/linearize/README.md

@@ -5,6 +5,10 @@ Construct a linear, no-fork, best version of the Dash blockchain.
 
 https://github.com/dashpay/dash_hash
 
+Block files written by recent Dash Core versions hold zstd-compressed records; reading
+them with `linearize-data.py` needs the `zstandard` module (`pip install zstandard`)
+or Python 3.14+.
+
 ## Step 1: Download hash list
 
     $ ./linearize-hashes.py linearize.cfg > hashlist.txt

contrib/linearize/linearize-data.py

@@ -18,6 +18,19 @@
 import glob
 from collections import namedtuple
 
+try:
+    from compression import zstd as _zstd  # Python >= 3.14
+    zstd_decompress = _zstd.decompress
+except ImportError:
+    try:
+        import zstandard
+        zstd_decompress = zstandard.ZstdDecompressor().decompress
+    except ImportError:
+        zstd_decompress = None
+
+# High bit of the record size field: the payload is one zstd frame of the block
+BLOCK_RECORD_COMPRESSED = 0x80000000
+
 settings = {}
 
 def calc_hash_str(blk_hdr):
@@ -77,7 +90,7 @@ def getFirstBlockFileId(block_dir_path):
     return blkId
 
 # Block header and extent on disk
-BlockExtent = namedtuple('BlockExtent', ['fn', 'offset', 'inhdr', 'blkhdr', 'size'])
+BlockExtent = namedtuple('BlockExtent', ['fn', 'offset', 'inhdr', 'blkhdr', 'size', 'compressed'])
 
 class BlockDataCopier:
     def __init__(self, settings, blkindex, blkmap):
@@ -165,6 +178,8 @@ def fetchBlock(self, extent):
         '''Fetch block contents from disk given extents'''
         with open(self.inFileName(extent.fn), "rb") as f:
             f.seek(extent.offset)
+            if extent.compressed:
+                return zstd_decompress(f.read(extent.size))[80:]
             return f.read(extent.size)
 
     def copyOneBlock(self):
@@ -206,25 +221,42 @@ def run(self):
                 continue
             inLenLE = inhdr[4:]
             su = struct.unpack("<I", inLenLE)
-            inLen = su[0] - 80 # length without header
-            blk_hdr = self.inF.read(80)
-            inExtent = BlockExtent(self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen)
+            compressed = bool(su[0] & BLOCK_RECORD_COMPRESSED)
+            if compressed:
+                if zstd_decompress is None:
+                    print("Compressed block records need the 'zstandard' module or Python >= 3.14")
+                    sys.exit(1)
+                payload_len = su[0] & ~BLOCK_RECORD_COMPRESSED
+                payload_offset = self.inF.tell()
+                rawblock = zstd_decompress(self.inF.read(payload_len))
+                blk_hdr = rawblock[:80]
+                rawblock = rawblock[80:]
+                inLen = len(rawblock)
+                # bootstrap files always hold uncompressed records
+                inhdr = inMagic + struct.pack("<I", len(blk_hdr) + inLen)
+                inExtent = BlockExtent(self.inFn, payload_offset, inhdr, blk_hdr, payload_len, compressed)
+            else:
+                inLen = su[0] - 80 # length without header
+                blk_hdr = self.inF.read(80)
+                inExtent = BlockExtent(self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen, compressed)
 
             self.hash_str = calc_hash_str(blk_hdr)
             if not self.hash_str in blkmap:
                 # Because blocks can be written to files out-of-order as of 0.10, the script
                 # may encounter blocks it doesn't know about. Treat as debug output.
                 if settings['debug_output'] == 'true':
                     print("Skipping unknown block " + self.hash_str)
-                self.inF.seek(inLen, os.SEEK_CUR)
+                if not compressed:
+                    self.inF.seek(inLen, os.SEEK_CUR)
                 continue
 
             blkHeight = self.blkmap[self.hash_str]
             self.blkCountIn += 1
 
             if self.blkCountOut == blkHeight:
                 # If in-order block, just copy
-                rawblock = self.inF.read(inLen)
+                if not compressed:
+                    rawblock = self.inF.read(inLen)
                 self.writeBlock(inhdr, blk_hdr, rawblock)
 
                 # See if we can catch up to prior out-of-order blocks
@@ -237,9 +269,11 @@ def run(self):
                     # If there is space in the cache, read the data
                     # Reading the data in file sequence instead of seeking and fetching it later is preferred,
                     # but we don't want to fill up memory
-                    self.outOfOrderData[blkHeight] = self.inF.read(inLen)
+                    if not compressed:
+                        rawblock = self.inF.read(inLen)
+                    self.outOfOrderData[blkHeight] = rawblock
                     self.outOfOrderSize += inLen
-                else: # If no space in cache, seek forward
+                elif not compressed: # If no space in cache, seek forward
                     self.inF.seek(inLen, os.SEEK_CUR)
 
         print("Done (%i blocks written)" % (self.blkCountOut))

depends/Makefile

@@ -40,6 +40,7 @@ NO_BDB ?=
 NO_SQLITE ?=
 NO_WALLET ?=
 NO_ZMQ ?=
+NO_ZSTD ?=
 NO_UPNP ?=
 NO_USDT ?=
 NO_NATPMP ?=
@@ -170,10 +171,11 @@ upnp_packages_$(NO_UPNP) = $(upnp_packages)
 natpmp_packages_$(NO_NATPMP) = $(natpmp_packages)
 
 zmq_packages_$(NO_ZMQ) = $(zmq_packages)
+zstd_packages_$(NO_ZSTD) = $(zstd_packages)
 multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages)
 usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages)
 
-packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_)
+packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) $(zstd_packages_)
 native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
 
 ifneq ($(zmq_packages_),)
@@ -246,6 +248,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@no_qt@|$(NO_QT)|' \
             -e 's|@no_qr@|$(NO_QR)|' \
             -e 's|@no_zmq@|$(NO_ZMQ)|' \
+            -e 's|@no_zstd@|$(NO_ZSTD)|' \
             -e 's|@no_wallet@|$(NO_WALLET)|' \
             -e 's|@no_bdb@|$(NO_BDB)|' \
             -e 's|@no_sqlite@|$(NO_SQLITE)|' \

depends/README.md

@@ -82,6 +82,7 @@ The following can be set when running make: `make FOO=bar`
 - `NO_QT`: Don't download/build/cache Qt and its dependencies
 - `NO_QR`: Don't download/build/cache packages needed for enabling qrencode
 - `NO_ZMQ`: Don't download/build/cache packages needed for enabling ZeroMQ
+- `NO_ZSTD`: Don't download/build/cache zstd (block files are then stored uncompressed)
 - `NO_WALLET`: Don't download/build/cache libs needed to enable the wallet
 - `NO_BDB`: Don't download/build/cache BerkeleyDB
 - `NO_SQLITE`: Don't download/build/cache SQLite

depends/config.site.in

@@ -70,6 +70,10 @@ if test -z "$enable_zmq" && test -n "@no_zmq@"; then
   enable_zmq=no
 fi
 
+if test -z "$with_zstd" && test -n "@no_zstd@"; then
+  with_zstd=no
+fi
+
 if test -z "$enable_usdt" && test -n "@no_usdt@"; then
   enable_usdt=no
 fi

depends/packages/packages.mk

@@ -18,6 +18,8 @@ sqlite_packages=sqlite
 
 zmq_packages=zeromq
 
+zstd_packages=zstd
+
 upnp_packages=miniupnpc
 natpmp_packages=libnatpmp
 

depends/packages/zstd.mk

@@ -0,0 +1,19 @@
+package=zstd
+$(package)_version=1.5.7
+$(package)_download_path=https://github.com/facebook/zstd/releases/download/v$($(package)_version)
+$(package)_file_name=$(package)-$($(package)_version).tar.gz
+$(package)_sha256_hash=eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3
+
+define $(package)_set_vars
+  $(package)_build_opts=CC="$($(package)_cc)" AR="$($(package)_ar)"
+  $(package)_build_opts+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" LDFLAGS="$($(package)_ldflags)"
+  $(package)_build_opts+=ZSTD_LEGACY_SUPPORT=0 ZSTD_LIB_DEPRECATED=0 ZSTD_LIB_DICTBUILDER=0
+endef
+
+define $(package)_build_cmds
+  $(MAKE) -C lib libzstd.a $($(package)_build_opts)
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) -C lib install-static install-includes install-pc PREFIX=$(host_prefix) DESTDIR=$($(package)_staging_dir) $($(package)_build_opts)
+endef

doc/build-freebsd.md

@@ -10,7 +10,7 @@ This guide describes how to build dashd, command-line utilities, and GUI on Free
 Run the following as root to install the base dependencies for building.
 
 ```bash
-pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf libbacktrace
+pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf libbacktrace zstd
 
 ```
 

doc/build-netbsd.md

@@ -14,7 +14,7 @@ Install the required dependencies the usual way you [install software on NetBSD]
 The example commands below use `pkgin`.
 
 ```bash
-pkgin install autoconf automake libtool pkg-config git gmake boost libevent gmp
+pkgin install autoconf automake libtool pkg-config git gmake boost libevent gmp zstd
 
 ```
 

doc/build-openbsd.md

@@ -10,7 +10,7 @@ This guide describes how to build dashd, command-line utilities, and GUI on Open
 Run the following as root to install the base dependencies for building.
 
 ```bash
-pkg_add bash git gmake gmp libevent libtool boost
+pkg_add bash git gmake gmp libevent libtool boost zstd
 # Select the newest version of the following packages:
 pkg_add autoconf automake python
 

doc/build-osx.md

@@ -47,7 +47,7 @@ To install, run the following from your terminal:
 See [dependencies.md](dependencies.md) for a complete overview.
 
 ``` bash
-brew install automake libtool boost gmp pkg-config libevent
+brew install automake libtool boost gmp pkg-config libevent zstd
 ```
 
 ### 4. Clone Dash repository

doc/build-unix.md

@@ -64,6 +64,10 @@ executables, which are based on BerkeleyDB 4.8. If you do not care about wallet
 
 To build Dash Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode)
 
+zstd compresses block files and masternode lists on disk (see: `--with-zstd`, package `libzstd-dev`
+on Debian and Ubuntu, `libzstd-devel` on Fedora); a build without it cannot read block files written
+with it.
+
 Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`):
 
 ```sh
@@ -153,6 +157,10 @@ pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley D
 
 To build Dash Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode)
 
+zstd compresses block files and masternode lists on disk (see: `--with-zstd`, package `libzstd-dev`
+on Debian and Ubuntu, `libzstd-devel` on Fedora); a build without it cannot read block files written
+with it.
+
 Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`):
 
 ```sh
@@ -258,7 +266,7 @@ Setup and Build Example: Arch Linux
 This example lists the steps necessary to setup and build a command line only distribution of the latest changes on Arch Linux:
 
 ```sh
-pacman --sync --needed autoconf automake base-devel boost cmake gcc git libbacktrace libevent libtool make patch pkgconf python sqlite
+pacman --sync --needed autoconf automake base-devel boost cmake gcc git libbacktrace libevent libtool make patch pkgconf python sqlite zstd
 git clone https://github.com/dashpay/dash.git
 cd dash/
 ./autogen.sh

doc/dependencies.md

@@ -27,6 +27,7 @@ You can find installation instructions in the `build-*.md` file for your platfor
 | Dependency | Releases | Version used | Minimum required | Runtime |
 | --- | --- | --- | --- | --- |
 | libgmp | [link](https://gmplib.org/download/gmp/)<sup>[ \* ](#note1)</sup> | 6.3.0 | [6.2.0](https://github.com/dashpay/bls-signatures/pull/92) | No |
+| [zstd](../depends/packages/zstd.mk) (block file compression) | [link](https://github.com/facebook/zstd/releases) | 1.5.7 | 1.4.0 | No |
 
 ### GUI
 | Dependency | Releases | Version used | Minimum required | Runtime |

doc/release-notes-7939.md

@@ -0,0 +1,24 @@
+Block storage
+-------------
+
+Block files (`blocks/blk*.dat`) and undo files (`blocks/rev*.dat`) can now be written
+zstd-compressed, one frame per record, which saves about a fifth of the space of new block files
+and about a quarter of new undo files. The new `-blockcompression=<level>` option selects the zstd
+level. It defaults to 0 in this release, which stores records exactly as before, so nothing changes
+on disk unless the option is set. Level 3 is the recommended setting when opting in; levels above 5
+cost several times more CPU for no measurable gain and produce a startup warning. A future release
+is planned to enable compression by default.
+
+Files written by earlier versions are read as before and are never rewritten, and this version
+reads compressed records whether or not it writes them. That keeps downgrades simple: a node that
+never set `-blockcompression` can be downgraded to any earlier version without a reindex, and once
+a later release enables compression by default, a downgrade to this version needs no reindex
+either. Only a downgrade to a version before this one after compression has been used requires
+`-reindex`, since older versions cannot read the compressed records.
+
+zstd is an optional build dependency (`--without-zstd`, `NO_ZSTD=1` for depends). A build without
+it stores everything uncompressed, accepts only `-blockcompression=0`, and refuses to start on
+block files written with compression until `-reindex` re-downloads those blocks.
+
+`contrib/linearize/linearize-data.py` reads compressed records with the `zstandard` Python module
+or Python 3.14 and later, and always writes uncompressed records to the bootstrap file.

src/Makefile.am

@@ -484,6 +484,7 @@ BITCOIN_CORE_H = \
   util/vecdeque.h \
   util/vector.h \
   util/wpipe.h \
+  util/zstd.h \
   validation.h \
   validationinterface.h \
   versionbits.h \
@@ -532,7 +533,7 @@ obj/build.h: FORCE
 libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h
 
 # node #
-libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(LEVELDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS)
+libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(LEVELDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) $(ZSTD_CFLAGS)
 libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 libbitcoin_node_a_SOURCES = \
   active/context.cpp \
@@ -1038,7 +1039,7 @@ libbitcoin_common_a_SOURCES = \
 #
 
 # util #
-libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
+libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(ZSTD_CFLAGS)
 libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 libbitcoin_util_a_SOURCES = \
   bls/bls_batchverifier.h \
@@ -1088,6 +1089,7 @@ libbitcoin_util_a_SOURCES = \
   util/time.cpp \
   util/tokenpipe.cpp \
   util/wpipe.cpp \
+  util/zstd.cpp \
   $(BITCOIN_CORE_H)
 #
 
@@ -1126,7 +1128,7 @@ bitcoin_bin_ldadd = \
   $(LIBMEMENV) \
   $(LIBSECP256K1)
 
-bitcoin_bin_ldadd += $(BACKTRACE_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(GMP_LIBS)
+bitcoin_bin_ldadd += $(BACKTRACE_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(GMP_LIBS) $(ZSTD_LIBS)
 
 dashd_SOURCES = $(bitcoin_daemon_sources) init/bitcoind.cpp
 dashd_CPPFLAGS = $(bitcoin_bin_cppflags)
@@ -1247,8 +1249,8 @@ if BUILD_BITCOIN_KERNEL_LIB
 lib_LTLIBRARIES += $(LIBBITCOINKERNEL)
 
 libdashkernel_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(PTHREAD_FLAGS)
-libdashkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(LIBDASHBLS) $(GMP_LIBS)
-libdashkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL -isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include -isystem$(srcdir)/immer $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT)
+libdashkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(LIBDASHBLS) $(GMP_LIBS) $(ZSTD_LIBS)
+libdashkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL -isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include -isystem$(srcdir)/immer $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(ZSTD_CFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT)
 
 # libdashkernel requires default symbol visibility, explicitly specify that
 # here so that things still work even when user configures with
@@ -1396,6 +1398,7 @@ libdashkernel_la_SOURCES = \
   util/threadnames.cpp \
   util/time.cpp \
   util/tokenpipe.cpp \
+  util/zstd.cpp \
   validation.cpp \
   validationinterface.cpp \
   versionbits.cpp \