From 5b39683ef0ceaaaf33d6db96ad2ccaaf989d56d2 Mon Sep 17 00:00:00 2001 From: Honggyu Kim Date: Thu, 23 Jul 2026 21:28:15 +0900 Subject: [PATCH 1/8] Add dtoctl wrapper to drive DTO without manual LD_PRELOAD setup Using the DTO library today requires the user to understand two orthogonal mechanisms before a single program can be offloaded to DSA. The library has to be injected into the target through LD_PRELOAD, and its behavior is steered entirely through a set of roughly twenty DTO_* environment variables that must be discovered by reading the manual and then exported by hand. This is easy to get wrong: the LD_PRELOAD path and the variable names are stringly typed, an exported variable silently leaks into every later command in the same shell, and there is no validation or discoverability at the point of use. Provide a small wrapper, dtoctl, that turns this into a single self-describing command line. It follows the same model as hmctl from the hmsdk project: parse options, translate them into the environment the library expects, and then exec the target program so no state lingers in the caller's shell. On top of that model dtoctl also takes care of the injection itself by prepending the resolved libdto.so to LD_PRELOAD, which is possible because DTO works by transparently interposing on the standard memory APIs. Every DTO_* variable is exposed as an option so the mapping stays obvious and greppable. Only options that are actually given are written to the child environment; anything left unset keeps whatever the caller already exported or the library's own default, and an explicit option wins over an inherited variable. The library path is resolved from -l/--library, then $DTO_LIBRARY, then a compiled-in default, and execvp() is used so a bare command name on PATH works as well as an explicit path. Build and install dtoctl with both Make and CMake. Set its compiled-in library path to match the installation directory, since CMake's default /usr/local/lib differs from the Makefile's /usr/lib64. Allow that path to be overridden with DTOCTL_LIBPATH. With this in place the previous incantation $ export LD_PRELOAD=/usr/lib64/libdto.so $ export DTO_WAIT_METHOD=busypoll $ export DTO_CPU_SIZE_FRACTION=0.33 $ ./prog collapses to $ dtoctl -w busypoll -c 0.33 ./prog Signed-off-by: Honggyu Kim --- .gitignore | 1 + CMakeLists.txt | 9 ++ Makefile | 11 +- dtoctl.c | 366 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 385 insertions(+), 2 deletions(-) create mode 100644 dtoctl.c diff --git a/.gitignore b/.gitignore index ebb181e..977a993 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ dto-test* libdto.so* build*/ tests/baselines*/ +dtoctl diff --git a/CMakeLists.txt b/CMakeLists.txt index f63146a..4567c1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,15 @@ if (HAS_WAITPKG) target_compile_options(dto PRIVATE -mwaitpkg -march=native) endif() +# Build dtoctl with a default preload path matching the library installation. +get_filename_component(DTOCTL_LIBDIR "${CMAKE_INSTALL_LIBDIR}" ABSOLUTE + BASE_DIR "${CMAKE_INSTALL_PREFIX}") +set(DTOCTL_LIBPATH "${DTOCTL_LIBDIR}/libdto.so" CACHE FILEPATH + "Path to libdto.so that dtoctl prepends to LD_PRELOAD") +add_executable(dtoctl dtoctl.c) +target_compile_definitions(dtoctl PRIVATE LIBDTO_PATH="${DTOCTL_LIBPATH}") +install(TARGETS dtoctl RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + # Build dto-test and dto-test-wodto add_executable(dto-test dto-test.c) target_link_libraries(dto-test PRIVATE DTO::dto pthread) diff --git a/Makefile b/Makefile index 06cc2e1..c200ea1 100644 --- a/Makefile +++ b/Makefile @@ -2,20 +2,27 @@ # # SPDX-License-Identifier: MIT -all: libdto dto-test-wodto +all: libdto dtoctl dto-test-wodto DML_LIB_CXX=-D_GNU_SOURCE +# Path to libdto.so that dtoctl prepends to LD_PRELOAD (matches "make install"). +DTOCTL_LIBPATH ?= /usr/lib64/libdto.so + libdto: dto.c gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_STATS_SUPPORT -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg libdto_nostats: dto.c gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg +dtoctl: dtoctl.c + gcc -O2 -Wall dtoctl.c -DLIBDTO_PATH=\"$(DTOCTL_LIBPATH)\" -o dtoctl + install: cp libdto.so.1.0 /usr/lib64/ ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so.1 ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so + cp dtoctl /usr/bin/ install-local: ln -sf ./libdto.so.1.0 ./libdto.so.1 @@ -28,4 +35,4 @@ dto-test-wodto: dto-test.c gcc -g dto-test.c $(DML_LIB_CXX) -o dto-test-wodto -lpthread clean: - rm -rf *.o *.so dto-test + rm -rf *.o *.so dto-test dtoctl diff --git a/dtoctl.c b/dtoctl.c new file mode 100644 index 0000000..707cd3a --- /dev/null +++ b/dtoctl.c @@ -0,0 +1,366 @@ +/* Copyright (C) 2026 SK hynix, Inc. */ +/* SPDX-License-Identifier: MIT */ + +/* + * dtoctl -- a CLI wrapper that runs a program with the DSA Transparent Offload + * (DTO) library preloaded. It converts command line options into the DTO_* + * environment variables and prepends libdto.so to LD_PRELOAD so that users do + * not need to export the variables or set up LD_PRELOAD manually. + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* for asprintf() */ +#endif + +#include +#include +#include +#include +#include +#include + +#ifndef LIBDTO_PATH +#define LIBDTO_PATH "/usr/lib64/libdto.so" +#endif + +/* argp option keys for the long-only options (no short equivalents) */ +enum { + OPT_UMWAIT_DELAY = 0x100, + OPT_OVERLAPPING_MEMMOVE, + OPT_LOG_FILE, + OPT_LOG_LEVEL, + OPT_STATS_FILE, + OPT_NO_MEMCPY, + OPT_NO_MEMMOVE, + OPT_NO_MEMSET, + OPT_NO_MEMCMP, + OPT_NO_CACHE_CONTROL, + OPT_NO_AUTO_ADJUST, + OPT_STDC_ONLY, + OPT_STATS, +}; + +struct opts { + int idx; + char *exename; + + /* library / preload */ + const char *library; + + /* value options (NULL means "not specified") */ + const char *wait_method; + const char *min_bytes; + const char *cpu_fraction; + const char *numa_aware; + const char *wq_list; + const char *umwait_delay; + const char *overlapping_memmove; /* "0" or "1" after validation */ + const char *log_file; + const char *log_level; + const char *stats_file; + + /* flag options (false means "not specified") */ + bool no_memcpy; + bool no_memmove; + bool no_memset; + bool no_memcmp; + bool no_cache_control; + bool no_auto_adjust; + bool stdc_only; + bool stats; +}; + +struct opts opts; + +/* (a part of) output in --help option (generated by argp runtime) */ +const char *argp_program_bug_address = "https://github.com/intel/DTO/issues"; + +/* Option groups are separated using entries with only a .doc field. */ +static struct argp_option dtoctl_options[] = { + {.doc = "Library:", .group = 1}, + {.name = "library", + .key = 'l', + .arg = "PATH", + .doc = "Path to libdto.so to prepend to LD_PRELOAD. Overrides $DTO_LIBRARY " + "and the compiled-in default (" LIBDTO_PATH ")"}, + + {.doc = "Offload tuning:", .group = 2}, + {.name = "wait-method", + .key = 'w', + .arg = "METHOD", + .doc = "How to wait for DSA completion: yield, busypoll, umwait, or tpause " + "(sets DTO_WAIT_METHOD; default: busypoll)"}, + {.name = "min-bytes", + .key = 'b', + .arg = "BYTES", + .doc = "Minimum operation size offloaded to DSA (sets DTO_MIN_BYTES; " + "default: 65536)"}, + {.name = "cpu-fraction", + .key = 'c', + .arg = "FRACTION", + .doc = "Fraction (0.0 <= f < 1.0) of each operation done on CPU in parallel " + "to DSA (sets DTO_CPU_SIZE_FRACTION; default: 0.0)"}, + {.name = "numa-aware", + .key = 'n', + .arg = "MODE", + .doc = "NUMA awareness: 0 disable, 1 buffer-centric, 2 cpu-centric " + "(sets DTO_IS_NUMA_AWARE; default: 0)"}, + {.name = "wq-list", + .key = 'q', + .arg = "LIST", + .doc = "Semicolon-separated list of DSA WQs to use, e.g. \"wq0.0;wq2.0\" " + "(sets DTO_WQ_LIST; default: auto-discover all WQs)"}, + {.name = "umwait-delay", + .key = OPT_UMWAIT_DELAY, + .arg = "CYCLES", + .doc = "Delay in cycles for the umwait instruction (sets DTO_UMWAIT_DELAY; " + "default: 100000)"}, + {.name = "overlapping-memmove", + .key = OPT_OVERLAPPING_MEMMOVE, + .arg = "WHERE", + .doc = "Where to run memmove with overlapping buffers: cpu or dsa " + "(sets DTO_OVERLAPPING_MEMMOVE_ACTION; default: cpu)"}, + + {.doc = "Disable DSA offload for specific operations (DSA is used by default):", + .group = 3}, + {.name = "no-memcpy", + .key = OPT_NO_MEMCPY, + .doc = "Use system memcpy instead of DSA (sets DTO_DSA_MEMCPY=0)"}, + {.name = "no-memmove", + .key = OPT_NO_MEMMOVE, + .doc = "Use system memmove instead of DSA (sets DTO_DSA_MEMMOVE=0)"}, + {.name = "no-memset", + .key = OPT_NO_MEMSET, + .doc = "Use system memset instead of DSA (sets DTO_DSA_MEMSET=0)"}, + {.name = "no-memcmp", + .key = OPT_NO_MEMCMP, + .doc = "Use system memcmp instead of DSA (sets DTO_DSA_MEMCMP=0)"}, + + {.doc = "Other toggles:", .group = 4}, + {.name = "no-cache-control", + .key = OPT_NO_CACHE_CONTROL, + .doc = "Clear the DSA cache control flag to avoid cache pollution " + "(sets DTO_DSA_CC=0; default: cache control on)"}, + {.name = "no-auto-adjust", + .key = OPT_NO_AUTO_ADJUST, + .doc = "Disable auto tuning of cpu-fraction and min-bytes " + "(sets DTO_AUTO_ADJUST_KNOBS=0; default: auto tuning on)"}, + {.name = "stdc-only", + .key = OPT_STDC_ONLY, + .doc = "Use only the standard C memory functions, no DSA offload " + "(sets DTO_USESTDC_CALLS=1; default: DSA offload enabled)"}, + + {.doc = "Statistics and logging:", .group = 5}, + {.name = "stats", + .key = OPT_STATS, + .doc = "Enable stats collection; for debugging/profiling only " + "(sets DTO_COLLECT_STATS=1; default: off)"}, + {.name = "stats-file", + .key = OPT_STATS_FILE, + .arg = "PATH", + .doc = "Write the stats histogram to PATH instead of standard output " + "(sets DTO_STATS_FILE; default: standard output)"}, + {.name = "log-file", + .key = OPT_LOG_FILE, + .arg = "PATH", + .doc = "Redirect DTO output to PATH (suffixed by pid) (sets DTO_LOG_FILE; " + "default: standard output)"}, + {.name = "log-level", + .key = OPT_LOG_LEVEL, + .arg = "LEVEL", + .doc = "Verbosity of DTO logging: 0, 1, or 2 (sets DTO_LOG_LEVEL; " + "default: 0)"}, + + {NULL}, +}; + +static error_t parse_option(int key, char *arg, struct argp_state *state) +{ + struct opts *opts = state->input; + + switch (key) { + case 'l': + opts->library = arg; + break; + case 'w': + opts->wait_method = arg; + break; + case 'b': + opts->min_bytes = arg; + break; + case 'c': + opts->cpu_fraction = arg; + break; + case 'n': + opts->numa_aware = arg; + break; + case 'q': + opts->wq_list = arg; + break; + case OPT_UMWAIT_DELAY: + opts->umwait_delay = arg; + break; + case OPT_OVERLAPPING_MEMMOVE: + if (!strcmp(arg, "cpu")) + opts->overlapping_memmove = "0"; + else if (!strcmp(arg, "dsa")) + opts->overlapping_memmove = "1"; + else { + fprintf(stderr, + "Error: --overlapping-memmove must be 'cpu' or 'dsa'.\n\n"); + argp_usage(state); + } + break; + case OPT_NO_MEMCPY: + opts->no_memcpy = true; + break; + case OPT_NO_MEMMOVE: + opts->no_memmove = true; + break; + case OPT_NO_MEMSET: + opts->no_memset = true; + break; + case OPT_NO_MEMCMP: + opts->no_memcmp = true; + break; + case OPT_NO_CACHE_CONTROL: + opts->no_cache_control = true; + break; + case OPT_NO_AUTO_ADJUST: + opts->no_auto_adjust = true; + break; + case OPT_STDC_ONLY: + opts->stdc_only = true; + break; + case OPT_STATS: + opts->stats = true; + break; + case OPT_LOG_FILE: + opts->log_file = arg; + break; + case OPT_LOG_LEVEL: + opts->log_level = arg; + break; + case OPT_STATS_FILE: + opts->stats_file = arg; + break; + + case ARGP_KEY_ARG: + if (state->arg_num) + return ARGP_ERR_UNKNOWN; + if (opts->exename == NULL) { + /* remaining options will be processed in ARGP_KEY_ARGS */ + return ARGP_ERR_UNKNOWN; + } + break; + + case ARGP_KEY_ARGS: + /* process remaining non-option arguments */ + opts->exename = state->argv[state->next]; + opts->idx = state->next; + break; + + case ARGP_KEY_NO_ARGS: + case ARGP_KEY_END: + if (state->arg_num < 1) + argp_usage(state); + break; + + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +static void setup_child_environ(struct opts *opts) +{ + const char *lib, *cur; + char *preload; + + /* value options: only set when explicitly specified */ + if (opts->wait_method) + setenv("DTO_WAIT_METHOD", opts->wait_method, 1); + if (opts->min_bytes) + setenv("DTO_MIN_BYTES", opts->min_bytes, 1); + if (opts->cpu_fraction) + setenv("DTO_CPU_SIZE_FRACTION", opts->cpu_fraction, 1); + if (opts->numa_aware) + setenv("DTO_IS_NUMA_AWARE", opts->numa_aware, 1); + if (opts->wq_list) + setenv("DTO_WQ_LIST", opts->wq_list, 1); + if (opts->umwait_delay) + setenv("DTO_UMWAIT_DELAY", opts->umwait_delay, 1); + if (opts->overlapping_memmove) + setenv("DTO_OVERLAPPING_MEMMOVE_ACTION", opts->overlapping_memmove, 1); + if (opts->log_file) + setenv("DTO_LOG_FILE", opts->log_file, 1); + if (opts->log_level) + setenv("DTO_LOG_LEVEL", opts->log_level, 1); + if (opts->stats_file) + setenv("DTO_STATS_FILE", opts->stats_file, 1); + + /* flag options: only set when explicitly specified */ + if (opts->no_memcpy) + setenv("DTO_DSA_MEMCPY", "0", 1); + if (opts->no_memmove) + setenv("DTO_DSA_MEMMOVE", "0", 1); + if (opts->no_memset) + setenv("DTO_DSA_MEMSET", "0", 1); + if (opts->no_memcmp) + setenv("DTO_DSA_MEMCMP", "0", 1); + if (opts->no_cache_control) + setenv("DTO_DSA_CC", "0", 1); + if (opts->no_auto_adjust) + setenv("DTO_AUTO_ADJUST_KNOBS", "0", 1); + if (opts->stdc_only) + setenv("DTO_USESTDC_CALLS", "1", 1); + if (opts->stats) + setenv("DTO_COLLECT_STATS", "1", 1); + + /* resolve the libdto path and prepend it to LD_PRELOAD */ + lib = opts->library ? opts->library : getenv("DTO_LIBRARY"); + if (lib == NULL) + lib = LIBDTO_PATH; + + cur = getenv("LD_PRELOAD"); + if (cur == NULL || *cur == '\0') { + setenv("LD_PRELOAD", lib, 1); + return; + } + + /* + * LD_PRELOAD is a colon separated list, so it can be much longer than a + * single path. Allocate the result instead of formatting it into a + * fixed size buffer, which would silently drop part of what the caller + * had already preloaded. + */ + if (asprintf(&preload, "%s:%s", lib, cur) < 0) { + perror("cannot build LD_PRELOAD"); + exit(EXIT_FAILURE); + } + setenv("LD_PRELOAD", preload, 1); + free(preload); +} + +int main(int argc, char *argv[]) +{ + struct argp argp = { + .options = dtoctl_options, + .parser = parse_option, + .args_doc = " [...]", + .doc = "dtoctl -- run a program with DSA Transparent Offload (DTO) preloaded", + }; + + argp_parse(&argp, argc, argv, ARGP_IN_ORDER, NULL, &opts); + + /* pass only the target program and its arguments to execvp() */ + argc -= opts.idx; + argv += opts.idx; + + setup_child_environ(&opts); + + execvp(opts.exename, argv); + perror(opts.exename); + + return -1; +} From c09d8ba4f6a530ad46020ef852cc3e7266ffc475 Mon Sep 17 00:00:00 2001 From: Honggyu Kim Date: Sat, 5 Sep 2026 08:59:50 +0900 Subject: [PATCH 2/8] Add a dtoctl man page The dtoctl options mirror the DTO_* environment variables, but a user coming to the tool for the first time still needs a single authoritative place that explains what each option does, what values it accepts, and how the library path and LD_PRELOAD are resolved. Scattering that information across the source help text alone makes it hard to consult without running the tool. Add a manual page in the same troff-friendly markdown form the hmsdk project uses for hmctl, and render it as a section 1 man page for the user command. Besides the per-option reference it documents the library resolution order and the override semantics, and it gathers the common tuning goals - latency, power, and cache-pollution avoidance - as ready-to-run examples such as $ dtoctl -w umwait -c 0.0 --no-auto-adjust ./prog so users can map an intent directly onto a command instead of reverse engineering it from the individual variables. Keep the markdown as the source form and generate dtoctl.1 with pandoc during the Makefile and CMake builds. Both build systems install the generated page under man1, so it can be read with $ man 1 dtoctl Pandoc is optional: when it is unavailable, print a message recommending its installation and skip manual generation. The Makefile can still install an already generated page. Ignore its generated doc/dtoctl.1 and remove it on make clean; CMake keeps its generated page in the build directory. Signed-off-by: Honggyu Kim --- .gitignore | 1 + CMakeLists.txt | 18 +++++ Makefile | 25 ++++++- doc/dtoctl.md | 181 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 doc/dtoctl.md diff --git a/.gitignore b/.gitignore index 977a993..b9711b7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ libdto.so* build*/ tests/baselines*/ dtoctl +/doc/dtoctl.1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 4567c1e..0a334d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,24 @@ add_executable(dtoctl dtoctl.c) target_compile_definitions(dtoctl PRIVATE LIBDTO_PATH="${DTOCTL_LIBPATH}") install(TARGETS dtoctl RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +# Build and install the optional section 1 manual. +find_program(PANDOC_EXECUTABLE NAMES pandoc) +if(PANDOC_EXECUTABLE) + set(DTOCTL_MAN_PAGE "${CMAKE_CURRENT_BINARY_DIR}/doc/dtoctl.1") + add_custom_command( + OUTPUT "${DTOCTL_MAN_PAGE}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/doc" + COMMAND "${PANDOC_EXECUTABLE}" --standalone --to=man + "${CMAKE_CURRENT_SOURCE_DIR}/doc/dtoctl.md" -o "${DTOCTL_MAN_PAGE}" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/doc/dtoctl.md" + COMMENT "Generating dtoctl(1) manual" + VERBATIM) + add_custom_target(dtoctl-man ALL DEPENDS "${DTOCTL_MAN_PAGE}") + install(FILES "${DTOCTL_MAN_PAGE}" DESTINATION ${CMAKE_INSTALL_MANDIR}/man1) +else() + message(STATUS "Skipping dtoctl manual generation and installation: install pandoc to build the man page.") +endif() + # Build dto-test and dto-test-wodto add_executable(dto-test dto-test.c) target_link_libraries(dto-test PRIVATE DTO::dto pthread) diff --git a/Makefile b/Makefile index c200ea1..8b4b2cf 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,16 @@ # # SPDX-License-Identifier: MIT -all: libdto dtoctl dto-test-wodto +all: libdto dtoctl dto-test-wodto man + +.PHONY: man install-man DML_LIB_CXX=-D_GNU_SOURCE # Path to libdto.so that dtoctl prepends to LD_PRELOAD (matches "make install"). DTOCTL_LIBPATH ?= /usr/lib64/libdto.so +PANDOC ?= pandoc +MANDIR ?= /usr/share/man libdto: dto.c gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_STATS_SUPPORT -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg @@ -18,12 +22,28 @@ libdto_nostats: dto.c dtoctl: dtoctl.c gcc -O2 -Wall dtoctl.c -DLIBDTO_PATH=\"$(DTOCTL_LIBPATH)\" -o dtoctl -install: +man: doc/dtoctl.1 + +doc/dtoctl.1: doc/dtoctl.md + @if command -v "$(PANDOC)" >/dev/null 2>&1; then \ + "$(PANDOC)" --standalone --to=man $< -o $@; \ + else \ + echo "Skipping dtoctl manual generation: install pandoc to build the man page."; \ + fi + +install: install-man cp libdto.so.1.0 /usr/lib64/ ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so.1 ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so cp dtoctl /usr/bin/ +install-man: man + @if [ -f doc/dtoctl.1 ]; then \ + install -D -m 644 doc/dtoctl.1 "$(MANDIR)/man1/dtoctl.1"; \ + else \ + echo "Skipping dtoctl manual installation: no generated man page is available."; \ + fi + install-local: ln -sf ./libdto.so.1.0 ./libdto.so.1 ln -sf ./libdto.so.1.0 ./libdto.so @@ -36,3 +56,4 @@ dto-test-wodto: dto-test.c clean: rm -rf *.o *.so dto-test dtoctl + rm -f doc/dtoctl.1 diff --git a/doc/dtoctl.md b/doc/dtoctl.md new file mode 100644 index 0000000..af9f5e7 --- /dev/null +++ b/doc/dtoctl.md @@ -0,0 +1,181 @@ +% DTOCTL(1) Dtoctl User Manuals +% Honggyu Kim +% 2026 + +NAME +==== +dtoctl - run a program with DSA Transparent Offload (DTO) preloaded + + +SYNOPSIS +======== +dtoctl [_options_] COMMAND [_command-options_] + + +DESCRIPTION +=========== +The **dtoctl** tool runs `COMMAND` with the DSA Transparent Offload library +(**libdto.so**) preloaded, so that its `memcpy`, `memmove`, `memset`, and +`memcmp` calls are transparently offloaded to Intel DSA (Data Streaming +Accelerator). + +Without **dtoctl**, users must set **LD_PRELOAD** to point at **libdto.so** and +export a set of **DTO_\*** environment variables to control DTO behavior. +**dtoctl** replaces that manual setup: it converts its command line options into +the corresponding **DTO_\*** environment variables, prepends **libdto.so** to +**LD_PRELOAD**, and then executes `COMMAND`. + +Only the options that are explicitly given are applied. Options that are not +given leave any pre-existing **DTO_\*** environment variables (or the DTO +built-in defaults) untouched, and an explicitly given option overrides a +previously exported variable of the same name. + + +LIBRARY RESOLUTION +================== +The path of **libdto.so** to preload is resolved in the following order: + +1. The `-l`, `--library` option, if given. +2. The **DTO_LIBRARY** environment variable, if set. +3. The compiled-in default path (`/usr/local/lib/libdto.so` for a default + CMake build, or `/usr/lib64/libdto.so` for the Makefile build). + +The resolved path is prepended to any existing **LD_PRELOAD** value. + + +OPTIONS +======= + +Library +------- +-l _PATH_, \--library=_PATH_ +: Path to **libdto.so** to prepend to **LD_PRELOAD**. Overrides + **DTO_LIBRARY** and the compiled-in default. + Default: the compiled-in library path (see LIBRARY RESOLUTION). + +Offload tuning +-------------- +-w _METHOD_, \--wait-method=_METHOD_ +: How to wait for DSA completion: _yield_, _busypoll_, _umwait_, or _tpause_. + Sets **DTO_WAIT_METHOD**. Default: _busypoll_. + +-b _BYTES_, \--min-bytes=_BYTES_ +: Minimum operation size offloaded to DSA. Smaller operations run on the CPU. + Sets **DTO_MIN_BYTES**. Default: 65536. + +-c _FRACTION_, \--cpu-fraction=_FRACTION_ +: Fraction (0.0 <= f < 1.0) of each operation performed on the CPU in parallel + to DSA. Sets **DTO_CPU_SIZE_FRACTION**. Default: 0.0. + +-n _MODE_, \--numa-aware=_MODE_ +: NUMA awareness: _0_ disable, _1_ buffer-centric, _2_ cpu-centric. + Sets **DTO_IS_NUMA_AWARE**. Default: _0_. + +-q _LIST_, \--wq-list=_LIST_ +: Semicolon-separated list of DSA work queues to use, e.g. "wq0.0;wq2.0". + Names must match those under /dev/dsa/. Sets **DTO_WQ_LIST**. + Default: auto-discover all available WQs. + +\--umwait-delay=_CYCLES_ +: Delay in cycles for the umwait instruction. Sets **DTO_UMWAIT_DELAY**. + Default: 100000. + +\--overlapping-memmove=_WHERE_ +: Where to run memmove with overlapping buffers: _cpu_ or _dsa_. + Sets **DTO_OVERLAPPING_MEMMOVE_ACTION** (cpu -> 0, dsa -> 1). Default: _cpu_. + +Disable DSA offload for specific operations +------------------------------------------- +By default every one of these operations is offloaded to DSA; each option below +turns a single one back into its standard C library call on the CPU. + +\--no-memcpy +: Use the system memcpy instead of DSA. Sets **DTO_DSA_MEMCPY=0**. + +\--no-memmove +: Use the system memmove instead of DSA. Sets **DTO_DSA_MEMMOVE=0**. + +\--no-memset +: Use the system memset instead of DSA. Sets **DTO_DSA_MEMSET=0**. + +\--no-memcmp +: Use the system memcmp instead of DSA. Sets **DTO_DSA_MEMCMP=0**. + +Other toggles +------------- +\--no-cache-control +: Clear the DSA cache control flag to avoid cache pollution. + Sets **DTO_DSA_CC=0**. Default: cache control is on. + +\--no-auto-adjust +: Disable auto tuning of cpu-fraction and min-bytes. + Sets **DTO_AUTO_ADJUST_KNOBS=0**. Default: auto tuning is on. + +\--stdc-only +: Use only the standard C memory functions, no DSA offload. + Sets **DTO_USESTDC_CALLS=1**. Default: DSA offload is enabled. + +Statistics and logging +---------------------- +\--stats +: Enable stats collection. For debugging/profiling only, as it slows down the + workload. Sets **DTO_COLLECT_STATS=1**. Default: off. + +\--stats-file=_PATH_ +: Write the stats histogram to _PATH_ instead of standard output. + Sets **DTO_STATS_FILE**. Default: standard output. + +\--log-file=_PATH_ +: Redirect DTO output to _PATH_ (the file name is suffixed by the pid). + Sets **DTO_LOG_FILE**. Default: standard output. + +\--log-level=_LEVEL_ +: Verbosity of DTO logging: _0_, _1_, or _2_. Sets **DTO_LOG_LEVEL**. + Default: _0_. + +-?, \--help +: Print help message and list of options with description. + +\--usage +: Print usage string. + + +EXAMPLES +======== +Run a program with DTO using the installed **libdto.so**: + + $ dtoctl ./prog + +The equivalent of the manual setup + + $ export LD_PRELOAD=/usr/lib64/libdto.so + $ export DTO_WAIT_METHOD=busypoll + $ export DTO_CPU_SIZE_FRACTION=0.33 + $ export DTO_AUTO_ADJUST_KNOBS=1 + $ ./prog + +becomes a single command: + + $ dtoctl -w busypoll -c 0.33 ./prog + +Latency reduction mode (auto tuning on, busy polling): + + $ dtoctl -w busypoll ./prog + +Power reduction mode (offload everything to DSA, wait with umwait): + + $ dtoctl -w umwait -c 0.0 --no-auto-adjust ./prog + +Avoid cache pollution (offload everything, clear cache control): + + $ dtoctl -w yield -c 0.0 --no-auto-adjust --no-cache-control ./prog + +Use a locally built library and a specific set of work queues, and collect +stats: + + $ dtoctl -l ./libdto.so.1.0 -q "wq0.0;wq2.0;wq4.0;wq6.0" --stats ./prog + + +SEE ALSO +======== +**ld.so**(8) From 74eba2db85e00eb872261c8bf5f8dac4e97dc77c Mon Sep 17 00:00:00 2001 From: Honggyu Kim Date: Sat, 5 Sep 2026 08:59:50 +0900 Subject: [PATCH 3/8] Document dtoctl in the README The README is where users first learn how to configure and run DTO, and until now it only described the manual LD_PRELOAD and DTO_* export workflow. A reader following it has no reason to know that dtoctl exists or that it removes most of that ceremony, so the easier path stays hidden precisely where people look for guidance. Introduce dtoctl after the environment variable reference and the common usage models that accompany it, so that discussion stays in one piece and the new section can point back at those usage models. Contrast the old export sequence with the equivalent $ dtoctl -w busypoll -c 0.33 ./prog and spell out the resolution order for libdto.so along with the rule that only the options actually passed are applied. Keep the per-option reference in dtoctl(1) instead of repeating it here. The option list already exists in the --help output and in the manual, and a third copy in the README would only be one more place to forget when an option changes. Signed-off-by: Honggyu Kim --- README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/README.md b/README.md index bd3e239..4175970 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,39 @@ Although not the only usage models of DTO, the following are some common ones: DTO_WAIT_METHOD=yield or umwait (saves either cycles or power) +## Using dtoctl + +Setting `LD_PRELOAD` and exporting the `DTO_*` environment variables by hand is +error prone. The `dtoctl` tool provides a simpler interface: it takes the DTO +settings as command line options, converts them to the corresponding `DTO_*` +environment variables, prepends `libdto.so` to `LD_PRELOAD`, and then runs the +target program. For example, instead of + +```bash +export LD_PRELOAD=/usr/local/lib/libdto.so +export DTO_WAIT_METHOD=busypoll +export DTO_CPU_SIZE_FRACTION=0.33 +export DTO_AUTO_ADJUST_KNOBS=1 +./prog +``` + +you can run + +```bash +dtoctl -w busypoll -c 0.33 ./prog +``` + +Every environment variable listed above has a corresponding option, and only the +options you pass are applied: a `DTO_*` variable you already exported is left +untouched, and an explicit option overrides it. The `libdto.so` path is resolved +from `-l/--library`, then `$DTO_LIBRARY`, then the compiled-in default +(`/usr/local/lib/libdto.so` for a CMake build, `/usr/lib64/libdto.so` for the +Makefile build), and is prepended to any existing `LD_PRELOAD`. + +Run `dtoctl --help` for the option list, or `man 1 dtoctl` for the full manual, +which documents each option and gives a ready-to-run command for each of the +usage models above. + ## Build Pre-requisite packages: From 90e67d160160ef6b055534d6b56e0e9f3b90118e Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 21 Oct 2026 07:14:24 -0700 Subject: [PATCH 4/8] dto: honor DTO_WAIT_METHOD=yield The yield wait method is implemented in dsa_wait() and documented in the README, but dsa_init() never matched "yield" when parsing DTO_WAIT_METHOD, so DTO_WAIT_METHOD=yield silently ran with busypoll. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Mv7xKtQPkGoXQUvvuuhAEz --- dto.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dto.c b/dto.c index 5eb211e..c59c877 100644 --- a/dto.c +++ b/dto.c @@ -1279,6 +1279,10 @@ static int dsa_init(void) max_avg_waits = MAX_AVG_POLL_WAITS; } else LOG_ERROR("umwait not supported. Falling back to default wait method\n"); + } else if (!strncmp(env_str, wait_names[WAIT_YIELD], strlen(wait_names[WAIT_YIELD]))) { + wait_method = WAIT_YIELD; + min_avg_waits = MIN_AVG_YIELD_WAITS; + max_avg_waits = MAX_AVG_YIELD_WAITS; } else if (!strncmp(env_str, wait_names[WAIT_TPAUSE], strlen(wait_names[WAIT_TPAUSE]))) { if (waitpkg_support) { wait_method = WAIT_TPAUSE; From 67005a13e48147bc16d4334ac3888022a47bafaf Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 21 Oct 2026 07:18:17 -0700 Subject: [PATCH 5/8] dtoctl: validate options, resolve the library up front, add --dry-run Make the wrapper catch the mistakes it exists to prevent instead of passing them on to libdto, which silently falls back to a default: - Validate every option value: wait method names, integer ranges for --numa-aware and --log-level, a cpu fraction in [0, 1), a positive umwait delay, and non-empty paths and WQ list. --min-bytes takes an optional K/M/G suffix and is passed on as plain bytes; libdto would read "64K" as 64 and "abc" as 0. - Resolve libdto.so to an absolute path and fail with exit status 125 if it cannot be read. ld.so only warns about a missing preload and runs the program without DTO, and a relative path stopped working as soon as the program changed directory. When built with LIBDTO_REL_PATH, prefer the library installed next to dtoctl so a relocated install finds its own copy. - Drop other libdto.so entries from an inherited LD_PRELOAD, so one process never loads two copies of DTO. - --stats-file implies --stats; libdto ignores DTO_STATS_FILE otherwise. - Add -i/--ignore-env to unset inherited DTO_* variables. The --no-* flags can only turn features off, so this is the way to undo an exported DTO_USESTDC_CALLS=1 or DTO_DSA_MEMCPY=0. - Add --dry-run to print the resolved library, LD_PRELOAD, each DTO_* variable with whether it came from an option or the environment, and the command, without running it. - Add --version, and exit with 126/127 when the program cannot be run or is not found, as env(1) and timeout(1) do. - Make the options local to main() and drop dead argument handling. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Mv7xKtQPkGoXQUvvuuhAEz --- dtoctl.c | 502 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 425 insertions(+), 77 deletions(-) diff --git a/dtoctl.c b/dtoctl.c index 707cd3a..7c72444 100644 --- a/dtoctl.c +++ b/dtoctl.c @@ -9,10 +9,14 @@ */ #ifndef _GNU_SOURCE -#define _GNU_SOURCE /* for asprintf() */ +#define _GNU_SOURCE /* for asprintf() and environ */ #endif #include +#include +#include +#include +#include #include #include #include @@ -23,6 +27,18 @@ #define LIBDTO_PATH "/usr/lib64/libdto.so" #endif +#ifndef DTO_VERSION +#define DTO_VERSION "unknown" +#endif + +/* exit statuses for dtoctl's own failures, following env(1) and timeout(1) */ +#define EXIT_DTOCTL_FAILURE 125 +#define EXIT_CANNOT_INVOKE 126 +#define EXIT_ENOENT 127 + +/* libdto copies DTO_WQ_LIST into a buffer of this size */ +#define WQ_LIST_MAX 256 + /* argp option keys for the long-only options (no short equivalents) */ enum { OPT_UMWAIT_DELAY = 0x100, @@ -38,6 +54,7 @@ enum { OPT_NO_AUTO_ADJUST, OPT_STDC_ONLY, OPT_STATS, + OPT_DRY_RUN, }; struct opts { @@ -46,6 +63,8 @@ struct opts { /* library / preload */ const char *library; + bool ignore_env; + bool dry_run; /* value options (NULL means "not specified") */ const char *wait_method; @@ -70,19 +89,30 @@ struct opts { bool stats; }; -struct opts opts; - -/* (a part of) output in --help option (generated by argp runtime) */ +/* (a part of) output in --help and --version options (generated by argp runtime) */ +const char *argp_program_version = "dtoctl (DTO) " DTO_VERSION; const char *argp_program_bug_address = "https://github.com/intel/DTO/issues"; +static const char *const wait_methods[] = { + "busypoll", "umwait", "yield", "tpause", NULL +}; + /* Option groups are separated using entries with only a .doc field. */ static struct argp_option dtoctl_options[] = { - {.doc = "Library:", .group = 1}, + {.doc = "General:", .group = 1}, {.name = "library", .key = 'l', .arg = "PATH", .doc = "Path to libdto.so to prepend to LD_PRELOAD. Overrides $DTO_LIBRARY " - "and the compiled-in default (" LIBDTO_PATH ")"}, + "and the installed library (" LIBDTO_PATH ")"}, + {.name = "ignore-env", + .key = 'i', + .doc = "Unset the DTO_* variables inherited from the environment, so that " + "only the options given here and DTO's defaults apply"}, + {.name = "dry-run", + .key = OPT_DRY_RUN, + .doc = "Print the library, the DTO_* environment and the command that " + "would run, then exit without running it"}, {.doc = "Offload tuning:", .group = 2}, {.name = "wait-method", @@ -93,8 +123,8 @@ static struct argp_option dtoctl_options[] = { {.name = "min-bytes", .key = 'b', .arg = "BYTES", - .doc = "Minimum operation size offloaded to DSA (sets DTO_MIN_BYTES; " - "default: 65536)"}, + .doc = "Minimum operation size offloaded to DSA; accepts a K, M or G suffix " + "(sets DTO_MIN_BYTES; default: 64K)"}, {.name = "cpu-fraction", .key = 'c', .arg = "FRACTION", @@ -158,12 +188,12 @@ static struct argp_option dtoctl_options[] = { {.name = "stats-file", .key = OPT_STATS_FILE, .arg = "PATH", - .doc = "Write the stats histogram to PATH instead of standard output " - "(sets DTO_STATS_FILE; default: standard output)"}, + .doc = "Write the stats histogram to PATH instead of standard output; " + "implies --stats (sets DTO_STATS_FILE; default: standard output)"}, {.name = "log-file", .key = OPT_LOG_FILE, .arg = "PATH", - .doc = "Redirect DTO output to PATH (suffixed by pid) (sets DTO_LOG_FILE; " + .doc = "Redirect DTO output to PATH.. (sets DTO_LOG_FILE; " "default: standard output)"}, {.name = "log-level", .key = OPT_LOG_LEVEL, @@ -171,33 +201,135 @@ static struct argp_option dtoctl_options[] = { .doc = "Verbosity of DTO logging: 0, 1, or 2 (sets DTO_LOG_LEVEL; " "default: 0)"}, - {NULL}, + {0}, }; +/* Parse a decimal integer, rejecting signs, empty strings and trailing garbage. */ +static bool parse_ulong(const char *s, unsigned long *val) +{ + char *end; + + if (!isdigit((unsigned char)*s)) + return false; + errno = 0; + *val = strtoul(s, &end, 10); + return errno == 0 && *end == '\0'; +} + +/* Parse a byte count with an optional K, M or G suffix (powers of 1024). */ +static bool parse_size(const char *s, unsigned long *val) +{ + unsigned int shift = 0; + unsigned long n; + char *end; + + if (!isdigit((unsigned char)*s)) + return false; + errno = 0; + n = strtoul(s, &end, 10); + if (errno) + return false; + + switch (*end) { + case 'k': case 'K': + shift = 10; + end++; + break; + case 'm': case 'M': + shift = 20; + end++; + break; + case 'g': case 'G': + shift = 30; + end++; + break; + } + if (*end != '\0' || n > (ULONG_MAX >> shift)) + return false; + + *val = n << shift; + return true; +} + +/* Accept an integer in [0, max] for options that select a mode or level. */ +static const char *check_level(struct argp_state *state, const char *name, + char *arg, unsigned long max) +{ + unsigned long val; + + if (!parse_ulong(arg, &val) || val > max) + argp_error(state, "invalid %s '%s': must be an integer from 0 to %lu", + name, arg, max); + return arg; +} + +static const char *check_path(struct argp_state *state, const char *name, char *arg) +{ + if (*arg == '\0') + argp_error(state, "--%s needs a non-empty PATH", name); + return arg; +} + static error_t parse_option(int key, char *arg, struct argp_state *state) { struct opts *opts = state->input; + unsigned long val = 0; + char *end, *bytes; + double frac; + int i; switch (key) { case 'l': - opts->library = arg; + opts->library = check_path(state, "library", arg); + break; + case 'i': + opts->ignore_env = true; + break; + case OPT_DRY_RUN: + opts->dry_run = true; break; case 'w': + for (i = 0; wait_methods[i]; i++) { + if (!strcmp(arg, wait_methods[i])) + break; + } + if (wait_methods[i] == NULL) + argp_error(state, "invalid wait method '%s': must be yield, " + "busypoll, umwait, or tpause", arg); opts->wait_method = arg; break; case 'b': - opts->min_bytes = arg; + if (!parse_size(arg, &val)) + argp_error(state, "invalid minimum size '%s': must be a byte " + "count, optionally with a K, M or G suffix", arg); + /* libdto takes plain bytes, so expand any suffix here */ + if (asprintf(&bytes, "%lu", val) < 0) + argp_failure(state, EXIT_DTOCTL_FAILURE, errno, "--min-bytes"); + opts->min_bytes = bytes; break; case 'c': + errno = 0; + frac = strtod(arg, &end); + /* the negated range check also rejects NaN */ + if (end == arg || *end != '\0' || errno || !(frac >= 0.0 && frac < 1.0)) + argp_error(state, "invalid cpu fraction '%s': must be a number " + ">= 0.0 and < 1.0", arg); opts->cpu_fraction = arg; break; case 'n': - opts->numa_aware = arg; + opts->numa_aware = check_level(state, "NUMA mode", arg, 2); break; case 'q': + if (*arg == '\0' || strlen(arg) >= WQ_LIST_MAX) + argp_error(state, "invalid WQ list: must be 1 to %d characters", + WQ_LIST_MAX - 1); opts->wq_list = arg; break; case OPT_UMWAIT_DELAY: + /* libdto treats 0 as "use the default", so reject it */ + if (!parse_ulong(arg, &val) || val == 0) + argp_error(state, "invalid umwait delay '%s': must be a positive " + "number of cycles", arg); opts->umwait_delay = arg; break; case OPT_OVERLAPPING_MEMMOVE: @@ -205,11 +337,9 @@ static error_t parse_option(int key, char *arg, struct argp_state *state) opts->overlapping_memmove = "0"; else if (!strcmp(arg, "dsa")) opts->overlapping_memmove = "1"; - else { - fprintf(stderr, - "Error: --overlapping-memmove must be 'cpu' or 'dsa'.\n\n"); - argp_usage(state); - } + else + argp_error(state, "invalid --overlapping-memmove '%s': must be " + "cpu or dsa", arg); break; case OPT_NO_MEMCPY: opts->no_memcpy = true; @@ -236,23 +366,18 @@ static error_t parse_option(int key, char *arg, struct argp_state *state) opts->stats = true; break; case OPT_LOG_FILE: - opts->log_file = arg; + opts->log_file = check_path(state, "log-file", arg); break; case OPT_LOG_LEVEL: - opts->log_level = arg; + opts->log_level = check_level(state, "log level", arg, 2); break; case OPT_STATS_FILE: - opts->stats_file = arg; + opts->stats_file = check_path(state, "stats-file", arg); break; case ARGP_KEY_ARG: - if (state->arg_num) - return ARGP_ERR_UNKNOWN; - if (opts->exename == NULL) { - /* remaining options will be processed in ARGP_KEY_ARGS */ - return ARGP_ERR_UNKNOWN; - } - break; + /* leave the program and its arguments to ARGP_KEY_ARGS */ + return ARGP_ERR_UNKNOWN; case ARGP_KEY_ARGS: /* process remaining non-option arguments */ @@ -260,9 +385,9 @@ static error_t parse_option(int key, char *arg, struct argp_state *state) opts->idx = state->next; break; - case ARGP_KEY_NO_ARGS: case ARGP_KEY_END: - if (state->arg_num < 1) + /* --dry-run can show the environment without a program */ + if (opts->exename == NULL && !opts->dry_run) argp_usage(state); break; @@ -272,95 +397,318 @@ static error_t parse_option(int key, char *arg, struct argp_state *state) return 0; } -static void setup_child_environ(struct opts *opts) +/* DTO_* variables set from options, in the order they were set, for --dry-run */ +static const char *option_vars[32]; +static int nr_option_vars; + +static void set_option_var(const char *name, const char *value) { - const char *lib, *cur; - char *preload; + if (setenv(name, value, 1)) { + perror("dtoctl: setenv"); + exit(EXIT_DTOCTL_FAILURE); + } + option_vars[nr_option_vars++] = name; +} + +static bool is_option_var(const char *name, int len) +{ + for (int i = 0; i < nr_option_vars; i++) { + if ((int)strlen(option_vars[i]) == len && !strncmp(option_vars[i], name, len)) + return true; + } + return false; +} +/* Unset every DTO_* variable inherited from the caller (--ignore-env). */ +static void clear_dto_environ(void) +{ + char **env, *eq, *name; + bool found; + + /* unsetenv() reshuffles environ, so rescan it after every removal */ + do { + found = false; + for (env = environ; *env; env++) { + eq = strchr(*env, '='); + if (strncmp(*env, "DTO_", 4) || eq == NULL) + continue; + name = strndup(*env, eq - *env); + if (name == NULL || unsetenv(name)) { + perror("dtoctl: unsetenv"); + exit(EXIT_DTOCTL_FAILURE); + } + free(name); + found = true; + break; + } + } while (found); +} + +static void setup_dto_environ(struct opts *opts) +{ /* value options: only set when explicitly specified */ if (opts->wait_method) - setenv("DTO_WAIT_METHOD", opts->wait_method, 1); + set_option_var("DTO_WAIT_METHOD", opts->wait_method); if (opts->min_bytes) - setenv("DTO_MIN_BYTES", opts->min_bytes, 1); + set_option_var("DTO_MIN_BYTES", opts->min_bytes); if (opts->cpu_fraction) - setenv("DTO_CPU_SIZE_FRACTION", opts->cpu_fraction, 1); + set_option_var("DTO_CPU_SIZE_FRACTION", opts->cpu_fraction); if (opts->numa_aware) - setenv("DTO_IS_NUMA_AWARE", opts->numa_aware, 1); + set_option_var("DTO_IS_NUMA_AWARE", opts->numa_aware); if (opts->wq_list) - setenv("DTO_WQ_LIST", opts->wq_list, 1); + set_option_var("DTO_WQ_LIST", opts->wq_list); if (opts->umwait_delay) - setenv("DTO_UMWAIT_DELAY", opts->umwait_delay, 1); + set_option_var("DTO_UMWAIT_DELAY", opts->umwait_delay); if (opts->overlapping_memmove) - setenv("DTO_OVERLAPPING_MEMMOVE_ACTION", opts->overlapping_memmove, 1); + set_option_var("DTO_OVERLAPPING_MEMMOVE_ACTION", opts->overlapping_memmove); if (opts->log_file) - setenv("DTO_LOG_FILE", opts->log_file, 1); + set_option_var("DTO_LOG_FILE", opts->log_file); if (opts->log_level) - setenv("DTO_LOG_LEVEL", opts->log_level, 1); + set_option_var("DTO_LOG_LEVEL", opts->log_level); if (opts->stats_file) - setenv("DTO_STATS_FILE", opts->stats_file, 1); + set_option_var("DTO_STATS_FILE", opts->stats_file); /* flag options: only set when explicitly specified */ if (opts->no_memcpy) - setenv("DTO_DSA_MEMCPY", "0", 1); + set_option_var("DTO_DSA_MEMCPY", "0"); if (opts->no_memmove) - setenv("DTO_DSA_MEMMOVE", "0", 1); + set_option_var("DTO_DSA_MEMMOVE", "0"); if (opts->no_memset) - setenv("DTO_DSA_MEMSET", "0", 1); + set_option_var("DTO_DSA_MEMSET", "0"); if (opts->no_memcmp) - setenv("DTO_DSA_MEMCMP", "0", 1); + set_option_var("DTO_DSA_MEMCMP", "0"); if (opts->no_cache_control) - setenv("DTO_DSA_CC", "0", 1); + set_option_var("DTO_DSA_CC", "0"); if (opts->no_auto_adjust) - setenv("DTO_AUTO_ADJUST_KNOBS", "0", 1); + set_option_var("DTO_AUTO_ADJUST_KNOBS", "0"); if (opts->stdc_only) - setenv("DTO_USESTDC_CALLS", "1", 1); - if (opts->stats) - setenv("DTO_COLLECT_STATS", "1", 1); + set_option_var("DTO_USESTDC_CALLS", "1"); + /* libdto ignores DTO_STATS_FILE unless stats are collected */ + if (opts->stats || opts->stats_file) + set_option_var("DTO_COLLECT_STATS", "1"); +} - /* resolve the libdto path and prepend it to LD_PRELOAD */ - lib = opts->library ? opts->library : getenv("DTO_LIBRARY"); - if (lib == NULL) +/* The libdto.so installed relative to this dtoctl, if there is one. */ +static char *installed_library(void) +{ +#ifdef LIBDTO_REL_PATH + char exe[PATH_MAX], *path, *real; + ssize_t len; + + len = readlink("/proc/self/exe", exe, sizeof(exe) - 1); + if (len < 0) + return NULL; + exe[len] = '\0'; + + if (asprintf(&path, "%s/%s", dirname(exe), LIBDTO_REL_PATH) < 0) + return NULL; + real = realpath(path, NULL); + free(path); + if (real && access(real, R_OK)) { + free(real); + real = NULL; + } + return real; +#else + return NULL; +#endif +} + +/* + * Resolve the libdto.so to preload: --library, then $DTO_LIBRARY, then the + * library installed alongside dtoctl, then the compiled-in default. The + * result is an absolute path, so it still loads after the program changes + * directory. A missing library is an error here; ld.so would only print a + * warning and run the program without DTO. + */ +static char *resolve_library(struct opts *opts, const char **source) +{ + const char *lib; + char *path; + + if (opts->library) { + lib = opts->library; + *source = "--library"; + } else if ((lib = getenv("DTO_LIBRARY")) && *lib) { + *source = "DTO_LIBRARY"; + } else if ((path = installed_library())) { + *source = "installed with dtoctl"; + return path; + } else { lib = LIBDTO_PATH; + *source = "compiled-in default"; + } - cur = getenv("LD_PRELOAD"); - if (cur == NULL || *cur == '\0') { - setenv("LD_PRELOAD", lib, 1); - return; + path = realpath(lib, NULL); + if (path == NULL || access(path, R_OK)) { + fprintf(stderr, "dtoctl: cannot use DTO library '%s' (%s): %s\n", + lib, *source, strerror(errno)); + if (opts->library == NULL) + fprintf(stderr, "dtoctl: use -l/--library or DTO_LIBRARY to " + "choose a libdto.so\n"); + exit(EXIT_DTOCTL_FAILURE); + } + + /* ld.so splits LD_PRELOAD at colons and spaces */ + if (strpbrk(path, ": ")) { + fprintf(stderr, "dtoctl: DTO library path '%s' cannot be preloaded: " + "it contains a colon or a space\n", path); + exit(EXIT_DTOCTL_FAILURE); } + return path; +} + +/* true for an LD_PRELOAD entry that names a libdto.so (libdto.so, libdto.so.1, ...) */ +static bool is_libdto(const char *entry) +{ + const char *base = strrchr(entry, '/'); + + base = base ? base + 1 : entry; + return !strncmp(base, "libdto.so", strlen("libdto.so")); +} + +/* + * Put lib at the front of LD_PRELOAD. Other libdto.so entries are dropped: + * two copies of DTO in one process would both initialize and both open the + * work queues. + */ +static void setup_preload(const char *lib) +{ + const char *cur = getenv("LD_PRELOAD"); + char *preload, *copy, *entry, *save, *real; /* - * LD_PRELOAD is a colon separated list, so it can be much longer than a - * single path. Allocate the result instead of formatting it into a - * fixed size buffer, which would silently drop part of what the caller - * had already preloaded. + * LD_PRELOAD is a list, so it can be much longer than a single path. + * Size the result from the inputs: the kept entries and their colons + * never take more than the old value plus one byte. */ - if (asprintf(&preload, "%s:%s", lib, cur) < 0) { - perror("cannot build LD_PRELOAD"); - exit(EXIT_FAILURE); + preload = malloc(strlen(lib) + (cur ? strlen(cur) : 0) + 2); + copy = strdup(cur ? cur : ""); + if (preload == NULL || copy == NULL) { + perror("dtoctl: cannot build LD_PRELOAD"); + exit(EXIT_DTOCTL_FAILURE); } - setenv("LD_PRELOAD", preload, 1); + strcpy(preload, lib); + + for (entry = strtok_r(copy, ": ", &save); entry; + entry = strtok_r(NULL, ": ", &save)) { + if (is_libdto(entry)) { + real = realpath(entry, NULL); + if (real == NULL || strcmp(real, lib)) + fprintf(stderr, "dtoctl: dropping '%s' from LD_PRELOAD " + "in favor of '%s'\n", entry, lib); + free(real); + continue; + } + strcat(preload, ":"); + strcat(preload, entry); + } + + if (setenv("LD_PRELOAD", preload, 1)) { + perror("dtoctl: setenv"); + exit(EXIT_DTOCTL_FAILURE); + } + free(copy); free(preload); } +/* Print s, single-quoted if a shell would otherwise split or expand it. */ +static void print_quoted(const char *s) +{ + static const char safe[] = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_@%+=:,./-"; + + if (*s && strspn(s, safe) == strlen(s)) { + fputs(s, stdout); + return; + } + + putchar('\''); + for (; *s; s++) { + if (*s == '\'') + fputs("'\\''", stdout); + else + putchar(*s); + } + putchar('\''); +} + +static void print_var(const char *name, int name_len, const char *value, + const char *source) +{ + printf("%.*s=", name_len, name); + print_quoted(value); + if (source) + printf(" # %s", source); + putchar('\n'); +} + +/* --dry-run: show the library, the environment and the command to be run */ +static void print_dry_run(const char *lib, const char *source, char **argv) +{ + char **env; + int len; + + printf("library: %s (%s)\n", lib, source); + print_var("LD_PRELOAD", strlen("LD_PRELOAD"), getenv("LD_PRELOAD"), NULL); + + /* variables from options first, in option order, then inherited ones */ + for (int i = 0; i < nr_option_vars; i++) + print_var(option_vars[i], strlen(option_vars[i]), + getenv(option_vars[i]), "option"); + for (env = environ; *env; env++) { + len = strcspn(*env, "="); + if (!strncmp(*env, "DTO_", 4) && (*env)[len] == '=' && + !is_option_var(*env, len)) + print_var(*env, len, *env + len + 1, "inherited"); + } + + if (argv == NULL) + return; + fputs("command:", stdout); + for (int i = 0; argv[i]; i++) { + putchar(' '); + print_quoted(argv[i]); + } + putchar('\n'); +} + int main(int argc, char *argv[]) { + struct opts opts = {0}; struct argp argp = { .options = dtoctl_options, .parser = parse_option, .args_doc = " [...]", - .doc = "dtoctl -- run a program with DSA Transparent Offload (DTO) preloaded", + .doc = "dtoctl -- run a program with DSA Transparent Offload (DTO) preloaded" + "\vOnly the options given are applied: inherited DTO_* variables " + "are kept unless an option overrides them or -i is given. " + "Exit status is the program's own, or 125 if dtoctl fails, " + "126 if the program cannot be run, 127 if it is not found.", }; + const char *source; + char *lib; + int err; argp_parse(&argp, argc, argv, ARGP_IN_ORDER, NULL, &opts); - /* pass only the target program and its arguments to execvp() */ - argc -= opts.idx; - argv += opts.idx; + /* clear inherited variables first; this also drops DTO_LIBRARY */ + if (opts.ignore_env) + clear_dto_environ(); + setup_dto_environ(&opts); - setup_child_environ(&opts); + lib = resolve_library(&opts, &source); + setup_preload(lib); - execvp(opts.exename, argv); - perror(opts.exename); + if (opts.dry_run) { + print_dry_run(lib, source, opts.exename ? argv + opts.idx : NULL); + return 0; + } + + /* pass only the target program and its arguments to execvp() */ + execvp(opts.exename, argv + opts.idx); + err = errno; + fprintf(stderr, "dtoctl: cannot run '%s': %s\n", opts.exename, strerror(err)); - return -1; + return err == ENOENT ? EXIT_ENOENT : EXIT_CANNOT_INVOKE; } From 8ed0204aafdd6eca48f9c83a57ffb1a3cf87c9e7 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 21 Oct 2026 07:21:12 -0700 Subject: [PATCH 6/8] dtoctl: follow the install prefix, report the version, add tests CMake cached the default library path on the first configure, so a later CMAKE_INSTALL_PREFIX change was ignored, and "cmake --install --prefix" produced a dtoctl that preloaded /usr/local/lib/libdto.so. Compute the default on every configure, pass dtoctl the library's path relative to its own bindir so a relocated install finds its own copy, and keep DTOCTL_LIBPATH as an explicit override. Pass the project version to both builds for --version, and make the Makefile's install target build dtoctl first. Add tests/test_dtoctl.sh as the "dtoctl" CTest (label functional). It needs no DSA: it checks the option to variable mapping, validation, library resolution, LD_PRELOAD handling and exit statuses through --dry-run and the program's environment, and that libdto applies the settings through its own configuration trace. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Mv7xKtQPkGoXQUvvuuhAEz --- CMakeLists.txt | 35 ++++++++-- Makefile | 6 +- tests/test_dtoctl.sh | 152 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 8 deletions(-) create mode 100755 tests/test_dtoctl.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a334d2..e12bcf9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,13 +100,27 @@ if (HAS_WAITPKG) target_compile_options(dto PRIVATE -mwaitpkg -march=native) endif() -# Build dtoctl with a default preload path matching the library installation. -get_filename_component(DTOCTL_LIBDIR "${CMAKE_INSTALL_LIBDIR}" ABSOLUTE - BASE_DIR "${CMAKE_INSTALL_PREFIX}") -set(DTOCTL_LIBPATH "${DTOCTL_LIBDIR}/libdto.so" CACHE FILEPATH - "Path to libdto.so that dtoctl prepends to LD_PRELOAD") +# Build dtoctl. By default it preloads the libdto.so installed alongside it, +# found relative to the dtoctl binary so "cmake --install --prefix" still works, +# and falls back to the configured install path. Set DTOCTL_LIBPATH to always +# preload a specific library instead. +set(DTOCTL_LIBPATH "" CACHE FILEPATH + "Path to libdto.so that dtoctl preloads by default (empty: the installed libdto.so)") add_executable(dtoctl dtoctl.c) -target_compile_definitions(dtoctl PRIVATE LIBDTO_PATH="${DTOCTL_LIBPATH}") +target_compile_definitions(dtoctl PRIVATE DTO_VERSION="${PROJECT_VERSION}") +if(DTOCTL_LIBPATH) + target_compile_definitions(dtoctl PRIVATE LIBDTO_PATH="${DTOCTL_LIBPATH}") +else() + # Not cached, so it follows later changes to CMAKE_INSTALL_PREFIX. + get_filename_component(DTOCTL_LIBDIR "${CMAKE_INSTALL_LIBDIR}" ABSOLUTE + BASE_DIR "${CMAKE_INSTALL_PREFIX}") + get_filename_component(DTOCTL_BINDIR "${CMAKE_INSTALL_BINDIR}" ABSOLUTE + BASE_DIR "${CMAKE_INSTALL_PREFIX}") + file(RELATIVE_PATH DTOCTL_REL_LIBPATH "${DTOCTL_BINDIR}" "${DTOCTL_LIBDIR}/libdto.so") + target_compile_definitions(dtoctl PRIVATE + LIBDTO_PATH="${DTOCTL_LIBDIR}/libdto.so" + LIBDTO_REL_PATH="${DTOCTL_REL_LIBPATH}") +endif() install(TARGETS dtoctl RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) # Build and install the optional section 1 manual. @@ -150,6 +164,15 @@ if(DTO_BUILD_TESTS) TIMEOUT 120 ) + # dtoctl option handling and environment setup (no DSA hardware needed) + add_test(NAME dtoctl + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_dtoctl.sh + $ $) + set_tests_properties(dtoctl PROPERTIES + LABELS "functional" + TIMEOUT 60 + ) + # ---- A/B performance regression test ---- # The baseline object is compiled from unguarded upstream sources that # always reference accel-config and numa, so the perf test can only be diff --git a/Makefile b/Makefile index 8b4b2cf..849efdf 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,8 @@ DML_LIB_CXX=-D_GNU_SOURCE DTOCTL_LIBPATH ?= /usr/lib64/libdto.so PANDOC ?= pandoc MANDIR ?= /usr/share/man +# Version reported by "dtoctl --version", taken from the CMake project. +DTO_VERSION := $(shell grep -o 'DTO VERSION [0-9.]*' CMakeLists.txt | cut -d' ' -f3) libdto: dto.c gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_STATS_SUPPORT -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg @@ -20,7 +22,7 @@ libdto_nostats: dto.c gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg dtoctl: dtoctl.c - gcc -O2 -Wall dtoctl.c -DLIBDTO_PATH=\"$(DTOCTL_LIBPATH)\" -o dtoctl + gcc -O2 -Wall dtoctl.c -DLIBDTO_PATH=\"$(DTOCTL_LIBPATH)\" -DDTO_VERSION=\"$(DTO_VERSION)\" -o dtoctl man: doc/dtoctl.1 @@ -31,7 +33,7 @@ doc/dtoctl.1: doc/dtoctl.md echo "Skipping dtoctl manual generation: install pandoc to build the man page."; \ fi -install: install-man +install: dtoctl install-man cp libdto.so.1.0 /usr/lib64/ ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so.1 ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so diff --git a/tests/test_dtoctl.sh b/tests/test_dtoctl.sh new file mode 100755 index 0000000..8cf4cf0 --- /dev/null +++ b/tests/test_dtoctl.sh @@ -0,0 +1,152 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# +# Tests for dtoctl's option handling, library resolution and environment setup. +# No DSA hardware is needed: most checks read dtoctl's --dry-run output or the +# environment the program receives, and the end-to-end check reads libdto's own +# configuration trace, which libdto prints even when it falls back to the CPU. +# +# Usage: test_dtoctl.sh + +DTOCTL="$1" +LIB=$(realpath "$2") +LIBDIR=$(dirname "$LIB") +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT +failures=0 + +# Start from a clean environment so the caller's DTO setup cannot leak in. +unset LD_PRELOAD +for var in $(env | sed -n 's/^\(DTO_[A-Za-z0-9_]*\)=.*/\1/p'); do + unset "$var" +done + +# run : capture the command's output and exit status +run() { + out=$("$@" 2>&1) + rc=$? + cmd="$*" +} + +fail() { + echo "FAIL: $1" + echo " command: $cmd" + printf '%s\n' "$out" | sed 's/^/ | /' + failures=$((failures + 1)) +} + +# has : the last output has a line matching pattern +has() { + if printf '%s\n' "$out" | grep -q -- "$2"; then echo "PASS: $1"; else fail "$1"; fi +} + +# lacks : no line of the last output matches pattern +lacks() { + if printf '%s\n' "$out" | grep -q -- "$2"; then fail "$1"; else echo "PASS: $1"; fi +} + +# status : the last command exited with code +status() { + if [ "$rc" -eq "$2" ]; then echo "PASS: $1"; else fail "$1 (exit $rc, expected $2)"; fi +} + +# --- every option sets its variable --- +run "$DTOCTL" -l "$LIB" --dry-run -w umwait -b 64K -c 0.25 -n 2 -q 'wq0.0;wq2.0' \ + --umwait-delay 5000 --overlapping-memmove=dsa --no-memcpy --no-memmove \ + --no-memset --no-memcmp --no-cache-control --no-auto-adjust --stdc-only \ + --log-file=/tmp/dto.log --log-level 1 --stats-file=/tmp/dto.stats prog 'a b' +status "dry-run succeeds" 0 +for v in DTO_WAIT_METHOD=umwait DTO_MIN_BYTES=65536 DTO_CPU_SIZE_FRACTION=0.25 \ + DTO_IS_NUMA_AWARE=2 "DTO_WQ_LIST='wq0.0;wq2.0'" DTO_UMWAIT_DELAY=5000 \ + DTO_OVERLAPPING_MEMMOVE_ACTION=1 DTO_DSA_MEMCPY=0 DTO_DSA_MEMMOVE=0 \ + DTO_DSA_MEMSET=0 DTO_DSA_MEMCMP=0 DTO_DSA_CC=0 DTO_AUTO_ADJUST_KNOBS=0 \ + DTO_USESTDC_CALLS=1 DTO_LOG_FILE=/tmp/dto.log DTO_LOG_LEVEL=1 \ + DTO_STATS_FILE=/tmp/dto.stats DTO_COLLECT_STATS=1; do + has "option sets $v" "^$v # option\$" +done +has "dry-run shows the library" "^library: $LIB (--library)\$" +has "dry-run shows LD_PRELOAD" "^LD_PRELOAD=$LIB\$" +has "dry-run shows the command" "^command: prog 'a b'\$" + +# --- size suffixes --- +for s in 4096:4096 4k:4096 1M:1048576 2G:2147483648; do + run "$DTOCTL" -l "$LIB" --dry-run -b "${s%%:*}" true + has "-b ${s%%:*} is ${s##*:} bytes" "^DTO_MIN_BYTES=${s##*:} # option\$" +done + +# --- inherited variables --- +run env DTO_WAIT_METHOD=umwait DTO_MIN_BYTES=4096 "$DTOCTL" -l "$LIB" --dry-run -w busypoll true +has "option overrides an inherited variable" "^DTO_WAIT_METHOD=busypoll # option\$" +has "inherited variable is kept" "^DTO_MIN_BYTES=4096 # inherited\$" + +run env DTO_MIN_BYTES=4096 DTO_USESTDC_CALLS=1 "$DTOCTL" -i -l "$LIB" --dry-run true +lacks "-i drops inherited variables" "^DTO_" + +run env DTO_LIBRARY="$LIB" "$DTOCTL" -i --dry-run true +lacks "-i ignores DTO_LIBRARY" "(DTO_LIBRARY)" + +# --- the program receives the environment --- +run "$DTOCTL" -l "$LIB" --stdc-only -w yield env +has "program sees DTO_WAIT_METHOD" "^DTO_WAIT_METHOD=yield\$" +has "program sees LD_PRELOAD" "^LD_PRELOAD=$LIB\$" + +run "$DTOCTL" -l "$LIB" --stdc-only echo -w -l +has "options after the program are the program's" "^-w -l\$" + +# --- library resolution and LD_PRELOAD --- +run env DTO_LIBRARY="$LIB" "$DTOCTL" --dry-run true +has "DTO_LIBRARY selects the library" "^library: $LIB (DTO_LIBRARY)\$" + +run sh -c 'cd "$1" && shift && exec "$@"' sh "$LIBDIR" \ + "$DTOCTL" -l "./$(basename "$LIB")" --dry-run true +has "relative library path becomes absolute" "^LD_PRELOAD=$LIB\$" + +run "$DTOCTL" -l "$TMP/missing/libdto.so" true +status "missing library fails" 125 +has "missing library is reported" "cannot use DTO library" + +run env LD_PRELOAD="$TMP/libother.so $TMP/old/libdto.so.1" "$DTOCTL" -l "$LIB" --dry-run true +has "existing LD_PRELOAD entries are kept after libdto" "^LD_PRELOAD=$LIB:$TMP/libother.so\$" +has "another libdto in LD_PRELOAD is dropped" "dropping '$TMP/old/libdto.so.1'" + +# --- invalid values are rejected --- +for bad in "-w bogus" "-b 64Q" "-b abc" "-b -1" "-c 1.5" "-c nan" "-n 3" \ + "--log-level x" "--umwait-delay 0" "--overlapping-memmove=gpu"; do + run "$DTOCTL" -l "$LIB" $bad true + status "rejects $bad" 64 +done +run "$DTOCTL" -l "$LIB" -q '' true +status "rejects an empty WQ list" 64 + +# --- exit status --- +run "$DTOCTL" -l "$LIB" +status "no program is a usage error" 64 + +run "$DTOCTL" -l "$LIB" --dry-run +status "--dry-run needs no program" 0 +lacks "--dry-run without a program prints no command" "^command:" + +run "$DTOCTL" -l "$LIB" "$TMP/no-such-program" +status "program not found" 127 + +touch "$TMP/not-executable" +run "$DTOCTL" -l "$LIB" "$TMP/not-executable" +status "program not executable" 126 + +run "$DTOCTL" -l "$LIB" --stdc-only sh -c 'exit 7' +status "program's exit status is returned" 7 + +run "$DTOCTL" --version +has "--version" "^dtoctl (DTO) " + +# --- libdto applies the settings (DSA is optional) --- +run "$DTOCTL" -l "$LIB" --log-level 2 -w yield -b 8K -c 0.5 --no-auto-adjust true +has "libdto applies the options" \ + "dsa_min_size: 8192, cpu_size_fraction: 0.50, wait_method: yield, auto_adjust_knobs: 0" + +echo +if [ "$failures" -ne 0 ]; then + echo "$failures dtoctl test(s) failed" + exit 1 +fi +echo "All dtoctl tests passed" From dd3bcf6dc56aa141f461b50dca72ddde67452e4a Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 21 Oct 2026 07:22:27 -0700 Subject: [PATCH 7/8] docs: document dtoctl's validation, library resolution and --dry-run Update the dtoctl man page and README section for the new behavior: option validation, -i/--ignore-env, --dry-run, --version, size suffixes, --stats-file implying --stats, how the library is resolved, and the exit statuses. Add a note on sudo, which resets the environment and strips LD_PRELOAD when it sits between dtoctl and the program, and on settings being inherited by child processes. Fix stale defaults in the README's variable list: DTO_WAIT_METHOD defaults to busypoll and DTO_MIN_BYTES to 65536, tpause is a valid wait method, and DTO_LOG_FILE gets .. appended. Describe the dtoctl test in TESTING.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Mv7xKtQPkGoXQUvvuuhAEz --- README.md | 39 +++++++++++--- TESTING.md | 14 +++++ doc/dtoctl.md | 139 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 159 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 4175970..bcfbffd 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,8 @@ Following environment variables control the behavior of DTO library: Should be enabled for debugging/profiling only, not for perf evaluation (enabling it slows down the workload). Default is 0. DTO_STATS_FILE= when DTO_COLLECT_STATS=1, redirect histogram output to the provided regular file instead of standard output. The file is truncated when DTO loads; failures fall back to standard output and emit an error. - DTO_WAIT_METHOD= (specifies the method to use while waiting for DSA to complete operation, default is yield) - DTO_MIN_BYTES=xxxx (specifies minimum size of API call needed for DSA operation execution, default is 16384 bytes) + DTO_WAIT_METHOD= (specifies the method to use while waiting for DSA to complete operation, default is busypoll) + DTO_MIN_BYTES=xxxx (specifies minimum size of API call needed for DSA operation execution, default is 65536 bytes) DTO_CPU_SIZE_FRACTION=0.xx (specifies fraction of job performed by CPU, in parallel to DSA). Default is 0.00 DTO_AUTO_ADJUST_KNOBS=0/1 (disables/enables auto tuning of cpu_size_fraction and dsa_min_bytes parameters. 0 -- disable, 1 -- enable (default)) DTO_IS_NUMA_AWARE=0/1/2 (disables/buffer-centric/cpu-centric numa awareness. 0 -- disable (default), 1 -- buffer-centric, 2 - cpu-centric) @@ -69,7 +69,7 @@ Following environment variables control the behavior of DTO library: DTO_DSA_CC=0/1, 1 (default) - DTO sets DSA Cache Control flag to 1 if DSA supports cache control, 0 - DTO sets DSA Cache Control flag to 0 DTO_OVERLAPPING_MEMMOVE_ACTION=0/1 0 (default) DTO submits memmove operations with overlapping buffers entirely to CPU, 1 - entirely to DSA DTO_UMWAIT_DELAY=xxxx defines delay for umwait command (check max possible value at: /sys/devices/system/cpu/umwait_control/max_time), default is 100000 - DTO_LOG_FILE= Redirect the DTO output to the specified file instead of std output (useful for debugging and statistics collection). file name is suffixed by process pid. + DTO_LOG_FILE= Redirect the DTO output to the specified file instead of std output (useful for debugging and statistics collection). file name is suffixed by the program name and process pid (..). DTO_LOG_LEVEL=0/1/2 controls the log level. higher value means more verbose logging (default 0). ``` @@ -117,12 +117,35 @@ you can run dtoctl -w busypoll -c 0.33 ./prog ``` -Every environment variable listed above has a corresponding option, and only the -options you pass are applied: a `DTO_*` variable you already exported is left -untouched, and an explicit option overrides it. The `libdto.so` path is resolved -from `-l/--library`, then `$DTO_LIBRARY`, then the compiled-in default +Every environment variable listed above has a corresponding option, and option +values are checked before the program runs (DTO itself silently falls back to a +default on a value it does not understand). Only the options you pass are +applied: a `DTO_*` variable you already exported is left untouched, an explicit +option overrides it, and `-i/--ignore-env` drops all inherited `DTO_*` +variables. + +The `libdto.so` path is resolved from `-l/--library`, then `$DTO_LIBRARY`, then +the library installed with `dtoctl`, then the compiled-in default (`/usr/local/lib/libdto.so` for a CMake build, `/usr/lib64/libdto.so` for the -Makefile build), and is prepended to any existing `LD_PRELOAD`. +Makefile build). It is made absolute and prepended to any existing +`LD_PRELOAD`, and `dtoctl` fails if the library cannot be read rather than +running the program without DTO. + +To see what would run without running it, use `--dry-run`: + +```bash +$ dtoctl --dry-run -w umwait -b 16K ./prog +library: /usr/lib64/libdto.so.1.0 (compiled-in default) +LD_PRELOAD=/usr/lib64/libdto.so.1.0 +DTO_WAIT_METHOD=umwait # option +DTO_MIN_BYTES=16384 # option +command: ./prog +``` + +DSA work queues are often accessible to root only. Run `sudo dtoctl ...` +rather than `dtoctl sudo ...`, which loses `LD_PRELOAD` and the `DTO_*` +variables, and pass settings as options: `sudo` does not pass on variables +exported in your shell. Run `dtoctl --help` for the option list, or `man 1 dtoctl` for the full manual, which documents each option and gives a ready-to-run command for each of the diff --git a/TESTING.md b/TESTING.md index a491d31..2cbe87c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -59,6 +59,20 @@ hardware -- DTO falls back to CPU transparently. ctest -R functional ``` +## dtoctl Tests + +Checks for the `dtoctl` wrapper: every option sets its `DTO_*` variable, +invalid values are rejected, the library path is resolved and validated, +`LD_PRELOAD` and inherited variables are handled, and exit statuses follow +`env(1)`. The last check runs a program under `dtoctl` and reads libdto's own +configuration trace. No DSA hardware is needed. + +```bash +ctest -R dtoctl +# or directly, from the source directory +tests/test_dtoctl.sh build/dtoctl build/libdto.so +``` + ## Performance Tests The performance test (`perf`) detects regressions in libdto by diff --git a/doc/dtoctl.md b/doc/dtoctl.md index af9f5e7..afca4ed 100644 --- a/doc/dtoctl.md +++ b/doc/dtoctl.md @@ -11,6 +11,8 @@ SYNOPSIS ======== dtoctl [_options_] COMMAND [_command-options_] +dtoctl \--dry-run [_options_] [COMMAND [_command-options_]] + DESCRIPTION =========== @@ -21,14 +23,23 @@ Accelerator). Without **dtoctl**, users must set **LD_PRELOAD** to point at **libdto.so** and export a set of **DTO_\*** environment variables to control DTO behavior. -**dtoctl** replaces that manual setup: it converts its command line options into -the corresponding **DTO_\*** environment variables, prepends **libdto.so** to -**LD_PRELOAD**, and then executes `COMMAND`. +**dtoctl** replaces that manual setup: it checks its command line options, +converts them into the corresponding **DTO_\*** environment variables, prepends +**libdto.so** to **LD_PRELOAD**, and then executes `COMMAND`. Nothing is left +behind in the calling shell. Only the options that are explicitly given are applied. Options that are not given leave any pre-existing **DTO_\*** environment variables (or the DTO built-in defaults) untouched, and an explicitly given option overrides a -previously exported variable of the same name. +previously exported variable of the same name. Use `-i` to ignore the +inherited **DTO_\*** variables instead. + +Option values are validated before `COMMAND` runs, and an invalid value is an +error. DTO itself silently falls back to a default when a variable holds a +value it does not understand. + +`COMMAND` and every process it starts inherit the settings, so +`dtoctl make` runs each compiler under DTO as well. LIBRARY RESOLUTION @@ -36,49 +47,80 @@ LIBRARY RESOLUTION The path of **libdto.so** to preload is resolved in the following order: 1. The `-l`, `--library` option, if given. -2. The **DTO_LIBRARY** environment variable, if set. -3. The compiled-in default path (`/usr/local/lib/libdto.so` for a default +2. The **DTO_LIBRARY** environment variable, if set and not empty. This + variable is read by **dtoctl** only, not by **libdto.so**. +3. For a CMake build, the **libdto.so** installed with **dtoctl**, found + relative to the **dtoctl** executable (`/../lib/libdto.so`), so + that an installation moved with `cmake --install --prefix` finds its own + library. +4. The compiled-in default path (`/usr/local/lib/libdto.so` for a default CMake build, or `/usr/lib64/libdto.so` for the Makefile build). -The resolved path is prepended to any existing **LD_PRELOAD** value. +The resolved path is made absolute, so it keeps working when `COMMAND` changes +directory, and **dtoctl** fails with exit status 125 if it cannot read the +library. (**ld.so** would only print a warning and run `COMMAND` without DTO.) +Run `dtoctl --dry-run` to see which library was chosen and why. + +The library is prepended to any existing **LD_PRELOAD** value. Any other +**libdto.so** entry already in **LD_PRELOAD** is removed, with a warning when it +names a different file, so that one process never loads two copies of DTO. OPTIONS ======= -Library +General ------- -l _PATH_, \--library=_PATH_ : Path to **libdto.so** to prepend to **LD_PRELOAD**. Overrides - **DTO_LIBRARY** and the compiled-in default. - Default: the compiled-in library path (see LIBRARY RESOLUTION). + **DTO_LIBRARY** and the installed library (see LIBRARY RESOLUTION). + +-i, \--ignore-env +: Unset every **DTO_\*** variable inherited from the environment, including + **DTO_LIBRARY**, so that only the options given on this command line and + the DTO built-in defaults apply. This is the way to undo an exported + variable that the `--no-*` options cannot, such as + **DTO_USESTDC_CALLS=1** or **DTO_DSA_MEMCPY=0**. + +\--dry-run +: Print the resolved library and where it came from, the resulting + **LD_PRELOAD**, every **DTO_\*** variable that `COMMAND` would see (each + marked `option` or `inherited`), and the command line, then exit without + running anything. `COMMAND` may be omitted. Offload tuning -------------- -w _METHOD_, \--wait-method=_METHOD_ : How to wait for DSA completion: _yield_, _busypoll_, _umwait_, or _tpause_. - Sets **DTO_WAIT_METHOD**. Default: _busypoll_. + _umwait_ and _tpause_ need a CPU with WAITPKG support; DTO falls back + otherwise. Sets **DTO_WAIT_METHOD**. Default: _busypoll_. -b _BYTES_, \--min-bytes=_BYTES_ : Minimum operation size offloaded to DSA. Smaller operations run on the CPU. - Sets **DTO_MIN_BYTES**. Default: 65536. + _BYTES_ may have a _K_, _M_, or _G_ suffix (powers of 1024), and is passed to + DTO as a plain byte count. With auto tuning on, this is the starting + value. Sets **DTO_MIN_BYTES**. Default: 64K. -c _FRACTION_, \--cpu-fraction=_FRACTION_ : Fraction (0.0 <= f < 1.0) of each operation performed on the CPU in parallel - to DSA. Sets **DTO_CPU_SIZE_FRACTION**. Default: 0.0. + to DSA. DTO uses two decimal places. With auto tuning on, this is the + starting value. Sets **DTO_CPU_SIZE_FRACTION**. Default: 0.0. -n _MODE_, \--numa-aware=_MODE_ -: NUMA awareness: _0_ disable, _1_ buffer-centric, _2_ cpu-centric. - Sets **DTO_IS_NUMA_AWARE**. Default: _0_. +: NUMA awareness: _0_ disable, _1_ buffer-centric, _2_ cpu-centric. Has no + effect if DTO was built without NUMA support. Sets **DTO_IS_NUMA_AWARE**. + Default: _0_. -q _LIST_, \--wq-list=_LIST_ : Semicolon-separated list of DSA work queues to use, e.g. "wq0.0;wq2.0". - Names must match those under /dev/dsa/. Sets **DTO_WQ_LIST**. - Default: auto-discover all available WQs. + Names must match those under /dev/dsa/, and the list can be at most 255 + characters. Required if DTO was built without accel-config support. + Sets **DTO_WQ_LIST**. Default: auto-discover all available WQs. \--umwait-delay=_CYCLES_ -: Delay in cycles for the umwait instruction. Sets **DTO_UMWAIT_DELAY**. - Default: 100000. +: Delay in cycles for the umwait instruction, a positive number. The + maximum is in /sys/devices/system/cpu/umwait_control/max_time. Sets + **DTO_UMWAIT_DELAY**. Default: 100000. \--overlapping-memmove=_WHERE_ : Where to run memmove with overlapping buffers: _cpu_ or _dsa_. @@ -122,11 +164,12 @@ Statistics and logging workload. Sets **DTO_COLLECT_STATS=1**. Default: off. \--stats-file=_PATH_ -: Write the stats histogram to _PATH_ instead of standard output. - Sets **DTO_STATS_FILE**. Default: standard output. +: Write the stats histogram to _PATH_ instead of standard output. Implies + `--stats`, since DTO ignores the file otherwise. Sets **DTO_STATS_FILE** + and **DTO_COLLECT_STATS=1**. Default: standard output. \--log-file=_PATH_ -: Redirect DTO output to _PATH_ (the file name is suffixed by the pid). +: Redirect DTO output to _PATH_._program_._pid_, one file per process. Sets **DTO_LOG_FILE**. Default: standard output. \--log-level=_LEVEL_ @@ -139,6 +182,42 @@ Statistics and logging \--usage : Print usage string. +-V, \--version +: Print the version. + + +EXIT STATUS +=========== +On success **dtoctl** does not return: `COMMAND` replaces it, so the exit +status is that of `COMMAND`. Otherwise: + +64 +: Invalid option or option value, or no `COMMAND`. + +125 +: **dtoctl** itself failed, for example because the library cannot be read. + +126 +: `COMMAND` was found but cannot be run. + +127 +: `COMMAND` was not found. + +`--dry-run`, `--help`, and `--version` exit with 0. + + +NOTES +===== +DSA work queues under /dev/dsa/ are often accessible to root only. Put +**sudo** in front of **dtoctl**, not between **dtoctl** and `COMMAND`: + + $ sudo dtoctl -w busypoll ./prog # works + $ dtoctl -w busypoll sudo ./prog # sudo drops LD_PRELOAD and DTO_* + +By default **sudo** also resets the environment, so **DTO_\*** variables +exported in the calling shell do not reach `sudo dtoctl`. Pass the settings as +**dtoctl** options instead. + EXAMPLES ======== @@ -171,11 +250,21 @@ Avoid cache pollution (offload everything, clear cache control): $ dtoctl -w yield -c 0.0 --no-auto-adjust --no-cache-control ./prog Use a locally built library and a specific set of work queues, and collect -stats: +stats into a file: + + $ dtoctl -l ./libdto.so.1.0 -q "wq0.0;wq2.0;wq4.0;wq6.0" \ + --stats-file=/tmp/dto-stats.txt ./prog + +Check what would run, ignoring anything exported in the shell: - $ dtoctl -l ./libdto.so.1.0 -q "wq0.0;wq2.0;wq4.0;wq6.0" --stats ./prog + $ dtoctl --dry-run -i -w umwait -b 16K ./prog + library: /usr/lib64/libdto.so.1.0 (compiled-in default) + LD_PRELOAD=/usr/lib64/libdto.so.1.0 + DTO_WAIT_METHOD=umwait # option + DTO_MIN_BYTES=16384 # option + command: ./prog SEE ALSO ======== -**ld.so**(8) +**ld.so**(8), **accel-config**(1) From 5ede05d53bb89a08117307f844f88b9d42b1c95f Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Tue, 27 Oct 2026 08:48:14 -0700 Subject: [PATCH 8/8] gitignore: ignore the libdto_explicit build output --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b9711b7..be47746 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ dto-test* libdto.so* +libdto_explicit.so* build*/ tests/baselines*/ dtoctl