[LXYan2333]

GCC can omit destination initialization for gathers with an all-ones mask. This leaves a false dependency on the previous destination value, limiting overlap between loop iterations when the gathered values feed further computation. See GCC Bugzilla 127946.

For GCC, use masked gather intrinsics with a zero merge source and an all-ones mask made opaque through empty inline assembly. This prevents GCC from removing destination initialization. The asm is nonvolatile and has no memory clobber; the intrinsic retains memory-access modeling.

Apply the workaround to 128-, 256-, and 512-bit integer and floating-point gathers. Preserve the original implementations for other compilers, and add a TODO to restrict the workaround once the fixed GCC version is known. Public APIs and explicitly masked gathers are unchanged.

On a Xeon Gold 6526Y unaffected by GDS, using GCC 14.3.1, a standalone gather-plus-eight-FMA microbenchmark shows median improvements of 2.1–5.0× at -O2 and 3.9–6.4× at -O3 across the tested widths, over five alternating baseline/patched runs. Gather-only performance is roughly unchanged.

benchmark source: (written by gpt astra, just demonstrate speedup, not part of this pr)

// Standalone benchmark for GCC PR 127946. Not part of the Highway build.
// Build with -O2 or -O3, -std=c++17, -I., build/libhwy.a, and -pthread.
// Compare the same source against baseline and patched Highway headers.
#include <stdint.h>
#include <stdio.h>

#include <cmath>
#include <limits>

#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "gather_benchmark.cc"
#include "hwy/foreach_target.h"  // IWYU pragma: keep

// Must come after foreach_target.h.
#include "hwy/aligned_allocator.h"
#include "hwy/highway.h"
#include "hwy/nanobenchmark.h"

HWY_BEFORE_NAMESPACE();
namespace hwy {
namespace HWY_NAMESPACE {
namespace {

// Measures durations, verifies results, prints timings.
template <class Benchmark>
void RunBenchmark(const char* caption) {
  printf("%10s: ", caption);
  const size_t kNumInputs = 1;
  const size_t num_items = Benchmark::NumItems() * size_t(Unpredictable1());
  const FuncInput inputs[kNumInputs] = {num_items};
  Result results[kNumInputs];

  Benchmark benchmark;

  Params p;
  p.verbose = false;
  p.max_evals = 7;
  p.target_rel_mad = 0.002;
  const size_t num_results = MeasureClosure(
      [&benchmark](const FuncInput input) { return benchmark(input); }, inputs,
      kNumInputs, results, p);
  if (num_results != kNumInputs) {
    printf("\n");
    HWY_WARN("MeasureClosure failed.\n");
    return;
  }

  benchmark.Verify(num_items);

  for (size_t i = 0; i < num_results; ++i) {
    const double cycles_per_item =
        results[i].ticks / static_cast<double>(results[i].input);
    const double mad = results[i].variability * cycles_per_item;
    printf("%6d: %6.3f (+/- %5.3f)\n", static_cast<int>(results[i].input),
           cycles_per_item, mad);
  }
}

// A chain of dependent FMAs makes a false gather-destination dependency carry
// over to the next iteration. Also measure gather alone to show workaround
// cost.
template <class D, int kFmaCount>
class BenchmarkGather {
  using T = TFromD<D>;
  using TI = MakeSigned<T>;

 public:
  static size_t NumItems() { return 1024; }

  BenchmarkGather()
      : table_(AllocateAligned<T>(NumItems())),
        indices_(AllocateAligned<TI>(NumItems())),
        output_(AllocateAligned<T>(NumItems())) {
    HWY_ASSERT(table_ && indices_ && output_);
    for (size_t i = 0; i < NumItems(); ++i) {
      table_[i] = static_cast<T>(i) * T{0.25} + T{1};
      indices_[i] = static_cast<TI>((i * 33 + 17) % NumItems());
    }
  }

  HWY_NOINLINE FuncOutput operator()(size_t num_items) {
    const D d;
    const RebindToSigned<D> di;
    const auto a = Set(d, static_cast<T>(0.123));
    const auto b = Set(d, static_cast<T>(0.456));
    const T* HWY_RESTRICT table = table_.get();
    const TI* HWY_RESTRICT indices = indices_.get();
    T* HWY_RESTRICT output = output_.get();
    for (size_t i = 0; i < num_items; i += Lanes(d)) {
      auto result = GatherIndex(d, table, Load(di, indices + i));
      for (int j = 0; j < kFmaCount; ++j) {
        result = MulAdd(result, a, b);
      }
      Store(result, d, output + i);
    }
    return BitCastScalar<MakeUnsigned<T>>(output[num_items - 1]);
  }

  void Verify(size_t num_items) {
    for (size_t i = 0; i < num_items; ++i) {
      T expected = table_[static_cast<size_t>(indices_[i])];
      for (int j = 0; j < kFmaCount; ++j) {
        expected =
            std::fma(expected, static_cast<T>(0.123), static_cast<T>(0.456));
      }
      const T tolerance = T{16} * std::numeric_limits<T>::epsilon() *
                          (T{1} + std::abs(expected));
      if (!(std::abs(output_[i] - expected) <= tolerance)) {
        HWY_ABORT("Gather: expected %e, actual %e at %zu\n",
                  static_cast<double>(expected),
                  static_cast<double>(output_[i]), i);
      }
    }
  }

 private:
  AlignedPtr<T> table_;
  AlignedPtr<TI> indices_;
  AlignedPtr<T> output_;
};

template <class T, size_t kLanes>
void RunGatherBenchmarks() {
  using D = FixedTag<T, kLanes>;
  char caption[40];
  snprintf(caption, sizeof(caption), "gather f%zu x %zu", sizeof(T) * 8,
           kLanes);
  RunBenchmark<BenchmarkGather<D, 0>>(caption);
  snprintf(caption, sizeof(caption), "gather+8fma f%zu x %zu", sizeof(T) * 8,
           kLanes);
  RunBenchmark<BenchmarkGather<D, 8>>(caption);
}

void RunBenchmarks() {
  printf("------------------------ %s\n", TargetName(HWY_TARGET));
#if HWY_MIN_BYTES >= 16
  RunGatherBenchmarks<float, 4>();
#if HWY_HAVE_FLOAT64
  RunGatherBenchmarks<double, 2>();
#endif
#endif
#if HWY_CAP_GE256
  RunGatherBenchmarks<float, 8>();
#if HWY_HAVE_FLOAT64
  RunGatherBenchmarks<double, 4>();
#endif
#endif
#if HWY_CAP_GE512
  RunGatherBenchmarks<float, 16>();
#if HWY_HAVE_FLOAT64
  RunGatherBenchmarks<double, 8>();
#endif
#endif
}

}  // namespace
}  // namespace HWY_NAMESPACE
}  // namespace hwy
HWY_AFTER_NAMESPACE();

#if HWY_ONCE
namespace hwy {
HWY_EXPORT(RunBenchmarks);
void RunGatherBenchmark() { HWY_DYNAMIC_DISPATCH(RunBenchmarks)(); }
}  // namespace hwy

int main() {
  hwy::RunGatherBenchmark();
  return 0;
}
#endif

Benchmark results

Intel Xeon Gold 6526Y (not affected by GDS), GCC 14.3.1, Highway AVX3_SPR target.

Values are median timer ticks per element over five alternating baseline/patched runs, pinned to one CPU. Lower is better; speedup = baseline / patched.

Gather + eight dependent FMAs

Optimization	Type	Vector width	Baseline	Patched	Speedup
-O2	float	128 bits	5.922	1.830	3.24×
-O2	float	256 bits	2.965	1.271	2.33×
-O2	float	512 bits	1.520	0.720	2.11×
-O2	double	128 bits	11.854	3.621	3.27×
-O2	double	256 bits	5.916	1.807	3.27×
-O2	double	512 bits	3.061	0.607	5.04×
-O3	float	128 bits	5.920	0.972	6.09×
-O3	float	256 bits	2.965	0.492	6.03×
-O3	float	512 bits	1.523	0.395	3.86×
-O3	double	128 bits	11.855	1.863	6.36×
-O3	double	256 bits	5.920	0.973	6.08×
-O3	double	512 bits	3.051	0.598	5.10×

Gather-only control

Optimization	Type	Vector width	Baseline	Patched	Speedup
-O2	float	128 bits	0.549	0.544	1.01×
-O2	float	256 bits	0.380	0.380	1.00×
-O2	float	512 bits	0.413	0.406	1.02×
-O2	double	128 bits	0.735	0.741	0.99×
-O2	double	256 bits	0.551	0.549	1.00×
-O2	double	512 bits	0.374	0.374	1.00×
-O3	float	128 bits	0.548	0.549	1.00×
-O3	float	256 bits	0.379	0.380	1.00×
-O3	float	512 bits	0.409	0.385	1.06×
-O3	double	128 bits	0.740	0.741	1.00×
-O3	double	256 bits	0.550	0.550	1.00×
-O3	double	512 bits	0.374	0.374	1.00×

[jan-wassenberg]

Wow, that is quite a difference, thanks for pointing this out.

[Andarwinux]

Please add a macro to allow users to opt out this.

[LXYan2333]

Please add a macro to allow users to opt out this.

In my test, clang always clear destination register of gather, even in non-loop code. But I'll add the macro anyway.

[Andarwinux]

Please add a macro to allow users to opt out this.

In my test, clang always clear destination register of gather, even in non-loop code. But I'll add the macro anyway.

Please add a macro to allow users to opt out this.

In my test, clang always clear destination register of gather, even in non-loop code. But I'll add the macro anyway.

In highway, workarounds should typically have macros to allow them to be disabled.

for example, if this bug is fixed in future gcc, then even older highway can manually disable this workaround.