From 2761bd8fae73a8531ef818e383933ea4d26ea1ac Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 21:36:29 +0000 Subject: [PATCH] Fix the four red jobs of the first full 6.0.0 run (38808518362) Three Windows toolchain installs failed on their new checksum check, one FFM test on the CI model: - OpenVINO (Windows): Invoke-WebRequest hands the published .sha256 over as byte[], and -split on a byte array stringifies the bytes in decimal, so the step compared against "53" (ASCII '5'). The file is fetched with curl.exe as text now, like the Linux job, and both hashes are printed. - Vulkan (Windows arm64): LunarG's checksum endpoint is sdk/sha/, not sdk/sha256/ (404). Fetched with curl.exe; a failed fetch names the measured hash so it can be pinned after a check on a trusted machine. - CUDA (Windows): the archive names were upstream's hand-written list (cuda_crt 13.4.59) while the hash came from the newest manifest, which names another cuda_crt. The manifest's relative_path now names the archive, the list holds components only, CUDA_REDIST_MANIFEST is pinned to 13.4.3 (the version the run printed). - llama-ffm: the draft model's SentencePiece tokenizer detokenizes "Hello" as " Hello" (the word-boundary marker); the round trip compares the trimmed text and the concatenated pieces. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01MjAMq3jnoveX5dcvLGhNQw --- .github/workflows/publish.yml | 75 +++++++++++-------- CLAUDE.md | 13 +++- TODO.md | 21 ++++-- .../ffm/FfmLlamaModelIntegrationTest.java | 11 ++- 4 files changed, 79 insertions(+), 41 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4da09afdf..c0dc64c01 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1904,7 +1904,7 @@ jobs: env: # The CUDA redist manifest (redistrib_.json) the archives are verified against; empty # takes the newest 13.4.x manifest the redist index lists (the run prints which). - CUDA_REDIST_MANIFEST: "" + CUDA_REDIST_MANIFEST: "13.4.3" SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -1930,24 +1930,27 @@ jobs: $ErrorActionPreference = "Stop" $cuda = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.4" $base = "https://developer.download.nvidia.com/compute/cuda/redist" + # Components only: the archive name (component version) and its SHA-256 both come from + # the redist manifest below, so they cannot disagree. A hand-written archive name checked + # against the newest manifest's hash did exactly that in run 38808518362 (cuda_crt 13.4.59 + # from upstream's list against the 13.4.3 manifest's cuda_crt). $parts = @( - "cuda_crt/cuda_crt-windows-x86_64-13.4.59", - "cuda_cudart/cuda_cudart-windows-x86_64-13.4.49", - "cuda_nvcc/cuda_nvcc-windows-x86_64-13.4.59", - "cuda_nvrtc/cuda_nvrtc-windows-x86_64-13.4.59", - "libcublas/libcublas-windows-x86_64-13.7.0.27", - "libnvvm/libnvvm-windows-x86_64-13.4.59", - "cuda_nvtx/cuda_nvtx-windows-x86_64-13.4.49", - "cuda_profiler_api/cuda_profiler_api-windows-x86_64-13.4.49", - "visual_studio_integration/visual_studio_integration-windows-x86_64-13.4.49", - "cccl/cccl-windows-x86_64-13.3.4.2.1" + "cuda_crt", + "cuda_cudart", + "cuda_nvcc", + "cuda_nvrtc", + "libcublas", + "libnvvm", + "cuda_nvtx", + "cuda_profiler_api", + "visual_studio_integration", + "cccl" ) New-Item -ItemType Directory -Force -Path $cuda | Out-Null # Every archive is checked against the SHA-256 NVIDIA publishes for it in the redist # manifest of this toolkit (redistrib_.json next to the archives): a truncated or - # swapped download fails here, not at link time. CUDA_REDIST_MANIFEST names the manifest - # version; it moves with the toolkit (TODO.md, delivery roadmap, item 3: the hashes are to - # be pinned into this file once a machine that can reach NVIDIA's host has read them). + # swapped download fails here, not at link time. CUDA_REDIST_MANIFEST pins the manifest + # version, and with it every component version (TODO.md, delivery roadmap, item 3). $manifestVersion = $env:CUDA_REDIST_MANIFEST if (-not $manifestVersion) { # Unpinned: the newest 13.4.x manifest the redist index lists (the index is NVIDIA's @@ -1960,13 +1963,14 @@ jobs: $manifestUrl = "$base/redistrib_$manifestVersion.json" Write-Host "Reading $manifestUrl" $manifest = Invoke-RestMethod -Uri $manifestUrl - foreach ($p in $parts) { - $dir, $name = $p.Split("/") - $url = "$base/$dir/windows-x86_64/$name-archive.zip" + foreach ($dir in $parts) { + $entry = $manifest.$dir.'windows-x86_64' + if (-not $entry -or -not $entry.relative_path -or -not $entry.sha256) { throw "the redist manifest $manifestUrl has no windows-x86_64 relative_path/sha256 for $dir" } + $url = "$base/$($entry.relative_path)" + $name = [System.IO.Path]::GetFileNameWithoutExtension($entry.relative_path) -replace '-archive$', '' $zip = "$env:RUNNER_TEMP\$name.zip" - $expected = $manifest.$dir.'windows-x86_64'.sha256 - if (-not $expected) { throw "the redist manifest $manifestUrl has no windows-x86_64 sha256 for $dir" } - Write-Host "Downloading $url (sha256 $expected)" + $expected = $entry.sha256 + Write-Host "Downloading $url ($dir $($manifest.$dir.version), sha256 $expected)" Invoke-WebRequest -Uri $url -OutFile $zip $actual = (Get-FileHash -Algorithm SHA256 $zip).Hash.ToLowerInvariant() if ($actual -ne $expected.ToLowerInvariant()) { throw "$name-archive.zip SHA-256 $actual does not match the manifest's $expected" } @@ -2415,14 +2419,21 @@ jobs: # Version and installer arguments as in upstream's release.yml (windows, vulkan arm64). run: | curl.exe -fSL -o "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" "https://sdk.lunarg.com/sdk/download/$env:VULKAN_VERSION/windows/vulkansdk-windows-X64-$env:VULKAN_VERSION.exe" - # Verified against the SHA-256 LunarG publishes for the installer (sdk/sha256/...; pinned - # into VULKAN_SDK_SHA256 once read, empty takes the published value). TODO.md, delivery - # roadmap, item 3. - $expected = $env:VULKAN_SDK_SHA256 - if (-not $expected) { $expected = ((Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/sha256/$env:VULKAN_VERSION/windows/vulkansdk-windows-X64-$env:VULKAN_VERSION.exe.txt" -UseBasicParsing).Content -split '\s+')[0] } - if (-not $expected) { throw "no SHA-256 for the Vulkan SDK installer (neither pinned nor published)" } + # Verified against the SHA-256 LunarG publishes for the installer (its download API serves + # it at sdk/sha///.txt -- the first run asked sdk/sha256/ and got a 404, + # run 38808518362; pinned into VULKAN_SDK_SHA256 once read, empty takes the published value). + # Fetched with curl.exe as text (Invoke-WebRequest's byte[] Content, see the OpenVINO job). + # TODO.md, delivery roadmap, item 3. $actual = (Get-FileHash -Algorithm SHA256 "$env:RUNNER_TEMP\VulkanSDK-Installer.exe").Hash.ToLowerInvariant() - if ($actual -ne $expected.ToLowerInvariant()) { throw "VulkanSDK-Installer.exe SHA-256 $actual does not match $expected" } + $expected = $env:VULKAN_SDK_SHA256 + if (-not $expected) { + $shaUrl = "https://sdk.lunarg.com/sdk/sha/$env:VULKAN_VERSION/windows/vulkansdk-windows-X64-$env:VULKAN_VERSION.exe.txt" + $published = curl.exe -fsSL $shaUrl + if ($LASTEXITCODE -ne 0 -or -not $published) { throw "no SHA-256 for the Vulkan SDK installer: VULKAN_SDK_SHA256 is empty and $shaUrl did not answer (the downloaded installer hashes to $actual -- verify it on a trusted machine and pin it)" } + $expected = (($published -join ' ') -split '\s+')[0] + } + Write-Host "VulkanSDK-Installer.exe SHA-256 actual=$actual expected=$expected" + if ($actual -ne $expected.ToLowerInvariant()) { throw "VulkanSDK-Installer.exe SHA-256 $actual does not match the published $expected" } & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install com.lunarg.vulkan.arm64 if (-not (Test-Path "C:\VulkanSDK\$env:VULKAN_VERSION")) { throw "Vulkan SDK $env:VULKAN_VERSION was not installed" } Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\$env:VULKAN_VERSION" @@ -2529,12 +2540,16 @@ jobs: Invoke-WebRequest -Uri $url -OutFile "$env:RUNNER_TEMP\openvino.zip" # Verified against the .sha256 the toolkit server publishes next to the archive (pinned # into OPENVINO_WINDOWS_SHA256 once read; empty takes the published value). TODO.md, - # delivery roadmap, item 3. + # delivery roadmap, item 3. The .sha256 is fetched with curl.exe as text, like the Linux + # job: Invoke-WebRequest hands an octet-stream response over as byte[], and -split on a + # byte array stringifies the bytes in decimal -- the first run compared against "53", + # the ASCII code of the hash's first character (run 38808518362). $expected = $env:OPENVINO_WINDOWS_SHA256 - if (-not $expected) { $expected = ((Invoke-WebRequest -Uri "$url.sha256" -UseBasicParsing).Content -split '\s+')[0] } + if (-not $expected) { $expected = ((curl.exe -fsSL "$url.sha256") -join ' ' -split '\s+')[0] } if (-not $expected) { throw "no SHA-256 for $url (neither pinned nor published)" } $actual = (Get-FileHash -Algorithm SHA256 "$env:RUNNER_TEMP\openvino.zip").Hash.ToLowerInvariant() - if ($actual -ne $expected.ToLowerInvariant()) { throw "openvino.zip SHA-256 $actual does not match $expected" } + Write-Host "openvino.zip SHA-256 actual=$actual expected=$expected" + if ($actual -ne $expected.ToLowerInvariant()) { throw "openvino.zip SHA-256 $actual does not match the published $expected" } Expand-Archive -Path "$env:RUNNER_TEMP\openvino.zip" -DestinationPath "C:\openvino" -Force # The archive extracts into a nested versioned folder; point OpenVINO_DIR at its runtime/cmake. $root = (Get-ChildItem "C:\openvino" -Directory | Select-Object -First 1).FullName diff --git a/CLAUDE.md b/CLAUDE.md index 50ac54baa..fc7032c72 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -555,10 +555,15 @@ To change the CUDA version, update the following places: [`docs/benchmarks/gpu-module-jars.md`](docs/benchmarks/gpu-module-jars.md)). Server applications run on exactly those GPUs, which is why this deviates from upstream, who ship to RTX owners and say in a comment that they rely on the default. On a bump, re-read ggml's list at the new tag and - **extend** ours rather than dropping it. **Copy that list from upstream's - `.github/actions/windows-setup-cuda/action.yml` at the pinned `GIT_TAG`** — the component versions - differ per component (cuBLAS and CCCL have their own numbering) and cannot be derived from the CUDA - version. (This replaced `Jimver/cuda-toolkit`, which never shipped 13.4.) + **extend** ours rather than dropping it. **The component set follows upstream's + `.github/actions/windows-setup-cuda/action.yml` at the pinned `GIT_TAG`, the component versions + follow NVIDIA's redist manifest** (`CUDA_REDIST_MANIFEST`, e.g. `13.4.3` = `redistrib_13.4.3.json`): + the step reads each component's `relative_path` and `sha256` from the manifest, so the archive it + downloads and the hash it checks cannot name two different versions. They did once -- the first + full run of 6.0.0 (38808518362) paired upstream's hand-written `cuda_crt-…-13.4.59` with the newest + manifest's hash for another `cuda_crt` and failed on the first archive. Component versions differ + per component (cuBLAS and CCCL have their own numbering), which is why the manifest and not the + CUDA version decides them. (This replaced `Jimver/cuda-toolkit`, which never shipped 13.4.) 3. **Only for a major bump:** the backend name `cuda13` — in `llama/CMakeLists.txt` (`JLLAMA_BACKEND`), `.github/natives.csv` (the two `cuda13-*` rows), the generated pom executions (`check-natives.py pom`), the build jobs' artifact names, `LlamaLoader.MODULE_BACKENDS` and the diff --git a/TODO.md b/TODO.md index 1a8e3331e..3a440c34c 100644 --- a/TODO.md +++ b/TODO.md @@ -63,12 +63,21 @@ sccache archives (Linux and Windows) against pinned SHA-256s; the CUDA redist ar NVIDIA's own manifest, the OpenVINO archives and the Vulkan arm64 installer against the checksum each vendor publishes next to the file.)* Still open, each with the reason: -- [ ] **Pin the vendor checksums into `publish.yml`** (`CUDA_REDIST_MANIFEST`, `OPENVINO_LINUX_SHA256`, - `OPENVINO_WINDOWS_SHA256`, `VULKAN_SDK_SHA256`): the steps verify against the vendor's published - value when the variable is empty and print it; the session that wrote them could not reach - `developer.download.nvidia.com`, `storage.openvinotoolkit.org` or `sdk.lunarg.com` (proxy 403), - so the first Publish run after this lands is where the values are read off the log and pinned. - Until then a swapped archive on the vendor host would pass; a truncated or wrong download does not. +- [ ] **Pin the vendor checksums into `publish.yml`** (`OPENVINO_LINUX_SHA256`, `OPENVINO_WINDOWS_SHA256`, + `VULKAN_SDK_SHA256`; `CUDA_REDIST_MANIFEST` is pinned to `13.4.3` since the first run printed + it): the steps verify against the vendor's published value when the variable is empty and print + both values in full; the session that wrote them could not reach `developer.download.nvidia.com`, + `storage.openvinotoolkit.org` or `sdk.lunarg.com` (proxy 403), so a Publish run is where the + values are read off the log and pinned. Until then a swapped archive on the vendor host would + pass; a truncated or wrong download does not. **What the first full run (38808518362) taught, + all three Windows steps red on their install:** `Invoke-WebRequest` returns an octet-stream + `.sha256` as `byte[]`, and `-split` on a byte array stringifies the bytes in decimal, so the + OpenVINO step compared against `53` (ASCII `'5'`, the hash's first character) -- the published + files are fetched with `curl.exe` as text now, like the Linux job; LunarG's checksum endpoint is + `sdk/sha///.txt`, not `sdk/sha256/` (404), and a failed fetch now names the + measured hash so it can be pinned after a check on a trusted machine instead of costing another + run; and the CUDA archive names come from the manifest (`relative_path`) instead of upstream's + hand-written list, whose `cuda_crt 13.4.59` was not the `cuda_crt` the newest manifest hashes. - [ ] **The oneAPI Windows installer has no checksum endpoint** Intel publishes next to the file (`registrationcenter-download.intel.com/.../intel-deep-learning-essentials-2025.3.3.18_offline.exe`); pinning needs one download of the 1.6 GB installer on a machine that can reach it, then diff --git a/llama-ffm/src/test/java/net/ladenthin/llama/ffm/FfmLlamaModelIntegrationTest.java b/llama-ffm/src/test/java/net/ladenthin/llama/ffm/FfmLlamaModelIntegrationTest.java index c1ce7a2d8..dacb66b50 100644 --- a/llama-ffm/src/test/java/net/ladenthin/llama/ffm/FfmLlamaModelIntegrationTest.java +++ b/llama-ffm/src/test/java/net/ladenthin/llama/ffm/FfmLlamaModelIntegrationTest.java @@ -126,10 +126,19 @@ void rawCompletionAndItsStream() throws IOException { void tokenizeAndDetokenizeRoundTrip() throws IOException { int[] tokens = model.tokenize("Hello", false); assertThat(tokens.length, greaterThan(0)); - assertEquals("Hello", model.detokenize(tokens)); + // A SentencePiece tokenizer (the CI draft model) renders the first piece's word-boundary + // marker as a leading space, " Hello"; a byte-level tokenizer (the GPT-2 test fixture) + // gives "Hello". The round trip is the text up to that boundary. + String detokenized = model.detokenize(tokens); + assertEquals("Hello", detokenized.trim()); JsonNode json = JSON.readTree(model.tokenizeJson("Hello", false, true)); assertEquals(tokens.length, json.path("tokens").size()); assertArrayEquals(tokens, JSON.convertValue(json.path("tokens").findValues("id"), int[].class)); + StringBuilder pieces = new StringBuilder(); + for (JsonNode piece : json.path("tokens").findValues("piece")) { + pieces.append(piece.asText()); + } + assertEquals(detokenized, pieces.toString()); assertTrue(JSON.readTree(model.detokenizeJson(tokens)).has("content")); }