From 52c12e025c2fdddcb01aa4a6fa6321664de08cde Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Sun, 1 Nov 2026 13:15:48 +0200 Subject: [PATCH 1/8] Add DXIL shader debug file support to debug-files upload Detect DXC shader artifacts before handing files to symbolic: - dxil: DXIL containers (DXBC magic); debug ID from the 32-hex file stem, code ID from the Aftermath hash in sentry-shader-manifest.tsv next to the files - dxilpdb: DXC shader PDBs (MSF magic plus ILDN part in the first 64 KiB), which symbolic cannot parse; debug ID from the file stem Add the two kinds as --type values, send an optional code_id in the assemble request, and tolerate object kinds unknown to symbolic in the assemble response. --- src/api/data_types/chunking/dif.rs | 10 ++ src/api/mod.rs | 20 ++- src/commands/debug_files/find.rs | 2 +- src/commands/debug_files/upload.rs | 2 + src/utils/chunks/types.rs | 15 +++ src/utils/dif.rs | 15 ++- src/utils/dif_upload/mod.rs | 180 +++++++++++++++++++++++++- src/utils/dif_upload/shader.rs | 200 +++++++++++++++++++++++++++++ 8 files changed, 439 insertions(+), 5 deletions(-) create mode 100644 src/utils/dif_upload/shader.rs diff --git a/src/api/data_types/chunking/dif.rs b/src/api/data_types/chunking/dif.rs index f8ddc20b24..2a32e51a40 100644 --- a/src/api/data_types/chunking/dif.rs +++ b/src/api/data_types/chunking/dif.rs @@ -14,6 +14,8 @@ pub struct ChunkedDifRequest<'a> { pub name: Cow<'a, str>, #[serde(skip_serializing_if = "Option::is_none")] pub debug_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub code_id: Option>, pub chunks: &'a [Digest], #[serde(skip)] hash: Digest, @@ -28,6 +30,7 @@ impl<'a> ChunkedDifRequest<'a> { chunks, hash, debug_id: None, + code_id: None, } } @@ -37,6 +40,13 @@ impl<'a> ChunkedDifRequest<'a> { self.debug_id = debug_id; self } + + /// Set the provided code_id on the request, or clear it if + /// `None` is passed. + pub fn with_code_id(mut self, code_id: Option>) -> Self { + self.code_id = code_id; + self + } } #[derive(Debug, Deserialize)] diff --git a/src/api/mod.rs b/src/api/mod.rs index 4fda4ae5b3..f02fae6511 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -36,7 +36,7 @@ use parking_lot::Mutex; use regex::{Captures, Regex}; use secrecy::{ExposeSecret as _, SecretString}; use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use sha1_smol::Digest; use symbolic::common::DebugId; use symbolic::debuginfo::ObjectKind; @@ -1821,13 +1821,29 @@ pub struct SentryCliRelease { #[derive(Debug, Deserialize, Default)] pub struct DebugInfoData { - #[serde(default, rename = "type")] + #[serde( + default, + rename = "type", + deserialize_with = "deserialize_lenient_object_kind" + )] pub kind: Option, #[serde(default)] #[expect(dead_code)] pub features: Vec, } +/// Deserializes an object kind, mapping values unknown to symbolic (such as +/// server-side shader kinds) to `None` instead of failing. +fn deserialize_lenient_object_kind<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let kind: Option = Option::deserialize(deserializer)?; + Ok(kind.and_then(|k| k.parse().ok())) +} + /// Debug information files as processed and stored on the server. /// Can be dSYMs, ELF debug infos, Breakpad symbols, etc... #[derive(Debug, Deserialize)] diff --git a/src/commands/debug_files/find.rs b/src/commands/debug_files/find.rs index 4cd78c313b..54675cf028 100644 --- a/src/commands/debug_files/find.rs +++ b/src/commands/debug_files/find.rs @@ -150,7 +150,7 @@ fn find_ids( DifType::Breakpad => find_ids_for_breakpad(&dirent, &remaining), DifType::Proguard => find_ids_for_proguard(&dirent, &proguard_uuids), DifType::Jvm => find_ids_for_sourcebundle(&dirent, &remaining), - DifType::Wasm => None, + DifType::Wasm | DifType::Dxil | DifType::DxilPdb => None, }) .flatten() .collect(); diff --git a/src/commands/debug_files/upload.rs b/src/commands/debug_files/upload.rs index d8e30d00fd..7c6ba0a876 100644 --- a/src/commands/debug_files/upload.rs +++ b/src/commands/debug_files/upload.rs @@ -247,6 +247,8 @@ pub fn execute(matches: &ArgMatches) -> Result<()> { "portablepdb" => upload.filter_format(DifFormat::Object(FileFormat::PortablePdb)), "jvm" => upload.filter_format(DifFormat::Object(FileFormat::SourceBundle)), "wasm" => upload.filter_format(DifFormat::Object(FileFormat::Wasm)), + "dxil" => upload.filter_format(DifFormat::Dxil), + "dxilpdb" => upload.filter_format(DifFormat::DxilPdb), "bcsymbolmap" => { upload.filter_format(DifFormat::BcSymbolMap); upload.filter_format(DifFormat::PList) diff --git a/src/utils/chunks/types.rs b/src/utils/chunks/types.rs index a8a1ceca2d..63115238f6 100644 --- a/src/utils/chunks/types.rs +++ b/src/utils/chunks/types.rs @@ -23,6 +23,12 @@ pub trait Assemblable { /// Returns the debug ID of the object. /// Types which do not have a debug ID should return `None`. fn debug_id(&self) -> Option; + + /// Returns the code ID of the object, if it has one that is not + /// derived by the server from the file contents. + fn code_id(&self) -> Option> { + None + } } /// Chunked arbitrary data with computed SHA1 checksums. @@ -105,6 +111,10 @@ where fn debug_id(&self) -> Option { (*self).debug_id() } + + fn code_id(&self) -> Option> { + (*self).code_id() + } } impl Assemblable for Chunked @@ -118,6 +128,10 @@ where fn debug_id(&self) -> Option { self.object().debug_id() } + + fn code_id(&self) -> Option> { + self.object().code_id() + } } impl<'c, T> From<&'c Chunked> for ChunkedDifRequest<'c> @@ -127,5 +141,6 @@ where fn from(value: &'c Chunked) -> Self { Self::new(value.name(), value.chunk_hashes(), value.checksum()) .with_debug_id(value.debug_id()) + .with_code_id(value.code_id()) } } diff --git a/src/utils/dif.rs b/src/utils/dif.rs index 321244f378..f9a3afe312 100644 --- a/src/utils/dif.rs +++ b/src/utils/dif.rs @@ -2,7 +2,7 @@ use std::path::Path; use std::str; use std::{fmt, iter}; -use anyhow::{bail, Context as _, Error, Result}; +use anyhow::{bail, format_err, Context as _, Error, Result}; use proguard::ProguardMapping; use serde::ser::{SerializeStruct as _, Serializer}; use serde::Serialize; @@ -22,6 +22,8 @@ pub enum DifType { PortablePdb, Wasm, Jvm, + Dxil, + DxilPdb, } impl DifType { @@ -37,6 +39,8 @@ impl DifType { DifType::Proguard => "proguard", DifType::Wasm => "wasm", DifType::Jvm => "jvm", + DifType::Dxil => "dxil", + DifType::DxilPdb => "dxilpdb", } } @@ -52,6 +56,8 @@ impl DifType { DifType::Proguard, DifType::Wasm, DifType::Jvm, + DifType::Dxil, + DifType::DxilPdb, ] } @@ -67,6 +73,8 @@ impl DifType { "proguard", "wasm", "jvm", + "dxil", + "dxilpdb", ] } } @@ -92,6 +100,8 @@ impl str::FromStr for DifType { "proguard" => Ok(DifType::Proguard), "wasm" => Ok(DifType::Wasm), "jvm" => Ok(DifType::Jvm), + "dxil" => Ok(DifType::Dxil), + "dxilpdb" => Ok(DifType::DxilPdb), _ => bail!("Invalid debug info file type"), } } @@ -257,6 +267,9 @@ impl DifFile<'static> { Some(DifType::Breakpad) => DifFile::open_object(&path, FileFormat::Breakpad), Some(DifType::Proguard) => DifFile::open_proguard(&path), Some(DifType::Jvm) => DifFile::open_object(&path, FileFormat::SourceBundle), + Some(DifType::Dxil | DifType::DxilPdb) => { + Err(format_err!("Shader debug files can only be uploaded")) + } None => DifFile::try_open(&path), }; diff --git a/src/utils/dif_upload/mod.rs b/src/utils/dif_upload/mod.rs index b3fef2312e..8e10bbfb5b 100644 --- a/src/utils/dif_upload/mod.rs +++ b/src/utils/dif_upload/mod.rs @@ -4,6 +4,7 @@ //! `DifUpload` for more information. mod error; +mod shader; use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; @@ -34,6 +35,7 @@ use zip::result::ZipError; use zip::ZipArchive; use self::error::ValidationError; +use self::shader::ShaderManifest; use crate::api::{Api, ChunkServerOptions}; use crate::config::Config; use crate::constants::{DEFAULT_MAX_DIF_SIZE, DEFAULT_MAX_WAIT}; @@ -56,6 +58,12 @@ enum ParsedDif<'a> { BcSymbolMap, UuidMap, Il2Cpp, + /// A DXIL shader container with an optional NVIDIA Aftermath shader hash. + Dxil { + code_id: Option, + }, + /// A DXC shader PDB. + DxilPdb, } impl<'slf, 'data: 'slf> AsSelf<'slf> for ParsedDif<'data> { @@ -145,6 +153,32 @@ impl<'data> DifMatch<'data> { }) } + /// Creates a [`DifMatch`] from a DXC shader debug file. + /// + /// The `debug_id` is derived from the file stem by the caller. For DXIL + /// containers, `code_id` carries the Aftermath shader hash if known. + fn from_shader( + kind: ShaderDifKind, + debug_id: DebugId, + name: String, + data: ByteView<'static>, + code_id: Option, + ) -> Self { + let parsed = match kind { + ShaderDifKind::Dxil => ParsedDif::Dxil { code_id }, + ShaderDifKind::DxilPdb => ParsedDif::DxilPdb, + }; + let dif = SelfCell::new(data, |_| parsed); + + Self { + _backing: None, + dif, + name, + debug_id: Some(debug_id), + attachments: None, + } + } + fn from_plist(uuid: DebugId, name: String, data: ByteView<'static>) -> Result { let dif = SelfCell::try_new(data, |buf| { UuidMapping::parse_plist(uuid, unsafe { &*buf }).map(|_| ParsedDif::UuidMap) @@ -183,6 +217,8 @@ impl<'data> DifMatch<'data> { ParsedDif::BcSymbolMap => None, ParsedDif::UuidMap => None, ParsedDif::Il2Cpp => None, + ParsedDif::Dxil { .. } => None, + ParsedDif::DxilPdb => None, } } @@ -192,6 +228,16 @@ impl<'data> DifMatch<'data> { ParsedDif::BcSymbolMap => DifFormat::BcSymbolMap, ParsedDif::UuidMap => DifFormat::PList, ParsedDif::Il2Cpp => DifFormat::Il2Cpp, + ParsedDif::Dxil { .. } => DifFormat::Dxil, + ParsedDif::DxilPdb => DifFormat::DxilPdb, + } + } + + /// Returns the code ID of this DIF, if it is not derived by the server. + pub fn code_id(&self) -> Option<&str> { + match self.dif.get() { + ParsedDif::Dxil { code_id } => code_id.as_deref(), + _ => None, } } @@ -202,6 +248,8 @@ impl<'data> DifMatch<'data> { ParsedDif::BcSymbolMap => self.dif.owner(), ParsedDif::UuidMap => self.dif.owner(), ParsedDif::Il2Cpp => self.dif.owner(), + ParsedDif::Dxil { .. } => self.dif.owner(), + ParsedDif::DxilPdb => self.dif.owner(), } } @@ -272,6 +320,8 @@ impl Display for DifMatch<'_> { ParsedDif::BcSymbolMap => String::from("bcsymbolmap"), ParsedDif::UuidMap => String::from("uuidmap"), ParsedDif::Il2Cpp => String::from("il2cpp"), + ParsedDif::Dxil { .. } => String::from("dxil"), + ParsedDif::DxilPdb => String::from("dxilpdb"), }; write!( @@ -301,6 +351,10 @@ impl Assemblable for DifMatch<'_> { fn debug_id(&self) -> Option { self.debug_id } + + fn code_id(&self) -> Option> { + DifMatch::code_id(self).map(Cow::Borrowed) + } } type ZipFileArchive = ZipArchive>; @@ -595,6 +649,7 @@ fn search_difs(options: &DifUpload) -> Result>> { pb.set_style(progress_style); let mut age_overrides = BTreeMap::new(); + let mut shader_manifests = ShaderManifestCache::default(); let mut collected = Vec::new(); for base_path in &options.paths { if base_path == Path::new("") { @@ -608,7 +663,13 @@ fn search_difs(options: &DifUpload) -> Result>> { debug!("trying to process {name}"); pb.set_message(&name); - if Archive::peek(&buffer) != FileFormat::Unknown { + if let Some(kind) = ShaderDifKind::detect(&buffer) { + if let Some(dif) = + collect_shader_dif(&source, name, buffer, options, kind, &mut shader_manifests)? + { + collected.push(dif); + } + } else if Archive::peek(&buffer) != FileFormat::Unknown { let mut difs = collect_object_dif(source, name, buffer, options, &mut age_overrides)?; collected.append(difs.as_mut()); @@ -674,6 +735,119 @@ impl Display for AuxDifKind { } } +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum ShaderDifKind { + Dxil, + DxilPdb, +} + +impl ShaderDifKind { + /// Detects DXC shader debug files from their leading bytes. + /// + /// This must run before [`Archive::peek`], since shader PDBs carry the MSF + /// magic but cannot be parsed by symbolic. + fn detect(data: &[u8]) -> Option { + if shader::is_dxil(data) { + Some(Self::Dxil) + } else if shader::is_dxil_pdb(data) { + Some(Self::DxilPdb) + } else { + None + } + } + + fn format(self) -> DifFormat { + match self { + Self::Dxil => DifFormat::Dxil, + Self::DxilPdb => DifFormat::DxilPdb, + } + } +} + +impl Display for ShaderDifKind { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Dxil => write!(f, "DXIL"), + Self::DxilPdb => write!(f, "DXIL PDB"), + } + } +} + +/// Per-directory cache of shader manifests found during the search. +#[derive(Debug, Default)] +struct ShaderManifestCache { + manifests: BTreeMap>, + warned_missing: bool, +} + +impl ShaderManifestCache { + /// Looks up the code ID of a `.dxil` file from the manifest in its directory. + fn code_id_for(&mut self, path: &Path) -> Option { + let dir = path.parent()?.to_path_buf(); + let manifest = self + .manifests + .entry(dir) + .or_insert_with(|| ShaderManifest::load_for(path)) + .as_ref()?; + + let file_name = path.file_name()?.to_str()?; + manifest.code_id(file_name) + } + + fn warn_missing_once(&mut self, name: &str) { + if !self.warned_missing { + self.warned_missing = true; + warn!( + "No {} entry for {name}; DXIL files without an entry are uploaded without code ID", + shader::MANIFEST_FILE_NAME + ); + } + } +} + +/// Collects a DXC shader debug file into a [`DifMatch`]. +/// +/// The debug ID is derived from the 32-hex file stem. For DXIL containers found +/// in the file system, the code ID is looked up in the shader manifest next to +/// the file. +fn collect_shader_dif<'a>( + source: &DifSource<'_>, + name: String, + buffer: ByteView<'static>, + options: &DifUpload, + kind: ShaderDifKind, + manifests: &mut ShaderManifestCache, +) -> Result>> { + if !options.valid_format(kind.format()) { + return Ok(None); + } + + let Some(debug_id) = shader::debug_id_from_stem(&name) else { + warn!("Skipping {kind} with invalid filename: {name}"); + return Ok(None); + }; + + let code_id = match (kind, source) { + (ShaderDifKind::Dxil, DifSource::FileSystem(path)) => { + let code_id = manifests.code_id_for(path); + if code_id.is_none() { + manifests.warn_missing_once(&name); + } + code_id + } + _ => None, + }; + + let dif = DifMatch::from_shader(kind, debug_id, name.clone(), buffer, code_id); + + if let Err(e) = options.validate_dif(&dif) { + error::handle(&name, &e)?; + return Ok(None); + } + + Ok(Some(dif)) +} + /// Collects a possible BCSymbolmap or PList into a [`DifMatch`]. /// /// The `name` is the relative path of the file processed, while `buffer` contains the @@ -1224,6 +1398,10 @@ pub enum DifFormat { PList, /// A Unity il2cpp line mapping file. Il2Cpp, + /// A DXIL shader container produced by DXC. + Dxil, + /// A DXC shader PDB. + DxilPdb, } /// Searches, processes and uploads debug information files (DIFs). diff --git a/src/utils/dif_upload/shader.rs b/src/utils/dif_upload/shader.rs new file mode 100644 index 0000000000..cea3e8aa4d --- /dev/null +++ b/src/utils/dif_upload/shader.rs @@ -0,0 +1,200 @@ +//! Detection and metadata helpers for DXC shader debug files. +//! +//! Two kinds of files are handled here: +//! +//! - `dxil`: a DXIL container (`DXBC` magic). Its debug ID is the DXC hash +//! encoded in the file stem, and its code ID is the NVIDIA Aftermath hash +//! taken from a manifest written next to the files. +//! - `dxilpdb`: a DXC shader PDB. It carries the MSF magic but has no DBI +//! stream, so symbolic cannot parse it. Its debug ID is also taken from the +//! file stem. + +use std::collections::HashMap; +use std::ffi::OsStr; +use std::fs; +use std::path::Path; + +use log::{debug, warn}; +use symbolic::common::DebugId; + +/// Name of the manifest file written by the Unreal cook hook. +pub const MANIFEST_FILE_NAME: &str = "sentry-shader-manifest.tsv"; + +const DXBC_MAGIC: &[u8] = b"DXBC"; +const MSF_MAGIC: &[u8] = b"Microsoft C/C++ MSF 7.00\r\n\x1aDS"; +const ILDN_MARKER: &[u8] = b"ILDN"; +const ILDN_SCAN_LIMIT: usize = 64 * 1024; + +/// Returns `true` if the buffer looks like a DXIL container. +pub fn is_dxil(data: &[u8]) -> bool { + data.starts_with(DXBC_MAGIC) +} + +/// Returns `true` if the buffer looks like a DXC shader PDB. +/// +/// Such files start with the MSF magic and carry an `ILDN` part within the +/// first 64 KiB. +pub fn is_dxil_pdb(data: &[u8]) -> bool { + if !data.starts_with(MSF_MAGIC) { + return false; + } + + let haystack = &data[..data.len().min(ILDN_SCAN_LIMIT)]; + haystack + .windows(ILDN_MARKER.len()) + .any(|window| window == ILDN_MARKER) +} + +/// Derives the debug ID from the file stem of a shader debug file. +/// +/// DXC names shader artifacts after their 128-bit hash, encoded as 32 lowercase +/// hex characters. The hash is used verbatim as UUID with appendix 0. +pub fn debug_id_from_stem(name: &str) -> Option { + let stem = Path::new(name).file_stem()?.to_str()?; + if stem.len() != 32 || !stem.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + + stem.parse().ok() +} + +/// Maps `.dxil` file names to their NVIDIA Aftermath shader hash. +#[derive(Debug, Default)] +pub struct ShaderManifest { + entries: HashMap, +} + +impl ShaderManifest { + /// Loads the manifest next to the given file, if present. + /// + /// Returns `None` if the directory contains no manifest or if it cannot be + /// read. + pub fn load_for(path: &Path) -> Option { + let manifest_path = path.parent()?.join(MANIFEST_FILE_NAME); + let contents = match fs::read_to_string(&manifest_path) { + Ok(contents) => contents, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return None, + Err(err) => { + warn!( + "Could not read shader manifest {}: {err}", + manifest_path.display() + ); + return None; + } + }; + + let manifest = Self::parse(&contents); + debug!( + "loaded shader manifest {} with {} entries", + manifest_path.display(), + manifest.entries.len() + ); + Some(manifest) + } + + /// Parses manifest contents. + /// + /// Every line holds `\t\t[\t]`. + /// Malformed lines are skipped. + pub fn parse(contents: &str) -> Self { + let entries = contents + .lines() + .filter_map(|line| { + let mut columns = line.trim_end_matches('\r').split('\t'); + let hash = columns.next()?.trim().parse::().ok()?; + let _pdb_name = columns.next()?; + let dxil_name = file_name_of(columns.next()?)?; + Some((dxil_name.to_owned(), hash)) + }) + .collect(); + + Self { entries } + } + + /// Returns the code ID for the given `.dxil` file name, if known. + pub fn code_id(&self, dxil_file_name: &str) -> Option { + self.entries + .get(dxil_file_name) + .map(|hash| format!("{hash:016x}")) + } +} + +/// Returns the final component of a path that may use either separator. +fn file_name_of(path: &str) -> Option<&str> { + let name = path + .trim() + .rsplit(['\\', '/']) + .next() + .filter(|name| !name.is_empty())?; + + Path::new(name).file_name().and_then(OsStr::to_str) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_dxil() { + assert!(is_dxil(b"DXBC\x01\x02\x03")); + assert!(!is_dxil(b"DXB")); + assert!(!is_dxil(b"\x7fELF")); + } + + #[test] + fn test_is_dxil_pdb() { + let mut data = MSF_MAGIC.to_vec(); + data.extend_from_slice(&[0; 4096]); + assert!(!is_dxil_pdb(&data), "PDB without ILDN must not match"); + + data.extend_from_slice(b"ILDN"); + assert!(is_dxil_pdb(&data)); + + let mut far = MSF_MAGIC.to_vec(); + far.resize(ILDN_SCAN_LIMIT, 0); + far.extend_from_slice(b"ILDN"); + assert!(!is_dxil_pdb(&far), "ILDN past 64 KiB must not match"); + + assert!(!is_dxil_pdb(b"DXBCILDN")); + } + + #[test] + fn test_debug_id_from_stem() { + let id = debug_id_from_stem("731322bf682bdc554eefdd89098b7383.dxil").unwrap(); + assert_eq!(id.to_string(), "731322bf-682b-dc55-4eef-dd89098b7383"); + assert_eq!(id.appendix(), 0); + + assert_eq!( + debug_id_from_stem("sub/dir/731322bf682bdc554eefdd89098b7383.pdb"), + Some(id) + ); + assert!(debug_id_from_stem("731322bf682bdc554eefdd89098b738.dxil").is_none()); + assert!(debug_id_from_stem("not-a-hash.dxil").is_none()); + assert!(debug_id_from_stem("731322bf682bdc554eefdd89098b73zz.dxil").is_none()); + } + + #[test] + fn test_manifest_parse() { + let contents = "709220027517385083\t000057e2ed5c597bf3ca2651c6e13a1d.pdb\tD:\\x\\000057e2ed5c597bf3ca2651c6e13a1d.dxil\r\n\ + 14389219712596414370\t0001182625509522f9bcf6fce85c2ffc.pdb\tD:/y/0001182625509522f9bcf6fce85c2ffc.dxil\t\n\ + 123\t\n\ + notanumber\tfoo.pdb\tfoo.dxil\n\ + 42\tbar.pdb\tbar.dxil\tSomeGroup\n"; + let manifest = ShaderManifest::parse(contents); + + assert_eq!( + manifest.code_id("000057e2ed5c597bf3ca2651c6e13a1d.dxil"), + Some("09d7a7db7fc4397b".to_owned()) + ); + assert_eq!( + manifest.code_id("0001182625509522f9bcf6fce85c2ffc.dxil"), + Some("c7b0c7175c5b8fa2".to_owned()) + ); + assert_eq!( + manifest.code_id("bar.dxil"), + Some("000000000000002a".to_owned()) + ); + assert_eq!(manifest.code_id("foo.dxil"), None); + assert_eq!(manifest.code_id("missing.dxil"), None); + } +} From 1dfdacdd4b9f2049d62222f961c6dc9b4f158af8 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Sun, 1 Nov 2026 16:06:19 +0200 Subject: [PATCH 2/8] Forward the shader group from the cook manifest in DXIL assemble requests --- src/api/data_types/chunking/dif.rs | 10 +++++ src/utils/chunks/types.rs | 15 +++++++ src/utils/dif_upload/mod.rs | 66 ++++++++++++++++++++++-------- src/utils/dif_upload/shader.rs | 48 ++++++++++++++++++++-- 4 files changed, 119 insertions(+), 20 deletions(-) diff --git a/src/api/data_types/chunking/dif.rs b/src/api/data_types/chunking/dif.rs index 2a32e51a40..3c80aa08fa 100644 --- a/src/api/data_types/chunking/dif.rs +++ b/src/api/data_types/chunking/dif.rs @@ -16,6 +16,8 @@ pub struct ChunkedDifRequest<'a> { pub debug_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub code_id: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub shader_group: Option>, pub chunks: &'a [Digest], #[serde(skip)] hash: Digest, @@ -31,6 +33,7 @@ impl<'a> ChunkedDifRequest<'a> { hash, debug_id: None, code_id: None, + shader_group: None, } } @@ -47,6 +50,13 @@ impl<'a> ChunkedDifRequest<'a> { self.code_id = code_id; self } + + /// Set the provided shader_group on the request, or clear it if + /// `None` is passed. + pub fn with_shader_group(mut self, shader_group: Option>) -> Self { + self.shader_group = shader_group; + self + } } #[derive(Debug, Deserialize)] diff --git a/src/utils/chunks/types.rs b/src/utils/chunks/types.rs index 63115238f6..6151ed5f9d 100644 --- a/src/utils/chunks/types.rs +++ b/src/utils/chunks/types.rs @@ -29,6 +29,12 @@ pub trait Assemblable { fn code_id(&self) -> Option> { None } + + /// Returns the shader group of the object, if it is a shader debug file + /// that belongs to one. + fn shader_group(&self) -> Option> { + None + } } /// Chunked arbitrary data with computed SHA1 checksums. @@ -115,6 +121,10 @@ where fn code_id(&self) -> Option> { (*self).code_id() } + + fn shader_group(&self) -> Option> { + (*self).shader_group() + } } impl Assemblable for Chunked @@ -132,6 +142,10 @@ where fn code_id(&self) -> Option> { self.object().code_id() } + + fn shader_group(&self) -> Option> { + self.object().shader_group() + } } impl<'c, T> From<&'c Chunked> for ChunkedDifRequest<'c> @@ -142,5 +156,6 @@ where Self::new(value.name(), value.chunk_hashes(), value.checksum()) .with_debug_id(value.debug_id()) .with_code_id(value.code_id()) + .with_shader_group(value.shader_group()) } } diff --git a/src/utils/dif_upload/mod.rs b/src/utils/dif_upload/mod.rs index 8e10bbfb5b..4205ddc9c2 100644 --- a/src/utils/dif_upload/mod.rs +++ b/src/utils/dif_upload/mod.rs @@ -58,9 +58,11 @@ enum ParsedDif<'a> { BcSymbolMap, UuidMap, Il2Cpp, - /// A DXIL shader container with an optional NVIDIA Aftermath shader hash. + /// A DXIL shader container with an optional NVIDIA Aftermath shader hash + /// and an optional shader group taken from the manifest. Dxil { code_id: Option, + shader_group: Option, }, /// A DXC shader PDB. DxilPdb, @@ -156,16 +158,20 @@ impl<'data> DifMatch<'data> { /// Creates a [`DifMatch`] from a DXC shader debug file. /// /// The `debug_id` is derived from the file stem by the caller. For DXIL - /// containers, `code_id` carries the Aftermath shader hash if known. + /// containers, `manifest_entry` carries the Aftermath shader hash and the + /// shader group if known. fn from_shader( kind: ShaderDifKind, debug_id: DebugId, name: String, data: ByteView<'static>, - code_id: Option, + manifest_entry: ShaderManifestEntry, ) -> Self { let parsed = match kind { - ShaderDifKind::Dxil => ParsedDif::Dxil { code_id }, + ShaderDifKind::Dxil => ParsedDif::Dxil { + code_id: manifest_entry.code_id, + shader_group: manifest_entry.shader_group, + }, ShaderDifKind::DxilPdb => ParsedDif::DxilPdb, }; let dif = SelfCell::new(data, |_| parsed); @@ -236,7 +242,16 @@ impl<'data> DifMatch<'data> { /// Returns the code ID of this DIF, if it is not derived by the server. pub fn code_id(&self) -> Option<&str> { match self.dif.get() { - ParsedDif::Dxil { code_id } => code_id.as_deref(), + ParsedDif::Dxil { code_id, .. } => code_id.as_deref(), + _ => None, + } + } + + /// Returns the shader group of this DIF, if it is a DXIL container with a + /// manifest entry that carries one. + pub fn shader_group(&self) -> Option<&str> { + match self.dif.get() { + ParsedDif::Dxil { shader_group, .. } => shader_group.as_deref(), _ => None, } } @@ -355,6 +370,10 @@ impl Assemblable for DifMatch<'_> { fn code_id(&self) -> Option> { DifMatch::code_id(self).map(Cow::Borrowed) } + + fn shader_group(&self) -> Option> { + DifMatch::shader_group(self).map(Cow::Borrowed) + } } type ZipFileArchive = ZipArchive>; @@ -773,6 +792,13 @@ impl Display for ShaderDifKind { } } +/// Manifest metadata resolved for a single `.dxil` file. +#[derive(Debug, Default)] +struct ShaderManifestEntry { + code_id: Option, + shader_group: Option, +} + /// Per-directory cache of shader manifests found during the search. #[derive(Debug, Default)] struct ShaderManifestCache { @@ -781,8 +807,13 @@ struct ShaderManifestCache { } impl ShaderManifestCache { - /// Looks up the code ID of a `.dxil` file from the manifest in its directory. - fn code_id_for(&mut self, path: &Path) -> Option { + /// Looks up the code ID and shader group of a `.dxil` file from the + /// manifest in its directory. + fn entry_for(&mut self, path: &Path) -> ShaderManifestEntry { + self.lookup(path).unwrap_or_default() + } + + fn lookup(&mut self, path: &Path) -> Option { let dir = path.parent()?.to_path_buf(); let manifest = self .manifests @@ -791,7 +822,10 @@ impl ShaderManifestCache { .as_ref()?; let file_name = path.file_name()?.to_str()?; - manifest.code_id(file_name) + Some(ShaderManifestEntry { + code_id: manifest.code_id(file_name), + shader_group: manifest.shader_group(file_name).map(str::to_owned), + }) } fn warn_missing_once(&mut self, name: &str) { @@ -808,8 +842,8 @@ impl ShaderManifestCache { /// Collects a DXC shader debug file into a [`DifMatch`]. /// /// The debug ID is derived from the 32-hex file stem. For DXIL containers found -/// in the file system, the code ID is looked up in the shader manifest next to -/// the file. +/// in the file system, the code ID and shader group are looked up in the shader +/// manifest next to the file. fn collect_shader_dif<'a>( source: &DifSource<'_>, name: String, @@ -827,18 +861,18 @@ fn collect_shader_dif<'a>( return Ok(None); }; - let code_id = match (kind, source) { + let manifest_entry = match (kind, source) { (ShaderDifKind::Dxil, DifSource::FileSystem(path)) => { - let code_id = manifests.code_id_for(path); - if code_id.is_none() { + let entry = manifests.entry_for(path); + if entry.code_id.is_none() { manifests.warn_missing_once(&name); } - code_id + entry } - _ => None, + _ => ShaderManifestEntry::default(), }; - let dif = DifMatch::from_shader(kind, debug_id, name.clone(), buffer, code_id); + let dif = DifMatch::from_shader(kind, debug_id, name.clone(), buffer, manifest_entry); if let Err(e) = options.validate_dif(&dif) { error::handle(&name, &e)?; diff --git a/src/utils/dif_upload/shader.rs b/src/utils/dif_upload/shader.rs index cea3e8aa4d..1a916c5003 100644 --- a/src/utils/dif_upload/shader.rs +++ b/src/utils/dif_upload/shader.rs @@ -58,10 +58,17 @@ pub fn debug_id_from_stem(name: &str) -> Option { stem.parse().ok() } -/// Maps `.dxil` file names to their NVIDIA Aftermath shader hash. +/// Metadata recorded for a single `.dxil` file in the manifest. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ManifestEntry { + hash: u64, + group: Option, +} + +/// Maps `.dxil` file names to their NVIDIA Aftermath shader hash and group. #[derive(Debug, Default)] pub struct ShaderManifest { - entries: HashMap, + entries: HashMap, } impl ShaderManifest { @@ -95,6 +102,7 @@ impl ShaderManifest { /// Parses manifest contents. /// /// Every line holds `\t\t[\t]`. + /// The optional group is trimmed; an empty group is treated as absent. /// Malformed lines are skipped. pub fn parse(contents: &str) -> Self { let entries = contents @@ -104,7 +112,12 @@ impl ShaderManifest { let hash = columns.next()?.trim().parse::().ok()?; let _pdb_name = columns.next()?; let dxil_name = file_name_of(columns.next()?)?; - Some((dxil_name.to_owned(), hash)) + let group = columns + .next() + .map(str::trim) + .filter(|group| !group.is_empty()) + .map(str::to_owned); + Some((dxil_name.to_owned(), ManifestEntry { hash, group })) }) .collect(); @@ -115,7 +128,12 @@ impl ShaderManifest { pub fn code_id(&self, dxil_file_name: &str) -> Option { self.entries .get(dxil_file_name) - .map(|hash| format!("{hash:016x}")) + .map(|entry| format!("{:016x}", entry.hash)) + } + + /// Returns the shader group for the given `.dxil` file name, if known. + pub fn shader_group(&self, dxil_file_name: &str) -> Option<&str> { + self.entries.get(dxil_file_name)?.group.as_deref() } } @@ -197,4 +215,26 @@ mod tests { assert_eq!(manifest.code_id("foo.dxil"), None); assert_eq!(manifest.code_id("missing.dxil"), None); } + + #[test] + fn test_manifest_shader_group() { + let contents = "1\ta.pdb\ta.dxil\n\ + 2\tb.pdb\tb.dxil\tGlobalShaders\n\ + 3\tc.pdb\tc.dxil\t Padded Group \r\n\ + 4\td.pdb\td.dxil\t \n\ + 5\te.pdb\te.dxil\t\n"; + let manifest = ShaderManifest::parse(contents); + + assert_eq!(manifest.shader_group("a.dxil"), None); + assert_eq!(manifest.shader_group("b.dxil"), Some("GlobalShaders")); + assert_eq!(manifest.shader_group("c.dxil"), Some("Padded Group")); + assert_eq!(manifest.shader_group("d.dxil"), None); + assert_eq!(manifest.shader_group("e.dxil"), None); + assert_eq!(manifest.shader_group("missing.dxil"), None); + + assert_eq!( + manifest.code_id("d.dxil"), + Some("0000000000000004".to_owned()) + ); + } } From ff627acf38531fe08e88c0a75ec24a43933ade74 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Mon, 2 Nov 2026 17:39:56 +0200 Subject: [PATCH 3/8] Simplify shader debug file collection Keep the manifest as a plain map loaded once per directory, build the DXIL match with an optional ShaderInfo instead of a cache and entry types, and expose code_id and shader_group only through Assemblable. Update the --type help snapshots for the two new kinds. --- src/utils/dif_upload/mod.rs | 167 +++--------- src/utils/dif_upload/shader.rs | 258 ++++++------------ .../debug_files-upload-help.trycmd | 3 +- .../_cases/upload_dif/upload_dif-help.trycmd | 3 +- .../upload_dsym/upload_dsym-help.trycmd | 3 +- 5 files changed, 129 insertions(+), 305 deletions(-) diff --git a/src/utils/dif_upload/mod.rs b/src/utils/dif_upload/mod.rs index 4205ddc9c2..8188034893 100644 --- a/src/utils/dif_upload/mod.rs +++ b/src/utils/dif_upload/mod.rs @@ -35,7 +35,7 @@ use zip::result::ZipError; use zip::ZipArchive; use self::error::ValidationError; -use self::shader::ShaderManifest; +use self::shader::{ShaderInfo, ShaderManifest}; use crate::api::{Api, ChunkServerOptions}; use crate::config::Config; use crate::constants::{DEFAULT_MAX_DIF_SIZE, DEFAULT_MAX_WAIT}; @@ -58,13 +58,7 @@ enum ParsedDif<'a> { BcSymbolMap, UuidMap, Il2Cpp, - /// A DXIL shader container with an optional NVIDIA Aftermath shader hash - /// and an optional shader group taken from the manifest. - Dxil { - code_id: Option, - shader_group: Option, - }, - /// A DXC shader PDB. + Dxil(Option), DxilPdb, } @@ -155,26 +149,17 @@ impl<'data> DifMatch<'data> { }) } - /// Creates a [`DifMatch`] from a DXC shader debug file. - /// - /// The `debug_id` is derived from the file stem by the caller. For DXIL - /// containers, `manifest_entry` carries the Aftermath shader hash and the - /// shader group if known. fn from_shader( kind: ShaderDifKind, debug_id: DebugId, name: String, data: ByteView<'static>, - manifest_entry: ShaderManifestEntry, + info: Option, ) -> Self { - let parsed = match kind { - ShaderDifKind::Dxil => ParsedDif::Dxil { - code_id: manifest_entry.code_id, - shader_group: manifest_entry.shader_group, - }, + let dif = SelfCell::new(data, |_| match kind { + ShaderDifKind::Dxil => ParsedDif::Dxil(info), ShaderDifKind::DxilPdb => ParsedDif::DxilPdb, - }; - let dif = SelfCell::new(data, |_| parsed); + }); Self { _backing: None, @@ -223,7 +208,7 @@ impl<'data> DifMatch<'data> { ParsedDif::BcSymbolMap => None, ParsedDif::UuidMap => None, ParsedDif::Il2Cpp => None, - ParsedDif::Dxil { .. } => None, + ParsedDif::Dxil(_) => None, ParsedDif::DxilPdb => None, } } @@ -234,28 +219,11 @@ impl<'data> DifMatch<'data> { ParsedDif::BcSymbolMap => DifFormat::BcSymbolMap, ParsedDif::UuidMap => DifFormat::PList, ParsedDif::Il2Cpp => DifFormat::Il2Cpp, - ParsedDif::Dxil { .. } => DifFormat::Dxil, + ParsedDif::Dxil(_) => DifFormat::Dxil, ParsedDif::DxilPdb => DifFormat::DxilPdb, } } - /// Returns the code ID of this DIF, if it is not derived by the server. - pub fn code_id(&self) -> Option<&str> { - match self.dif.get() { - ParsedDif::Dxil { code_id, .. } => code_id.as_deref(), - _ => None, - } - } - - /// Returns the shader group of this DIF, if it is a DXIL container with a - /// manifest entry that carries one. - pub fn shader_group(&self) -> Option<&str> { - match self.dif.get() { - ParsedDif::Dxil { shader_group, .. } => shader_group.as_deref(), - _ => None, - } - } - /// Returns the raw binary data of this DIF. pub fn data(&self) -> &[u8] { match self.dif.get() { @@ -263,7 +231,7 @@ impl<'data> DifMatch<'data> { ParsedDif::BcSymbolMap => self.dif.owner(), ParsedDif::UuidMap => self.dif.owner(), ParsedDif::Il2Cpp => self.dif.owner(), - ParsedDif::Dxil { .. } => self.dif.owner(), + ParsedDif::Dxil(_) => self.dif.owner(), ParsedDif::DxilPdb => self.dif.owner(), } } @@ -335,7 +303,7 @@ impl Display for DifMatch<'_> { ParsedDif::BcSymbolMap => String::from("bcsymbolmap"), ParsedDif::UuidMap => String::from("uuidmap"), ParsedDif::Il2Cpp => String::from("il2cpp"), - ParsedDif::Dxil { .. } => String::from("dxil"), + ParsedDif::Dxil(_) => String::from("dxil"), ParsedDif::DxilPdb => String::from("dxilpdb"), }; @@ -368,11 +336,17 @@ impl Assemblable for DifMatch<'_> { } fn code_id(&self) -> Option> { - DifMatch::code_id(self).map(Cow::Borrowed) + match self.dif.get() { + ParsedDif::Dxil(Some(info)) => Some(Cow::Borrowed(&info.code_id)), + _ => None, + } } fn shader_group(&self) -> Option> { - DifMatch::shader_group(self).map(Cow::Borrowed) + match self.dif.get() { + ParsedDif::Dxil(Some(info)) => info.shader_group.as_deref().map(Cow::Borrowed), + _ => None, + } } } @@ -668,7 +642,7 @@ fn search_difs(options: &DifUpload) -> Result>> { pb.set_style(progress_style); let mut age_overrides = BTreeMap::new(); - let mut shader_manifests = ShaderManifestCache::default(); + let mut shader_manifests = BTreeMap::new(); let mut collected = Vec::new(); for base_path in &options.paths { if base_path == Path::new("") { @@ -774,105 +748,54 @@ impl ShaderDifKind { None } } - - fn format(self) -> DifFormat { - match self { - Self::Dxil => DifFormat::Dxil, - Self::DxilPdb => DifFormat::DxilPdb, - } - } } impl Display for ShaderDifKind { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - Self::Dxil => write!(f, "DXIL"), - Self::DxilPdb => write!(f, "DXIL PDB"), - } - } -} - -/// Manifest metadata resolved for a single `.dxil` file. -#[derive(Debug, Default)] -struct ShaderManifestEntry { - code_id: Option, - shader_group: Option, -} - -/// Per-directory cache of shader manifests found during the search. -#[derive(Debug, Default)] -struct ShaderManifestCache { - manifests: BTreeMap>, - warned_missing: bool, -} - -impl ShaderManifestCache { - /// Looks up the code ID and shader group of a `.dxil` file from the - /// manifest in its directory. - fn entry_for(&mut self, path: &Path) -> ShaderManifestEntry { - self.lookup(path).unwrap_or_default() - } - - fn lookup(&mut self, path: &Path) -> Option { - let dir = path.parent()?.to_path_buf(); - let manifest = self - .manifests - .entry(dir) - .or_insert_with(|| ShaderManifest::load_for(path)) - .as_ref()?; - - let file_name = path.file_name()?.to_str()?; - Some(ShaderManifestEntry { - code_id: manifest.code_id(file_name), - shader_group: manifest.shader_group(file_name).map(str::to_owned), - }) - } - - fn warn_missing_once(&mut self, name: &str) { - if !self.warned_missing { - self.warned_missing = true; - warn!( - "No {} entry for {name}; DXIL files without an entry are uploaded without code ID", - shader::MANIFEST_FILE_NAME - ); + ShaderDifKind::Dxil => write!(f, "DXIL"), + ShaderDifKind::DxilPdb => write!(f, "DXIL PDB"), } } } /// Collects a DXC shader debug file into a [`DifMatch`]. /// -/// The debug ID is derived from the 32-hex file stem. For DXIL containers found -/// in the file system, the code ID and shader group are looked up in the shader -/// manifest next to the file. +/// The debug ID is the 32-hex file stem. The code ID and shader group of DXIL +/// containers come from the manifest next to the file, loaded once per +/// directory into `manifests`. fn collect_shader_dif<'a>( source: &DifSource<'_>, name: String, buffer: ByteView<'static>, options: &DifUpload, kind: ShaderDifKind, - manifests: &mut ShaderManifestCache, + manifests: &mut BTreeMap, ) -> Result>> { - if !options.valid_format(kind.format()) { - return Ok(None); - } - - let Some(debug_id) = shader::debug_id_from_stem(&name) else { - warn!("Skipping {kind} with invalid filename: {name}"); - return Ok(None); + let file_stem = Path::new(&name) + .file_stem() + .map(|stem| stem.to_string_lossy()) + .unwrap_or_default(); + let debug_id = match file_stem.parse() { + Ok(debug_id) => debug_id, + Err(_) => { + warn!("Skipping {kind} with invalid filename: {name}"); + return Ok(None); + } }; - let manifest_entry = match (kind, source) { - (ShaderDifKind::Dxil, DifSource::FileSystem(path)) => { - let entry = manifests.entry_for(path); - if entry.code_id.is_none() { - manifests.warn_missing_once(&name); - } - entry - } - _ => ShaderManifestEntry::default(), + let info = match (kind, source) { + (ShaderDifKind::Dxil, DifSource::FileSystem(path)) => path.parent().and_then(|dir| { + manifests + .entry(dir.to_path_buf()) + .or_insert_with(|| shader::load_manifest(dir)) + .get(path.file_name()?.to_str()?) + .cloned() + }), + _ => None, }; - let dif = DifMatch::from_shader(kind, debug_id, name.clone(), buffer, manifest_entry); + let dif = DifMatch::from_shader(kind, debug_id, name.clone(), buffer, info); if let Err(e) = options.validate_dif(&dif) { error::handle(&name, &e)?; diff --git a/src/utils/dif_upload/shader.rs b/src/utils/dif_upload/shader.rs index 1a916c5003..09f8206874 100644 --- a/src/utils/dif_upload/shader.rs +++ b/src/utils/dif_upload/shader.rs @@ -1,151 +1,79 @@ -//! Detection and metadata helpers for DXC shader debug files. -//! -//! Two kinds of files are handled here: -//! -//! - `dxil`: a DXIL container (`DXBC` magic). Its debug ID is the DXC hash -//! encoded in the file stem, and its code ID is the NVIDIA Aftermath hash -//! taken from a manifest written next to the files. -//! - `dxilpdb`: a DXC shader PDB. It carries the MSF magic but has no DBI -//! stream, so symbolic cannot parse it. Its debug ID is also taken from the -//! file stem. +//! Detection and manifest lookup for DXC shader debug files. use std::collections::HashMap; -use std::ffi::OsStr; use std::fs; use std::path::Path; -use log::{debug, warn}; -use symbolic::common::DebugId; +use log::warn; -/// Name of the manifest file written by the Unreal cook hook. -pub const MANIFEST_FILE_NAME: &str = "sentry-shader-manifest.tsv"; - -const DXBC_MAGIC: &[u8] = b"DXBC"; +const MANIFEST_FILE_NAME: &str = "sentry-shader-manifest.tsv"; const MSF_MAGIC: &[u8] = b"Microsoft C/C++ MSF 7.00\r\n\x1aDS"; -const ILDN_MARKER: &[u8] = b"ILDN"; const ILDN_SCAN_LIMIT: usize = 64 * 1024; -/// Returns `true` if the buffer looks like a DXIL container. +/// Returns `true` if the buffer is a DXIL container. pub fn is_dxil(data: &[u8]) -> bool { - data.starts_with(DXBC_MAGIC) + data.starts_with(b"DXBC") } -/// Returns `true` if the buffer looks like a DXC shader PDB. +/// Returns `true` if the buffer is a DXC shader PDB. /// -/// Such files start with the MSF magic and carry an `ILDN` part within the -/// first 64 KiB. +/// These carry the MSF magic but no DBI stream, so symbolic cannot parse them. +/// They are told apart from regular PDBs by the `ILDN` part near the start of +/// the file. pub fn is_dxil_pdb(data: &[u8]) -> bool { - if !data.starts_with(MSF_MAGIC) { - return false; - } - - let haystack = &data[..data.len().min(ILDN_SCAN_LIMIT)]; - haystack - .windows(ILDN_MARKER.len()) - .any(|window| window == ILDN_MARKER) + data.starts_with(MSF_MAGIC) + && data[..data.len().min(ILDN_SCAN_LIMIT)] + .windows(4) + .any(|window| window == b"ILDN") } -/// Derives the debug ID from the file stem of a shader debug file. -/// -/// DXC names shader artifacts after their 128-bit hash, encoded as 32 lowercase -/// hex characters. The hash is used verbatim as UUID with appendix 0. -pub fn debug_id_from_stem(name: &str) -> Option { - let stem = Path::new(name).file_stem()?.to_str()?; - if stem.len() != 32 || !stem.bytes().all(|b| b.is_ascii_hexdigit()) { - return None; - } - - stem.parse().ok() -} - -/// Metadata recorded for a single `.dxil` file in the manifest. +/// Manifest metadata of a single `.dxil` file. #[derive(Debug, Clone, PartialEq, Eq)] -struct ManifestEntry { - hash: u64, - group: Option, +pub struct ShaderInfo { + /// NVIDIA Aftermath shader hash as 16 hex digits. + pub code_id: String, + pub shader_group: Option, } -/// Maps `.dxil` file names to their NVIDIA Aftermath shader hash and group. -#[derive(Debug, Default)] -pub struct ShaderManifest { - entries: HashMap, -} - -impl ShaderManifest { - /// Loads the manifest next to the given file, if present. - /// - /// Returns `None` if the directory contains no manifest or if it cannot be - /// read. - pub fn load_for(path: &Path) -> Option { - let manifest_path = path.parent()?.join(MANIFEST_FILE_NAME); - let contents = match fs::read_to_string(&manifest_path) { - Ok(contents) => contents, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return None, - Err(err) => { - warn!( - "Could not read shader manifest {}: {err}", - manifest_path.display() - ); - return None; - } - }; - - let manifest = Self::parse(&contents); - debug!( - "loaded shader manifest {} with {} entries", - manifest_path.display(), - manifest.entries.len() - ); - Some(manifest) - } - - /// Parses manifest contents. - /// - /// Every line holds `\t\t[\t]`. - /// The optional group is trimmed; an empty group is treated as absent. - /// Malformed lines are skipped. - pub fn parse(contents: &str) -> Self { - let entries = contents - .lines() - .filter_map(|line| { - let mut columns = line.trim_end_matches('\r').split('\t'); - let hash = columns.next()?.trim().parse::().ok()?; - let _pdb_name = columns.next()?; - let dxil_name = file_name_of(columns.next()?)?; - let group = columns - .next() - .map(str::trim) - .filter(|group| !group.is_empty()) - .map(str::to_owned); - Some((dxil_name.to_owned(), ManifestEntry { hash, group })) - }) - .collect(); - - Self { entries } - } - - /// Returns the code ID for the given `.dxil` file name, if known. - pub fn code_id(&self, dxil_file_name: &str) -> Option { - self.entries - .get(dxil_file_name) - .map(|entry| format!("{:016x}", entry.hash)) - } - - /// Returns the shader group for the given `.dxil` file name, if known. - pub fn shader_group(&self, dxil_file_name: &str) -> Option<&str> { - self.entries.get(dxil_file_name)?.group.as_deref() +/// `.dxil` file names mapped to their manifest metadata. +pub type ShaderManifest = HashMap; + +/// Loads the manifest written next to the shader files in `dir`. +pub fn load_manifest(dir: &Path) -> ShaderManifest { + let path = dir.join(MANIFEST_FILE_NAME); + match fs::read_to_string(&path) { + Ok(contents) => parse_manifest(&contents), + Err(err) => { + warn!( + "Could not read {}: {err}; DXIL files are uploaded without code ID", + path.display() + ); + ShaderManifest::new() + } } } -/// Returns the final component of a path that may use either separator. -fn file_name_of(path: &str) -> Option<&str> { - let name = path - .trim() - .rsplit(['\\', '/']) - .next() - .filter(|name| !name.is_empty())?; - - Path::new(name).file_name().and_then(OsStr::to_str) +/// Parses lines of the form `\t\t[\t]`. +fn parse_manifest(contents: &str) -> ShaderManifest { + contents + .lines() + .filter_map(|line| { + let mut columns = line.split('\t'); + let hash: u64 = columns.next()?.parse().ok()?; + let _pdb_name = columns.next()?; + let dxil_name = columns.next()?.rsplit(['\\', '/']).next()?; + let shader_group = columns + .next() + .map(str::trim) + .filter(|group| !group.is_empty()) + .map(str::to_owned); + let info = ShaderInfo { + code_id: format!("{hash:016x}"), + shader_group, + }; + Some((dxil_name.to_owned(), info)) + }) + .collect() } #[cfg(test)] @@ -155,7 +83,6 @@ mod tests { #[test] fn test_is_dxil() { assert!(is_dxil(b"DXBC\x01\x02\x03")); - assert!(!is_dxil(b"DXB")); assert!(!is_dxil(b"\x7fELF")); } @@ -163,7 +90,7 @@ mod tests { fn test_is_dxil_pdb() { let mut data = MSF_MAGIC.to_vec(); data.extend_from_slice(&[0; 4096]); - assert!(!is_dxil_pdb(&data), "PDB without ILDN must not match"); + assert!(!is_dxil_pdb(&data)); data.extend_from_slice(b"ILDN"); assert!(is_dxil_pdb(&data)); @@ -171,70 +98,41 @@ mod tests { let mut far = MSF_MAGIC.to_vec(); far.resize(ILDN_SCAN_LIMIT, 0); far.extend_from_slice(b"ILDN"); - assert!(!is_dxil_pdb(&far), "ILDN past 64 KiB must not match"); + assert!(!is_dxil_pdb(&far)); assert!(!is_dxil_pdb(b"DXBCILDN")); } #[test] - fn test_debug_id_from_stem() { - let id = debug_id_from_stem("731322bf682bdc554eefdd89098b7383.dxil").unwrap(); - assert_eq!(id.to_string(), "731322bf-682b-dc55-4eef-dd89098b7383"); - assert_eq!(id.appendix(), 0); - - assert_eq!( - debug_id_from_stem("sub/dir/731322bf682bdc554eefdd89098b7383.pdb"), - Some(id) + fn test_parse_manifest() { + let manifest = parse_manifest( + "709220027517385083\t000057e2ed5c597bf3ca2651c6e13a1d.pdb\tD:\\x\\000057e2ed5c597bf3ca2651c6e13a1d.dxil\r\n\ + 2\tb.pdb\tD:/y/b.dxil\tGlobalShaders\n\ + 3\tc.pdb\tc.dxil\t Padded Group \r\n\ + 4\td.pdb\td.dxil\t \n\ + 123\t\n\ + notanumber\tfoo.pdb\tfoo.dxil\n", ); - assert!(debug_id_from_stem("731322bf682bdc554eefdd89098b738.dxil").is_none()); - assert!(debug_id_from_stem("not-a-hash.dxil").is_none()); - assert!(debug_id_from_stem("731322bf682bdc554eefdd89098b73zz.dxil").is_none()); - } - - #[test] - fn test_manifest_parse() { - let contents = "709220027517385083\t000057e2ed5c597bf3ca2651c6e13a1d.pdb\tD:\\x\\000057e2ed5c597bf3ca2651c6e13a1d.dxil\r\n\ - 14389219712596414370\t0001182625509522f9bcf6fce85c2ffc.pdb\tD:/y/0001182625509522f9bcf6fce85c2ffc.dxil\t\n\ - 123\t\n\ - notanumber\tfoo.pdb\tfoo.dxil\n\ - 42\tbar.pdb\tbar.dxil\tSomeGroup\n"; - let manifest = ShaderManifest::parse(contents); assert_eq!( - manifest.code_id("000057e2ed5c597bf3ca2651c6e13a1d.dxil"), - Some("09d7a7db7fc4397b".to_owned()) - ); - assert_eq!( - manifest.code_id("0001182625509522f9bcf6fce85c2ffc.dxil"), - Some("c7b0c7175c5b8fa2".to_owned()) + manifest["000057e2ed5c597bf3ca2651c6e13a1d.dxil"], + ShaderInfo { + code_id: "09d7a7db7fc4397b".into(), + shader_group: None, + } ); assert_eq!( - manifest.code_id("bar.dxil"), - Some("000000000000002a".to_owned()) + manifest["b.dxil"], + ShaderInfo { + code_id: "0000000000000002".into(), + shader_group: Some("GlobalShaders".into()), + } ); - assert_eq!(manifest.code_id("foo.dxil"), None); - assert_eq!(manifest.code_id("missing.dxil"), None); - } - - #[test] - fn test_manifest_shader_group() { - let contents = "1\ta.pdb\ta.dxil\n\ - 2\tb.pdb\tb.dxil\tGlobalShaders\n\ - 3\tc.pdb\tc.dxil\t Padded Group \r\n\ - 4\td.pdb\td.dxil\t \n\ - 5\te.pdb\te.dxil\t\n"; - let manifest = ShaderManifest::parse(contents); - - assert_eq!(manifest.shader_group("a.dxil"), None); - assert_eq!(manifest.shader_group("b.dxil"), Some("GlobalShaders")); - assert_eq!(manifest.shader_group("c.dxil"), Some("Padded Group")); - assert_eq!(manifest.shader_group("d.dxil"), None); - assert_eq!(manifest.shader_group("e.dxil"), None); - assert_eq!(manifest.shader_group("missing.dxil"), None); - assert_eq!( - manifest.code_id("d.dxil"), - Some("0000000000000004".to_owned()) + manifest["c.dxil"].shader_group.as_deref(), + Some("Padded Group") ); + assert_eq!(manifest["d.dxil"].shader_group, None); + assert_eq!(manifest.len(), 4); } } diff --git a/tests/integration/_cases/debug_files/not_windows/debug_files-upload-help.trycmd b/tests/integration/_cases/debug_files/not_windows/debug_files-upload-help.trycmd index d663e891d8..ccf5b25b59 100644 --- a/tests/integration/_cases/debug_files/not_windows/debug_files-upload-help.trycmd +++ b/tests/integration/_cases/debug_files/not_windows/debug_files-upload-help.trycmd @@ -16,7 +16,8 @@ Options: --auth-token Use the given Sentry auth token. -t, --type Only consider debug information files of the given type. By default, all types are considered. [possible values: bcsymbolmap, - breakpad, dsym, elf, jvm, pdb, pe, portablepdb, sourcebundle, wasm] + breakpad, dsym, dxil, dxilpdb, elf, jvm, pdb, pe, portablepdb, + sourcebundle, wasm] --log-level Set the log output verbosity. [possible values: trace, debug, info, warn, error] --no-unwind Do not scan for stack unwinding information. Specify this flag for diff --git a/tests/integration/_cases/upload_dif/upload_dif-help.trycmd b/tests/integration/_cases/upload_dif/upload_dif-help.trycmd index 3c212473cf..8f0df25c46 100644 --- a/tests/integration/_cases/upload_dif/upload_dif-help.trycmd +++ b/tests/integration/_cases/upload_dif/upload_dif-help.trycmd @@ -16,7 +16,8 @@ Options: --auth-token Use the given Sentry auth token. -t, --type Only consider debug information files of the given type. By default, all types are considered. [possible values: bcsymbolmap, - breakpad, dsym, elf, jvm, pdb, pe, portablepdb, sourcebundle, wasm] + breakpad, dsym, dxil, dxilpdb, elf, jvm, pdb, pe, portablepdb, + sourcebundle, wasm] --log-level Set the log output verbosity. [possible values: trace, debug, info, warn, error] --no-unwind Do not scan for stack unwinding information. Specify this flag for diff --git a/tests/integration/_cases/upload_dsym/upload_dsym-help.trycmd b/tests/integration/_cases/upload_dsym/upload_dsym-help.trycmd index b5d4a76c30..b3a5d3e555 100644 --- a/tests/integration/_cases/upload_dsym/upload_dsym-help.trycmd +++ b/tests/integration/_cases/upload_dsym/upload_dsym-help.trycmd @@ -16,7 +16,8 @@ Options: --auth-token Use the given Sentry auth token. -t, --type Only consider debug information files of the given type. By default, all types are considered. [possible values: bcsymbolmap, - breakpad, dsym, elf, jvm, pdb, pe, portablepdb, sourcebundle, wasm] + breakpad, dsym, dxil, dxilpdb, elf, jvm, pdb, pe, portablepdb, + sourcebundle, wasm] --log-level Set the log output verbosity. [possible values: trace, debug, info, warn, error] --no-unwind Do not scan for stack unwinding information. Specify this flag for From 4f57f5c33103650b786720c2a44d6fc72a9d7704 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Mon, 2 Nov 2026 19:01:38 +0200 Subject: [PATCH 4/8] Send assemble requests in batches of 1000 files A UE project uploads about 14,000 shader binaries per build; one assemble request with all of them runs for many minutes on the server and gets cut off by proxies. Split the request in both the first attempt and the poll loop and merge the responses. --- CHANGELOG.md | 8 ++++++++ src/utils/chunks/mod.rs | 3 +++ src/utils/chunks/upload.rs | 38 +++++++++++++++++++++++++++----------- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c26472c019..b68f16e113 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +### Features + +- Add `dxil` and `dxilpdb` debug file types to `debug-files upload` for GPU crash shader symbolication + +### Improvements + +- Send `debug-files upload` assemble requests in batches of 1000 files + ### Internal Changes - (docker) Bump the Docker image base from Alpine 3.22.0 to 3.22.7, whose `ca-certificates` already drops the distrusted eMudhra/emSign roots, and remove the `ca-certificates` version pin. diff --git a/src/utils/chunks/mod.rs b/src/utils/chunks/mod.rs index 1a984e49bc..2b86e5a5fa 100644 --- a/src/utils/chunks/mod.rs +++ b/src/utils/chunks/mod.rs @@ -29,6 +29,9 @@ use crate::utils::progress::{ProgressBar, ProgressBarMode, ProgressStyle}; /// Timeout for polling all assemble endpoints. pub const ASSEMBLE_POLL_INTERVAL: Duration = Duration::from_millis(1000); +/// Maximum number of files per assemble request. +pub const ASSEMBLE_BATCH_SIZE: usize = 1000; + /// A trait required by `BatchedSliceExt` to determine the logical size of a /// batch. Semantics and unit of the size depend on the batching use case, /// likely a number of bytes. See `BatchedSliceExt` for more information. diff --git a/src/utils/chunks/upload.rs b/src/utils/chunks/upload.rs index a472737dee..3d29de1288 100644 --- a/src/utils/chunks/upload.rs +++ b/src/utils/chunks/upload.rs @@ -7,12 +7,16 @@ use anyhow::Result; use indicatif::ProgressStyle; use crate::{ - api::{Api, AssembleDifsRequest, ChunkServerOptions, ChunkedFileState, DebugInfoFile}, + api::{ + Api, AssembleDifsRequest, AssembleDifsResponse, ChunkServerOptions, ChunkedFileState, + DebugInfoFile, + }, utils::progress::ProgressBar, }; use super::{ - Assemblable, Chunk, ChunkOptions, Chunked, MissingObjectsInfo, ASSEMBLE_POLL_INTERVAL, + Assemblable, Chunk, ChunkOptions, Chunked, MissingObjectsInfo, ASSEMBLE_BATCH_SIZE, + ASSEMBLE_POLL_INTERVAL, }; pub fn upload_chunked_objects( @@ -53,11 +57,15 @@ where T: AsRef<[u8]> + Assemblable, { let api = Api::current(); - let request: AssembleDifsRequest<'_> = objects.iter().collect(); - - let response = - api.authenticated()? - .assemble_difs(options.org(), options.project(), &request)?; + let mut response = AssembleDifsResponse::new(); + for batch in objects.chunks(ASSEMBLE_BATCH_SIZE) { + let request: AssembleDifsRequest<'_> = batch.iter().collect(); + response.extend(api.authenticated()?.assemble_difs( + options.org(), + options.project(), + &request, + )?); + } // We map all objects by their checksum, so we can access them faster when // iterating through the server response below. Since the caller will invoke @@ -184,12 +192,20 @@ where let assemble_start = Instant::now(); - let request: AssembleDifsRequest<'_> = chunked_objects.iter().copied().collect(); + let requests: Vec> = chunked_objects + .chunks(ASSEMBLE_BATCH_SIZE) + .map(|batch| batch.iter().copied().collect()) + .collect(); let response = loop { - let response = - api.authenticated()? - .assemble_difs(options.org(), options.project(), &request)?; + let mut response = AssembleDifsResponse::new(); + for request in &requests { + response.extend(api.authenticated()?.assemble_difs( + options.org(), + options.project(), + request, + )?); + } let chunks_missing = response .values() From 85cef5650e867c77f57c36ed00914e9fbdcdcccc Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Mon, 2 Nov 2026 20:21:51 +0200 Subject: [PATCH 5/8] Drop the dxilpdb debug file type Only the DXIL binary is consumed on the server, so the shader PDB type goes until line numbers are supported. --- CHANGELOG.md | 2 +- src/commands/debug_files/find.rs | 2 +- src/commands/debug_files/upload.rs | 1 - src/utils/dif.rs | 9 +-- src/utils/dif_upload/mod.rs | 68 ++++--------------- src/utils/dif_upload/shader.rs | 33 +-------- .../debug_files-upload-help.trycmd | 4 +- .../_cases/upload_dif/upload_dif-help.trycmd | 4 +- .../upload_dsym/upload_dsym-help.trycmd | 4 +- 9 files changed, 22 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b68f16e113..592f18766c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -- Add `dxil` and `dxilpdb` debug file types to `debug-files upload` for GPU crash shader symbolication +- Add `dxil` debug file type to `debug-files upload` for GPU crash shader symbolication ### Improvements diff --git a/src/commands/debug_files/find.rs b/src/commands/debug_files/find.rs index 54675cf028..56cc295e58 100644 --- a/src/commands/debug_files/find.rs +++ b/src/commands/debug_files/find.rs @@ -150,7 +150,7 @@ fn find_ids( DifType::Breakpad => find_ids_for_breakpad(&dirent, &remaining), DifType::Proguard => find_ids_for_proguard(&dirent, &proguard_uuids), DifType::Jvm => find_ids_for_sourcebundle(&dirent, &remaining), - DifType::Wasm | DifType::Dxil | DifType::DxilPdb => None, + DifType::Wasm | DifType::Dxil => None, }) .flatten() .collect(); diff --git a/src/commands/debug_files/upload.rs b/src/commands/debug_files/upload.rs index 7c6ba0a876..514f59ec6d 100644 --- a/src/commands/debug_files/upload.rs +++ b/src/commands/debug_files/upload.rs @@ -248,7 +248,6 @@ pub fn execute(matches: &ArgMatches) -> Result<()> { "jvm" => upload.filter_format(DifFormat::Object(FileFormat::SourceBundle)), "wasm" => upload.filter_format(DifFormat::Object(FileFormat::Wasm)), "dxil" => upload.filter_format(DifFormat::Dxil), - "dxilpdb" => upload.filter_format(DifFormat::DxilPdb), "bcsymbolmap" => { upload.filter_format(DifFormat::BcSymbolMap); upload.filter_format(DifFormat::PList) diff --git a/src/utils/dif.rs b/src/utils/dif.rs index f9a3afe312..be14305c1e 100644 --- a/src/utils/dif.rs +++ b/src/utils/dif.rs @@ -23,7 +23,6 @@ pub enum DifType { Wasm, Jvm, Dxil, - DxilPdb, } impl DifType { @@ -40,7 +39,6 @@ impl DifType { DifType::Wasm => "wasm", DifType::Jvm => "jvm", DifType::Dxil => "dxil", - DifType::DxilPdb => "dxilpdb", } } @@ -57,7 +55,6 @@ impl DifType { DifType::Wasm, DifType::Jvm, DifType::Dxil, - DifType::DxilPdb, ] } @@ -74,7 +71,6 @@ impl DifType { "wasm", "jvm", "dxil", - "dxilpdb", ] } } @@ -101,7 +97,6 @@ impl str::FromStr for DifType { "wasm" => Ok(DifType::Wasm), "jvm" => Ok(DifType::Jvm), "dxil" => Ok(DifType::Dxil), - "dxilpdb" => Ok(DifType::DxilPdb), _ => bail!("Invalid debug info file type"), } } @@ -267,9 +262,7 @@ impl DifFile<'static> { Some(DifType::Breakpad) => DifFile::open_object(&path, FileFormat::Breakpad), Some(DifType::Proguard) => DifFile::open_proguard(&path), Some(DifType::Jvm) => DifFile::open_object(&path, FileFormat::SourceBundle), - Some(DifType::Dxil | DifType::DxilPdb) => { - Err(format_err!("Shader debug files can only be uploaded")) - } + Some(DifType::Dxil) => Err(format_err!("Shader debug files can only be uploaded")), None => DifFile::try_open(&path), }; diff --git a/src/utils/dif_upload/mod.rs b/src/utils/dif_upload/mod.rs index 8188034893..4ff676a548 100644 --- a/src/utils/dif_upload/mod.rs +++ b/src/utils/dif_upload/mod.rs @@ -59,7 +59,6 @@ enum ParsedDif<'a> { UuidMap, Il2Cpp, Dxil(Option), - DxilPdb, } impl<'slf, 'data: 'slf> AsSelf<'slf> for ParsedDif<'data> { @@ -149,17 +148,13 @@ impl<'data> DifMatch<'data> { }) } - fn from_shader( - kind: ShaderDifKind, + fn from_dxil( debug_id: DebugId, name: String, data: ByteView<'static>, info: Option, ) -> Self { - let dif = SelfCell::new(data, |_| match kind { - ShaderDifKind::Dxil => ParsedDif::Dxil(info), - ShaderDifKind::DxilPdb => ParsedDif::DxilPdb, - }); + let dif = SelfCell::new(data, |_| ParsedDif::Dxil(info)); Self { _backing: None, @@ -209,7 +204,6 @@ impl<'data> DifMatch<'data> { ParsedDif::UuidMap => None, ParsedDif::Il2Cpp => None, ParsedDif::Dxil(_) => None, - ParsedDif::DxilPdb => None, } } @@ -220,7 +214,6 @@ impl<'data> DifMatch<'data> { ParsedDif::UuidMap => DifFormat::PList, ParsedDif::Il2Cpp => DifFormat::Il2Cpp, ParsedDif::Dxil(_) => DifFormat::Dxil, - ParsedDif::DxilPdb => DifFormat::DxilPdb, } } @@ -232,7 +225,6 @@ impl<'data> DifMatch<'data> { ParsedDif::UuidMap => self.dif.owner(), ParsedDif::Il2Cpp => self.dif.owner(), ParsedDif::Dxil(_) => self.dif.owner(), - ParsedDif::DxilPdb => self.dif.owner(), } } @@ -304,7 +296,6 @@ impl Display for DifMatch<'_> { ParsedDif::UuidMap => String::from("uuidmap"), ParsedDif::Il2Cpp => String::from("il2cpp"), ParsedDif::Dxil(_) => String::from("dxil"), - ParsedDif::DxilPdb => String::from("dxilpdb"), }; write!( @@ -656,9 +647,9 @@ fn search_difs(options: &DifUpload) -> Result>> { debug!("trying to process {name}"); pb.set_message(&name); - if let Some(kind) = ShaderDifKind::detect(&buffer) { + if shader::is_dxil(&buffer) { if let Some(dif) = - collect_shader_dif(&source, name, buffer, options, kind, &mut shader_manifests)? + collect_dxil_dif(&source, name, buffer, options, &mut shader_manifests)? { collected.push(dif); } @@ -728,48 +719,15 @@ impl Display for AuxDifKind { } } -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum ShaderDifKind { - Dxil, - DxilPdb, -} - -impl ShaderDifKind { - /// Detects DXC shader debug files from their leading bytes. - /// - /// This must run before [`Archive::peek`], since shader PDBs carry the MSF - /// magic but cannot be parsed by symbolic. - fn detect(data: &[u8]) -> Option { - if shader::is_dxil(data) { - Some(Self::Dxil) - } else if shader::is_dxil_pdb(data) { - Some(Self::DxilPdb) - } else { - None - } - } -} - -impl Display for ShaderDifKind { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - ShaderDifKind::Dxil => write!(f, "DXIL"), - ShaderDifKind::DxilPdb => write!(f, "DXIL PDB"), - } - } -} - -/// Collects a DXC shader debug file into a [`DifMatch`]. +/// Collects a DXIL shader container into a [`DifMatch`]. /// -/// The debug ID is the 32-hex file stem. The code ID and shader group of DXIL -/// containers come from the manifest next to the file, loaded once per -/// directory into `manifests`. -fn collect_shader_dif<'a>( +/// The debug ID is the 32-hex file stem. The code ID and shader group come from +/// the manifest next to the file, loaded once per directory into `manifests`. +fn collect_dxil_dif<'a>( source: &DifSource<'_>, name: String, buffer: ByteView<'static>, options: &DifUpload, - kind: ShaderDifKind, manifests: &mut BTreeMap, ) -> Result>> { let file_stem = Path::new(&name) @@ -779,13 +737,13 @@ fn collect_shader_dif<'a>( let debug_id = match file_stem.parse() { Ok(debug_id) => debug_id, Err(_) => { - warn!("Skipping {kind} with invalid filename: {name}"); + warn!("Skipping DXIL with invalid filename: {name}"); return Ok(None); } }; - let info = match (kind, source) { - (ShaderDifKind::Dxil, DifSource::FileSystem(path)) => path.parent().and_then(|dir| { + let info = match source { + DifSource::FileSystem(path) => path.parent().and_then(|dir| { manifests .entry(dir.to_path_buf()) .or_insert_with(|| shader::load_manifest(dir)) @@ -795,7 +753,7 @@ fn collect_shader_dif<'a>( _ => None, }; - let dif = DifMatch::from_shader(kind, debug_id, name.clone(), buffer, info); + let dif = DifMatch::from_dxil(debug_id, name.clone(), buffer, info); if let Err(e) = options.validate_dif(&dif) { error::handle(&name, &e)?; @@ -1357,8 +1315,6 @@ pub enum DifFormat { Il2Cpp, /// A DXIL shader container produced by DXC. Dxil, - /// A DXC shader PDB. - DxilPdb, } /// Searches, processes and uploads debug information files (DIFs). diff --git a/src/utils/dif_upload/shader.rs b/src/utils/dif_upload/shader.rs index 09f8206874..a9e2b7cd76 100644 --- a/src/utils/dif_upload/shader.rs +++ b/src/utils/dif_upload/shader.rs @@ -1,4 +1,4 @@ -//! Detection and manifest lookup for DXC shader debug files. +//! Detection and manifest lookup for DXIL shader containers. use std::collections::HashMap; use std::fs; @@ -7,26 +7,12 @@ use std::path::Path; use log::warn; const MANIFEST_FILE_NAME: &str = "sentry-shader-manifest.tsv"; -const MSF_MAGIC: &[u8] = b"Microsoft C/C++ MSF 7.00\r\n\x1aDS"; -const ILDN_SCAN_LIMIT: usize = 64 * 1024; /// Returns `true` if the buffer is a DXIL container. pub fn is_dxil(data: &[u8]) -> bool { data.starts_with(b"DXBC") } -/// Returns `true` if the buffer is a DXC shader PDB. -/// -/// These carry the MSF magic but no DBI stream, so symbolic cannot parse them. -/// They are told apart from regular PDBs by the `ILDN` part near the start of -/// the file. -pub fn is_dxil_pdb(data: &[u8]) -> bool { - data.starts_with(MSF_MAGIC) - && data[..data.len().min(ILDN_SCAN_LIMIT)] - .windows(4) - .any(|window| window == b"ILDN") -} - /// Manifest metadata of a single `.dxil` file. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ShaderInfo { @@ -86,23 +72,6 @@ mod tests { assert!(!is_dxil(b"\x7fELF")); } - #[test] - fn test_is_dxil_pdb() { - let mut data = MSF_MAGIC.to_vec(); - data.extend_from_slice(&[0; 4096]); - assert!(!is_dxil_pdb(&data)); - - data.extend_from_slice(b"ILDN"); - assert!(is_dxil_pdb(&data)); - - let mut far = MSF_MAGIC.to_vec(); - far.resize(ILDN_SCAN_LIMIT, 0); - far.extend_from_slice(b"ILDN"); - assert!(!is_dxil_pdb(&far)); - - assert!(!is_dxil_pdb(b"DXBCILDN")); - } - #[test] fn test_parse_manifest() { let manifest = parse_manifest( diff --git a/tests/integration/_cases/debug_files/not_windows/debug_files-upload-help.trycmd b/tests/integration/_cases/debug_files/not_windows/debug_files-upload-help.trycmd index ccf5b25b59..fcbfd7a85a 100644 --- a/tests/integration/_cases/debug_files/not_windows/debug_files-upload-help.trycmd +++ b/tests/integration/_cases/debug_files/not_windows/debug_files-upload-help.trycmd @@ -16,8 +16,8 @@ Options: --auth-token Use the given Sentry auth token. -t, --type Only consider debug information files of the given type. By default, all types are considered. [possible values: bcsymbolmap, - breakpad, dsym, dxil, dxilpdb, elf, jvm, pdb, pe, portablepdb, - sourcebundle, wasm] + breakpad, dsym, dxil, elf, jvm, pdb, pe, portablepdb, sourcebundle, + wasm] --log-level Set the log output verbosity. [possible values: trace, debug, info, warn, error] --no-unwind Do not scan for stack unwinding information. Specify this flag for diff --git a/tests/integration/_cases/upload_dif/upload_dif-help.trycmd b/tests/integration/_cases/upload_dif/upload_dif-help.trycmd index 8f0df25c46..4737b1b409 100644 --- a/tests/integration/_cases/upload_dif/upload_dif-help.trycmd +++ b/tests/integration/_cases/upload_dif/upload_dif-help.trycmd @@ -16,8 +16,8 @@ Options: --auth-token Use the given Sentry auth token. -t, --type Only consider debug information files of the given type. By default, all types are considered. [possible values: bcsymbolmap, - breakpad, dsym, dxil, dxilpdb, elf, jvm, pdb, pe, portablepdb, - sourcebundle, wasm] + breakpad, dsym, dxil, elf, jvm, pdb, pe, portablepdb, sourcebundle, + wasm] --log-level Set the log output verbosity. [possible values: trace, debug, info, warn, error] --no-unwind Do not scan for stack unwinding information. Specify this flag for diff --git a/tests/integration/_cases/upload_dsym/upload_dsym-help.trycmd b/tests/integration/_cases/upload_dsym/upload_dsym-help.trycmd index b3a5d3e555..e5e68eeb77 100644 --- a/tests/integration/_cases/upload_dsym/upload_dsym-help.trycmd +++ b/tests/integration/_cases/upload_dsym/upload_dsym-help.trycmd @@ -16,8 +16,8 @@ Options: --auth-token Use the given Sentry auth token. -t, --type Only consider debug information files of the given type. By default, all types are considered. [possible values: bcsymbolmap, - breakpad, dsym, dxil, dxilpdb, elf, jvm, pdb, pe, portablepdb, - sourcebundle, wasm] + breakpad, dsym, dxil, elf, jvm, pdb, pe, portablepdb, sourcebundle, + wasm] --log-level Set the log output verbosity. [possible values: trace, debug, info, warn, error] --no-unwind Do not scan for stack unwinding information. Specify this flag for From ef7ced641e0619b0c8e920734d086821d4a20a8c Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Tue, 3 Nov 2026 14:40:11 +0200 Subject: [PATCH 6/8] Drop the lenient object kind parsing --- src/api/mod.rs | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index f02fae6511..4fda4ae5b3 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -36,7 +36,7 @@ use parking_lot::Mutex; use regex::{Captures, Regex}; use secrecy::{ExposeSecret as _, SecretString}; use serde::de::DeserializeOwned; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; use sha1_smol::Digest; use symbolic::common::DebugId; use symbolic::debuginfo::ObjectKind; @@ -1821,29 +1821,13 @@ pub struct SentryCliRelease { #[derive(Debug, Deserialize, Default)] pub struct DebugInfoData { - #[serde( - default, - rename = "type", - deserialize_with = "deserialize_lenient_object_kind" - )] + #[serde(default, rename = "type")] pub kind: Option, #[serde(default)] #[expect(dead_code)] pub features: Vec, } -/// Deserializes an object kind, mapping values unknown to symbolic (such as -/// server-side shader kinds) to `None` instead of failing. -fn deserialize_lenient_object_kind<'de, D>( - deserializer: D, -) -> std::result::Result, D::Error> -where - D: Deserializer<'de>, -{ - let kind: Option = Option::deserialize(deserializer)?; - Ok(kind.and_then(|k| k.parse().ok())) -} - /// Debug information files as processed and stored on the server. /// Can be dSYMs, ELF debug infos, Breakpad symbols, etc... #[derive(Debug, Deserialize)] From dbd3f365cc2df5e8734215e6886735616194d3b5 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Tue, 3 Nov 2026 14:40:12 +0200 Subject: [PATCH 7/8] Send batched assemble requests through one helper --- src/utils/chunks/upload.rs | 45 ++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/src/utils/chunks/upload.rs b/src/utils/chunks/upload.rs index 3d29de1288..000821f6b7 100644 --- a/src/utils/chunks/upload.rs +++ b/src/utils/chunks/upload.rs @@ -44,6 +44,26 @@ where } } +/// Sends the assemble requests one by one and merges their responses. +/// +/// The server processes every file in a request before answering, so files are +/// split into batches of `ASSEMBLE_BATCH_SIZE` to keep each request short. +fn assemble_difs( + requests: &[AssembleDifsRequest<'_>], + options: &ChunkOptions<'_>, +) -> Result { + let api = Api::current(); + let mut response = AssembleDifsResponse::new(); + for request in requests { + response.extend(api.authenticated()?.assemble_difs( + options.org(), + options.project(), + request, + )?); + } + Ok(response) +} + /// Calls the assemble endpoint and returns the state for every object along /// with info on missing chunks. /// @@ -56,16 +76,11 @@ fn try_assemble<'m, T>( where T: AsRef<[u8]> + Assemblable, { - let api = Api::current(); - let mut response = AssembleDifsResponse::new(); - for batch in objects.chunks(ASSEMBLE_BATCH_SIZE) { - let request: AssembleDifsRequest<'_> = batch.iter().collect(); - response.extend(api.authenticated()?.assemble_difs( - options.org(), - options.project(), - &request, - )?); - } + let requests: Vec> = objects + .chunks(ASSEMBLE_BATCH_SIZE) + .map(|batch| batch.iter().collect()) + .collect(); + let response = assemble_difs(&requests, options)?; // We map all objects by their checksum, so we can access them faster when // iterating through the server response below. Since the caller will invoke @@ -185,7 +200,6 @@ where \n{wide_bar} {pos}/{len}", ); - let api = Api::current(); let pb = ProgressBar::new(chunked_objects.len()); pb.set_style(progress_style); pb.set_prefix(">"); @@ -198,14 +212,7 @@ where .collect(); let response = loop { - let mut response = AssembleDifsResponse::new(); - for request in &requests { - response.extend(api.authenticated()?.assemble_difs( - options.org(), - options.project(), - request, - )?); - } + let response = assemble_difs(&requests, options)?; let chunks_missing = response .values() From 295efe237a7f1c005e7f829dca6379bb0bfea3f9 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Tue, 3 Nov 2026 17:09:43 +0200 Subject: [PATCH 8/8] Keep a single changelog entry for the dxil upload --- CHANGELOG.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 592f18766c..6cdf3dbbb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,7 @@ ### Features -- Add `dxil` debug file type to `debug-files upload` for GPU crash shader symbolication - -### Improvements - -- Send `debug-files upload` assemble requests in batches of 1000 files +- (debug-files) Add the `dxil` debug file type for uploading compiled shaders used to symbolicate GPU crashes ([#3489](https://github.com/getsentry/sentry-cli/pull/3489)) ### Internal Changes