feat(debug-files): Upload DXIL shader binaries

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Features
+
+- (debug-files) Add the `dxil` debug file type for uploading compiled shaders used to symbolicate GPU crashes ([#3489](https://github.com/getsentry/sentry-cli/pull/3489))
+
 ### Internal Changes
 
 - (docker) Bump the Docker image base from Alpine 3.22.0 to 3.22.7, whose `ca-certificates` already drops the distrusted eMudhra/emSign roots, and remove the `ca-certificates` version pin.

src/api/data_types/chunking/dif.rs

@@ -14,6 +14,10 @@ pub struct ChunkedDifRequest<'a> {
     pub name: Cow<'a, str>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub debug_id: Option<DebugId>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub code_id: Option<Cow<'a, str>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub shader_group: Option<Cow<'a, str>>,
     pub chunks: &'a [Digest],
     #[serde(skip)]
     hash: Digest,
@@ -28,6 +32,8 @@ impl<'a> ChunkedDifRequest<'a> {
             chunks,
             hash,
             debug_id: None,
+            code_id: None,
+            shader_group: None,
         }
     }
 
@@ -37,6 +43,20 @@ impl<'a> ChunkedDifRequest<'a> {
         self.debug_id = debug_id;
         self
     }
+
+    /// Set the provided code_id on the request, or clear it if
+    /// `None` is passed.
+    pub fn with_code_id(mut self, code_id: Option<Cow<'a, str>>) -> Self {
+        self.code_id = code_id;
+        self
+    }
+
+    /// Set the provided shader_group on the request, or clear it if
+    /// `None` is passed.
+    pub fn with_shader_group(mut self, shader_group: Option<Cow<'a, str>>) -> Self {
+        self.shader_group = shader_group;
+        self
+    }
 }
 
 #[derive(Debug, Deserialize)]

src/commands/debug_files/find.rs

@@ -150,7 +150,7 @@ fn find_ids(
                 DifType::Breakpad => find_ids_for_breakpad(&dirent, &remaining),
                 DifType::Proguard => find_ids_for_proguard(&dirent, &proguard_uuids),
                 DifType::Jvm => find_ids_for_sourcebundle(&dirent, &remaining),
-                DifType::Wasm => None,
+                DifType::Wasm | DifType::Dxil => None,
             })
             .flatten()
             .collect();

src/commands/debug_files/upload.rs

@@ -247,6 +247,7 @@ pub fn execute(matches: &ArgMatches) -> Result<()> {
             "portablepdb" => upload.filter_format(DifFormat::Object(FileFormat::PortablePdb)),
             "jvm" => upload.filter_format(DifFormat::Object(FileFormat::SourceBundle)),
             "wasm" => upload.filter_format(DifFormat::Object(FileFormat::Wasm)),
+            "dxil" => upload.filter_format(DifFormat::Dxil),
             "bcsymbolmap" => {
                 upload.filter_format(DifFormat::BcSymbolMap);
                 upload.filter_format(DifFormat::PList)

src/utils/chunks/mod.rs

@@ -29,6 +29,9 @@ use crate::utils::progress::{ProgressBar, ProgressBarMode, ProgressStyle};
 /// Timeout for polling all assemble endpoints.
 pub const ASSEMBLE_POLL_INTERVAL: Duration = Duration::from_millis(1000);
 
+/// Maximum number of files per assemble request.
+pub const ASSEMBLE_BATCH_SIZE: usize = 1000;
+
 /// A trait required by `BatchedSliceExt` to determine the logical size of a
 /// batch. Semantics and unit of the size depend on the batching use case,
 /// likely a number of bytes. See `BatchedSliceExt` for more information.

src/utils/chunks/types.rs

@@ -23,6 +23,18 @@ pub trait Assemblable {
     /// Returns the debug ID of the object.
     /// Types which do not have a debug ID should return `None`.
     fn debug_id(&self) -> Option<DebugId>;
+
+    /// Returns the code ID of the object, if it has one that is not
+    /// derived by the server from the file contents.
+    fn code_id(&self) -> Option<Cow<'_, str>> {
+        None
+    }
+
+    /// Returns the shader group of the object, if it is a shader debug file
+    /// that belongs to one.
+    fn shader_group(&self) -> Option<Cow<'_, str>> {
+        None
+    }
 }
 
 /// Chunked arbitrary data with computed SHA1 checksums.
@@ -105,6 +117,14 @@ where
     fn debug_id(&self) -> Option<DebugId> {
         (*self).debug_id()
     }
+
+    fn code_id(&self) -> Option<Cow<'_, str>> {
+        (*self).code_id()
+    }
+
+    fn shader_group(&self) -> Option<Cow<'_, str>> {
+        (*self).shader_group()
+    }
 }
 
 impl<T> Assemblable for Chunked<T>
@@ -118,6 +138,14 @@ where
     fn debug_id(&self) -> Option<DebugId> {
         self.object().debug_id()
     }
+
+    fn code_id(&self) -> Option<Cow<'_, str>> {
+        self.object().code_id()
+    }
+
+    fn shader_group(&self) -> Option<Cow<'_, str>> {
+        self.object().shader_group()
+    }
 }
 
 impl<'c, T> From<&'c Chunked<T>> for ChunkedDifRequest<'c>
@@ -127,5 +155,7 @@ where
     fn from(value: &'c Chunked<T>) -> Self {
         Self::new(value.name(), value.chunk_hashes(), value.checksum())
             .with_debug_id(value.debug_id())
+            .with_code_id(value.code_id())
+            .with_shader_group(value.shader_group())
     }
 }

src/utils/chunks/upload.rs

@@ -7,12 +7,16 @@ use anyhow::Result;
 use indicatif::ProgressStyle;
 
 use crate::{
-    api::{Api, AssembleDifsRequest, ChunkServerOptions, ChunkedFileState, DebugInfoFile},
+    api::{
+        Api, AssembleDifsRequest, AssembleDifsResponse, ChunkServerOptions, ChunkedFileState,
+        DebugInfoFile,
+    },
     utils::progress::ProgressBar,
 };
 
 use super::{
-    Assemblable, Chunk, ChunkOptions, Chunked, MissingObjectsInfo, ASSEMBLE_POLL_INTERVAL,
+    Assemblable, Chunk, ChunkOptions, Chunked, MissingObjectsInfo, ASSEMBLE_BATCH_SIZE,
+    ASSEMBLE_POLL_INTERVAL,
 };
 
 pub fn upload_chunked_objects<T>(
@@ -40,6 +44,26 @@ where
     }
 }
 
+/// Sends the assemble requests one by one and merges their responses.
+///
+/// The server processes every file in a request before answering, so files are
+/// split into batches of `ASSEMBLE_BATCH_SIZE` to keep each request short.
+fn assemble_difs(
+    requests: &[AssembleDifsRequest<'_>],
+    options: &ChunkOptions<'_>,
+) -> Result<AssembleDifsResponse> {
+    let api = Api::current();
+    let mut response = AssembleDifsResponse::new();
+    for request in requests {
+        response.extend(api.authenticated()?.assemble_difs(
+            options.org(),
+            options.project(),
+            request,
+        )?);
+    }
+    Ok(response)
+}
+
 /// Calls the assemble endpoint and returns the state for every object along
 /// with info on missing chunks.
 ///
@@ -52,12 +76,11 @@ fn try_assemble<'m, T>(
 where
     T: AsRef<[u8]> + Assemblable,
 {
-    let api = Api::current();
-    let request: AssembleDifsRequest<'_> = objects.iter().collect();
-
-    let response =
-        api.authenticated()?
-            .assemble_difs(options.org(), options.project(), &request)?;
+    let requests: Vec<AssembleDifsRequest<'_>> = objects
+        .chunks(ASSEMBLE_BATCH_SIZE)
+        .map(|batch| batch.iter().collect())
+        .collect();
+    let response = assemble_difs(&requests, options)?;
 
     // We map all objects by their checksum, so we can access them faster when
     // iterating through the server response below. Since the caller will invoke
@@ -177,19 +200,19 @@ where
          \n{wide_bar}  {pos}/{len}",
     );
 
-    let api = Api::current();
     let pb = ProgressBar::new(chunked_objects.len());
     pb.set_style(progress_style);
     pb.set_prefix(">");
 
     let assemble_start = Instant::now();
 
-    let request: AssembleDifsRequest<'_> = chunked_objects.iter().copied().collect();
+    let requests: Vec<AssembleDifsRequest<'_>> = chunked_objects
+        .chunks(ASSEMBLE_BATCH_SIZE)
+        .map(|batch| batch.iter().copied().collect())
+        .collect();
 
     let response = loop {
-        let response =
-            api.authenticated()?
-                .assemble_difs(options.org(), options.project(), &request)?;
+        let response = assemble_difs(&requests, options)?;
 
         let chunks_missing = response
             .values()

src/utils/dif.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 use std::str;
 use std::{fmt, iter};
 
-use anyhow::{bail, Context as _, Error, Result};
+use anyhow::{bail, format_err, Context as _, Error, Result};
 use proguard::ProguardMapping;
 use serde::ser::{SerializeStruct as _, Serializer};
 use serde::Serialize;
@@ -22,6 +22,7 @@ pub enum DifType {
     PortablePdb,
     Wasm,
     Jvm,
+    Dxil,
 }
 
 impl DifType {
@@ -37,6 +38,7 @@ impl DifType {
             DifType::Proguard => "proguard",
             DifType::Wasm => "wasm",
             DifType::Jvm => "jvm",
+            DifType::Dxil => "dxil",
         }
     }
 
@@ -52,6 +54,7 @@ impl DifType {
             DifType::Proguard,
             DifType::Wasm,
             DifType::Jvm,
+            DifType::Dxil,
         ]
     }
 
@@ -67,6 +70,7 @@ impl DifType {
             "proguard",
             "wasm",
             "jvm",
+            "dxil",
         ]
     }
 }
@@ -92,6 +96,7 @@ impl str::FromStr for DifType {
             "proguard" => Ok(DifType::Proguard),
             "wasm" => Ok(DifType::Wasm),
             "jvm" => Ok(DifType::Jvm),
+            "dxil" => Ok(DifType::Dxil),
             _ => bail!("Invalid debug info file type"),
         }
     }
@@ -257,6 +262,7 @@ impl DifFile<'static> {
             Some(DifType::Breakpad) => DifFile::open_object(&path, FileFormat::Breakpad),
             Some(DifType::Proguard) => DifFile::open_proguard(&path),
             Some(DifType::Jvm) => DifFile::open_object(&path, FileFormat::SourceBundle),
+            Some(DifType::Dxil) => Err(format_err!("Shader debug files can only be uploaded")),
             None => DifFile::try_open(&path),
         };