New Modpack System

backend/migrations/20261015000000_server_modpack_files.sql

@@ -0,0 +1,20 @@
+CREATE TABLE server_modpack_files (
+    server char(6) NOT NULL REFERENCES server_modpacks (server) ON DELETE CASCADE,
+    path text NOT NULL,
+    project varchar(128),
+    version varchar(128),
+    title varchar(255),
+    icon_url text,
+    page_url text,
+    PRIMARY KEY (server, path)
+);
+
+INSERT INTO server_modpack_files (server, path)
+SELECT server, unnest(files)
+FROM server_modpacks
+ON CONFLICT DO NOTHING;
+
+ALTER TABLE server_modpacks
+    DROP COLUMN files,
+    ADD COLUMN game_version varchar(64),
+    ADD COLUMN loader_version varchar(64);

backend/src/db/server/resources.rs

@@ -152,6 +152,16 @@ pub(crate) async fn delete_installed(
         > 0)
 }
 
+#[derive(Debug, Clone, Serialize, utoipa::ToSchema)]
+pub(crate) struct ModpackFileRecord {
+    pub(crate) path: String,
+    pub(crate) project: Option<String>,
+    pub(crate) version: Option<String>,
+    pub(crate) title: Option<String>,
+    pub(crate) icon_url: Option<String>,
+    pub(crate) page_url: Option<String>,
+}
+
 #[derive(Debug, Clone)]
 pub(crate) struct ModpackRecord {
     pub(crate) provider: ResourceProvider,
@@ -162,49 +172,72 @@ pub(crate) struct ModpackRecord {
     pub(crate) title: String,
     pub(crate) icon_url: Option<String>,
     pub(crate) page_url: Option<String>,
-    pub(crate) files: Vec<String>,
+    pub(crate) game_version: Option<String>,
+    pub(crate) loader_version: Option<String>,
+    pub(crate) files: Vec<ModpackFileRecord>,
     pub(crate) installed_at: i64,
 }
 
 pub(crate) async fn get_modpack(
     database: &PgPool,
     server: ServerId,
 ) -> Result<Option<ModpackRecord>, sqlx::Error> {
-    Ok(query!(
+    let Some(record) = query!(
         r#"SELECT provider, project, version, version_number, loader, title,
-                  icon_url, page_url, files, installed_at
+                  icon_url, page_url, game_version, loader_version, installed_at
            FROM server_modpacks
            WHERE server = $1"#,
         server as ServerId
     )
     .fetch_optional(database)
     .await?
-    .and_then(|record| {
-        Some(ModpackRecord {
-            provider: ResourceProvider::parse_str(&record.provider)?,
-            project: record.project,
-            version: record.version,
-            version_number: record.version_number,
-            loader: record.loader.as_deref().and_then(ResourceLoader::parse_str),
-            title: record.title,
-            icon_url: record.icon_url,
-            page_url: record.page_url,
-            files: record.files,
-            installed_at: record.installed_at.timestamp(),
-        })
+    else {
+        return Ok(None);
+    };
+
+    let Some(provider) = ResourceProvider::parse_str(&record.provider) else {
+        return Ok(None);
+    };
+
+    let files = query_as!(
+        ModpackFileRecord,
+        r#"SELECT path, project, version, title, icon_url, page_url
+           FROM server_modpack_files
+           WHERE server = $1
+           ORDER BY path"#,
+        server as ServerId
+    )
+    .fetch_all(database)
+    .await?;
+
+    Ok(Some(ModpackRecord {
+        provider,
+        project: record.project,
+        version: record.version,
+        version_number: record.version_number,
+        loader: record.loader.as_deref().and_then(ResourceLoader::parse_str),
+        title: record.title,
+        icon_url: record.icon_url,
+        page_url: record.page_url,
+        game_version: record.game_version,
+        loader_version: record.loader_version,
+        files,
+        installed_at: record.installed_at.timestamp(),
     }))
 }
 
-pub(crate) async fn upsert_modpack(
+pub(crate) async fn replace_modpack(
     database: &PgPool,
     server: ServerId,
     modpack: &ModpackRecord,
 ) -> Result<(), sqlx::Error> {
+    let mut tx = database.begin().await?;
+
     query!(
         r#"INSERT INTO server_modpacks
                (server, provider, project, version, version_number, loader, title,
-                icon_url, page_url, files)
-           VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+                icon_url, page_url, game_version, loader_version)
+           VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
            ON CONFLICT (server) DO UPDATE SET
                provider = excluded.provider,
                project = excluded.project,
@@ -214,7 +247,8 @@ pub(crate) async fn upsert_modpack(
                title = excluded.title,
                icon_url = excluded.icon_url,
                page_url = excluded.page_url,
-               files = excluded.files,
+               game_version = excluded.game_version,
+               loader_version = excluded.loader_version,
                installed_at = now()"#,
         server as ServerId,
         modpack.provider.as_str(),
@@ -225,12 +259,53 @@ pub(crate) async fn upsert_modpack(
         modpack.title,
         modpack.icon_url,
         modpack.page_url,
-        &modpack.files
+        modpack.game_version,
+        modpack.loader_version
     )
-    .execute(database)
+    .execute(&mut *tx)
     .await?;
 
-    Ok(())
+    query!(
+        "DELETE FROM server_modpack_files WHERE server = $1",
+        server as ServerId
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    let mut paths = Vec::with_capacity(modpack.files.len());
+    let mut projects = Vec::with_capacity(modpack.files.len());
+    let mut versions = Vec::with_capacity(modpack.files.len());
+    let mut titles = Vec::with_capacity(modpack.files.len());
+    let mut icons = Vec::with_capacity(modpack.files.len());
+    let mut pages = Vec::with_capacity(modpack.files.len());
+    for file in &modpack.files {
+        paths.push(file.path.clone());
+        projects.push(file.project.clone());
+        versions.push(file.version.clone());
+        titles.push(file.title.clone());
+        icons.push(file.icon_url.clone());
+        pages.push(file.page_url.clone());
+    }
+
+    query!(
+        r#"INSERT INTO server_modpack_files
+               (server, path, project, version, title, icon_url, page_url)
+           SELECT $1, *
+           FROM UNNEST($2::text[], $3::varchar[], $4::varchar[], $5::varchar[],
+                       $6::text[], $7::text[])
+           ON CONFLICT DO NOTHING"#,
+        server as ServerId,
+        &paths,
+        &projects as &[Option<String>],
+        &versions as &[Option<String>],
+        &titles as &[Option<String>],
+        &icons as &[Option<String>],
+        &pages as &[Option<String>]
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    tx.commit().await
 }
 
 pub(crate) async fn delete_modpack(database: &PgPool, server: ServerId) -> Result<(), sqlx::Error> {

backend/src/resources/curseforge.rs

@@ -2,9 +2,9 @@ use reqwest::{Client, RequestBuilder};
 use serde::Deserialize;
 
 use super::{
-    ResolvedResource, ResourceError, ResourceKind, ResourceLoader, ResourceProject,
+    ProjectDetails, ResolvedResource, ResourceError, ResourceKind, ResourceLoader, ResourceProject,
     ResourceProvider, ResourceQuery, ResourceVersion, ResourceVersionType, encode_segment,
-    fetch_json, sanitize_file_name,
+    fetch_json, safe_page_url, sanitize_file_name,
 };
 
 const API: &str = "https://api.curseforge.com/v1";
@@ -134,6 +134,12 @@ fn count(value: f64) -> u64 {
     }
 }
 
+#[derive(Debug, serde::Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ModLookup {
+    mod_ids: Vec<i64>,
+}
+
 const fn version_type(release_type: u8) -> ResourceVersionType {
     match release_type {
         2 => ResourceVersionType::Beta,
@@ -215,6 +221,7 @@ fn to_version(file: File) -> ResourceVersion {
             .collect(),
         downloads: count(file.download_count),
         published_at: file.file_date,
+        changelog: None,
         file_name: Some(file.file_name),
         downloadable: file.download_url.is_some(),
     }
@@ -325,3 +332,35 @@ pub(super) async fn resolve(
         file_name: sanitize_file_name(&file.file_name, "curseforge-resource.jar"),
     })
 }
+
+pub(super) async fn projects(
+    client: &Client,
+    api_key: Option<&str>,
+    ids: &[String],
+) -> Result<Vec<ProjectDetails>, ResourceError> {
+    let mod_ids = ids
+        .iter()
+        .filter_map(|id| id.parse::<i64>().ok())
+        .collect::<Vec<_>>();
+    if mod_ids.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let response: Paged<Mod> = fetch_json(
+        PROVIDER,
+        authorized(client.post(format!("{API}/mods")), api_key)?.json(&ModLookup { mod_ids }),
+    )
+    .await?;
+
+    Ok(response
+        .data
+        .into_iter()
+        .map(|entry| ProjectDetails {
+            provider: PROVIDER,
+            id: entry.id.to_string(),
+            title: entry.name,
+            icon_url: safe_page_url(entry.logo.and_then(|logo| logo.url)),
+            page_url: safe_page_url(entry.links.and_then(|links| links.website_url)),
+        })
+        .collect())
+}

backend/src/resources/hangar.rs

@@ -148,6 +148,7 @@ fn to_version(project: &str, version: Version) -> ResourceVersion {
             .map(|stats| stats.total_downloads)
             .unwrap_or_default(),
         published_at: version.created_at,
+        changelog: None,
         file_name,
         downloadable: download.is_some_and(|download| {
             download.download_url.is_some() || download.external_url.is_some()

backend/src/resources/mod.rs

@@ -11,6 +11,7 @@ use redis::aio::ConnectionManager;
 use reqwest::{Client, RequestBuilder, StatusCode, Url};
 use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use tracing::{debug, error};
 
 use crate::service::{
@@ -25,6 +26,8 @@ use crate::service::{
 const SEARCH_TTL: u64 = 300;
 const VERSIONS_TTL: u64 = 300;
 const RESOLVE_TTL: u64 = 900;
+const DETAILS_TTL: u64 = 3600;
+const LOOKUP_CHUNK: usize = 100;
 
 pub(crate) const MAX_SEARCH_LIMIT: u8 = 50;
 pub(crate) const MAX_QUERY_LENGTH: usize = 120;
@@ -218,10 +221,27 @@ pub(crate) struct ResourceVersion {
     pub(crate) loaders: Vec<String>,
     pub(crate) downloads: u64,
     pub(crate) published_at: Option<String>,
+    #[serde(default)]
+    pub(crate) changelog: Option<String>,
     pub(crate) file_name: Option<String>,
     pub(crate) downloadable: bool,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
+pub(crate) struct ProjectDetails {
+    pub(crate) provider: ResourceProvider,
+    pub(crate) id: String,
+    pub(crate) title: String,
+    pub(crate) icon_url: Option<String>,
+    pub(crate) page_url: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct ResourceDependency {
+    pub(crate) project: Option<String>,
+    pub(crate) version: Option<String>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub(crate) struct ResolvedResource {
     pub(crate) url: Url,
@@ -542,4 +562,68 @@ impl ResourceClient<'_> {
 
         cache::cached(&mut self.cache, &key, RESOLVE_TTL, fetch).await
     }
+
+    pub(crate) async fn dependencies(
+        &mut self,
+        provider: ResourceProvider,
+        version: &str,
+    ) -> Result<Vec<ResourceDependency>, ResourceError> {
+        if provider != ResourceProvider::Modrinth {
+            return Ok(Vec::new());
+        }
+        validate_id(version, 128)?;
+
+        let key = format!("dependencies:{}:{version}", provider.as_str());
+        let http = self.http;
+        cache::cached(
+            &mut self.cache,
+            &key,
+            RESOLVE_TTL,
+            modrinth::dependencies(http, version),
+        )
+        .await
+    }
+
+    pub(crate) async fn identify_modrinth(
+        &self,
+        sha1: &[String],
+    ) -> Result<HashMap<String, (String, String)>, ResourceError> {
+        let mut found = HashMap::new();
+        for chunk in sha1.chunks(LOOKUP_CHUNK) {
+            found.extend(modrinth::identify(self.http, chunk).await?);
+        }
+        Ok(found)
+    }
+
+    pub(crate) async fn projects(
+        &mut self,
+        provider: ResourceProvider,
+        ids: &[String],
+    ) -> Result<Vec<ProjectDetails>, ResourceError> {
+        if !self.enabled(provider) {
+            return Err(ResourceError::ProviderDisabled);
+        }
+
+        let mut details = Vec::new();
+        for chunk in ids.chunks(LOOKUP_CHUNK) {
+            for id in chunk {
+                validate_id(id, 128)?;
+            }
+            let mut sorted = chunk.to_vec();
+            sorted.sort_unstable();
+            let key = format!("projects:{}:{}", provider.as_str(), sorted.join(","));
+            let (http, api_key) = (self.http, self.curseforge_api_key);
+            let fetch = async move {
+                match provider {
+                    ResourceProvider::Modrinth => modrinth::projects(http, &sorted).await,
+                    ResourceProvider::Curseforge => {
+                        curseforge::projects(http, api_key, &sorted).await
+                    }
+                    ResourceProvider::Hangar | ResourceProvider::Spigot => Ok(Vec::new()),
+                }
+            };
+            details.extend(cache::cached(&mut self.cache, &key, DETAILS_TTL, fetch).await?);
+        }
+        Ok(details)
+    }
 }