diff --git a/.sqlx/query-94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a.json b/.sqlx/query-fa1aa2f18dc9286a9d2f9072d0ef6e9538eb43ef12c45ac7e9c6f0d6314134b9.json similarity index 85% rename from .sqlx/query-94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a.json rename to .sqlx/query-fa1aa2f18dc9286a9d2f9072d0ef6e9538eb43ef12c45ac7e9c6f0d6314134b9.json index 3933afc..53467e3 100644 --- a/.sqlx/query-94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a.json +++ b/.sqlx/query-fa1aa2f18dc9286a9d2f9072d0ef6e9538eb43ef12c45ac7e9c6f0d6314134b9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n\t\t\t\t mac as \"mac: _\", win as \"win: _\", android as \"android: _\", ios as \"ios: _\",\n\t\t\t\t tag, created_at, commit_hash, prerelease\n\t\t\t FROM geode_versions\n\t\t\t\t WHERE tag = $1", + "query": "SELECT\n\t\t\t\t mac as \"mac: _\", win as \"win: _\", android as \"android: _\", ios as \"ios: _\",\n\t\t\t\t tag, created_at, commit_hash, prerelease, resources_url, resources_hash\n\t\t\t FROM geode_versions\n\t\t\t\t WHERE tag = $1", "describe": { "columns": [ { @@ -178,6 +178,28 @@ "name": "prerelease" } } + }, + { + "ordinal": 8, + "name": "resources_url", + "type_info": "Text", + "origin": { + "Table": { + "table": "geode_versions", + "name": "resources_url" + } + } + }, + { + "ordinal": 9, + "name": "resources_hash", + "type_info": "Text", + "origin": { + "Table": { + "table": "geode_versions", + "name": "resources_hash" + } + } } ], "parameters": { @@ -193,8 +215,10 @@ false, false, false, - false + false, + true, + true ] }, - "hash": "94a9f2bc14d5a6c21eb7f638e5d70bcdc7112d9ad0d153aaad6bf7b00e27d74a" + "hash": "fa1aa2f18dc9286a9d2f9072d0ef6e9538eb43ef12c45ac7e9c6f0d6314134b9" } diff --git a/migrations/20260721182316_add_download_link_to_geode_version.down.sql b/migrations/20260721182316_add_download_link_to_geode_version.down.sql new file mode 100644 index 0000000..8aa986d --- /dev/null +++ b/migrations/20260721182316_add_download_link_to_geode_version.down.sql @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS geode_version_download; + +ALTER TABLE geode_versions + DROP COLUMN resources_url, + DROP COLUMN resources_hash; diff --git a/migrations/20260721182316_add_download_link_to_geode_version.up.sql b/migrations/20260721182316_add_download_link_to_geode_version.up.sql new file mode 100644 index 0000000..1e62f4e --- /dev/null +++ b/migrations/20260721182316_add_download_link_to_geode_version.up.sql @@ -0,0 +1,11 @@ +CREATE TABLE geode_version_download ( + tag TEXT NOT NULL REFERENCES geode_versions(tag) ON DELETE CASCADE, + platform TEXT NOT NULL, + url TEXT NOT NULL, + hash TEXT NOT NULL, + PRIMARY KEY (tag, platform) +); + +ALTER TABLE geode_versions + ADD COLUMN resources_url TEXT, + ADD COLUMN resources_hash TEXT; diff --git a/src/config.rs b/src/config.rs index 2fe8938..bfa948f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -28,7 +28,7 @@ pub struct AppData { static_storage: PublicDisk, public_storage: PublicDisk, private_storage: PrivateDisk, - mod_storage: Option, + cdn_storage: Option, disable_downloads: bool, max_download_mb: u32, port: u16, @@ -79,7 +79,7 @@ pub async fn build_config() -> anyhow::Result { .time_to_live(Duration::from_mins(10)) .build(); - let mod_storage = if let Some(s3_config) = S3Configuration::from_env()? { + let cdn_storage = if let Some(s3_config) = S3Configuration::from_env()? { let backend = Arc::new(S3Backend::new(&s3_config)?); Some(PublicDisk::new(backend, s3_config.public_url)) } else { @@ -113,7 +113,7 @@ pub async fn build_config() -> anyhow::Result { format!("{app_url}/storage"), ), private_storage: PrivateDisk::new(Arc::new(LocalBackend::new("storage/private"))), - mod_storage, + cdn_storage, disable_downloads, max_download_mb, port, @@ -192,8 +192,8 @@ impl AppData { &self.private_storage } - pub fn mod_storage(&self) -> Option<&PublicDisk> { - self.mod_storage.as_ref() + pub fn cdn_storage(&self) -> Option<&PublicDisk> { + self.cdn_storage.as_ref() } pub fn mods_cache(&self) -> &Cache>> { diff --git a/src/database/repository/geode_versions.rs b/src/database/repository/geode_versions.rs new file mode 100644 index 0000000..934634c --- /dev/null +++ b/src/database/repository/geode_versions.rs @@ -0,0 +1,44 @@ +use sqlx::PgConnection; + +use crate::database::DatabaseError; + +#[tracing::instrument(skip_all, fields(tag = %tag, platform = %platform, url = %url, hash = %hash))] +pub async fn upsert_download( + tag: &str, + platform: &str, + url: &str, + hash: &str, + conn: &mut PgConnection, +) -> Result<(), DatabaseError> { + sqlx::query( + "INSERT INTO geode_version_download (tag, platform, url, hash) VALUES ($1, $2, $3, $4) + ON CONFLICT (tag, platform) DO UPDATE SET url = $3, hash = $4", + ) + .bind(tag) + .bind(platform) + .bind(url) + .bind(hash) + .execute(&mut *conn) + .await + .inspect_err(|e| tracing::error!("{:?}", e))?; + + Ok(()) +} + +#[tracing::instrument(skip_all, fields(tag = %tag, url = %url, hash = %hash))] +pub async fn update_resources_download( + tag: &str, + url: &str, + hash: &str, + conn: &mut PgConnection, +) -> Result<(), DatabaseError> { + sqlx::query("UPDATE geode_versions SET resources_url = $2, resources_hash = $3 WHERE tag = $1") + .bind(tag) + .bind(url) + .bind(hash) + .execute(&mut *conn) + .await + .inspect_err(|e| tracing::error!("{:?}", e))?; + + Ok(()) +} diff --git a/src/database/repository/mod.rs b/src/database/repository/mod.rs index 7253922..e15fe17 100644 --- a/src/database/repository/mod.rs +++ b/src/database/repository/mod.rs @@ -2,6 +2,7 @@ pub mod auth_tokens; pub mod dependencies; pub mod deprecations; pub mod developers; +pub mod geode_versions; pub mod github_login_attempts; pub mod github_web_logins; pub mod incompatibilities; diff --git a/src/endpoints/loader.rs b/src/endpoints/loader.rs index 3434566..4237bce 100644 --- a/src/endpoints/loader.rs +++ b/src/endpoints/loader.rs @@ -6,6 +6,7 @@ use utoipa::{IntoParams, ToSchema}; use sqlx::Acquire; use crate::endpoints::ApiError; +use crate::s3_worker::S3WorkerTask; use crate::{ config::AppData, extractors::auth::Auth, @@ -122,10 +123,12 @@ pub async fn create_version( return Err(ApiError::Authorization); } + let tag = payload.tag.trim_start_matches('v').to_string(); + let mut tx = pool.begin().await?; LoaderVersion::create_version( LoaderVersionCreate { - tag: payload.tag.trim_start_matches('v').to_string(), + tag: tag.clone(), prerelease: payload.prerelease, commit_hash: payload.commit_hash.clone(), win: payload.gd.win, @@ -139,6 +142,8 @@ pub async fn create_version( tx.commit().await?; + data.send_s3_task(S3WorkerTask::UploadLoader { tag }); + Ok(HttpResponse::NoContent()) } diff --git a/src/endpoints/mod_versions.rs b/src/endpoints/mod_versions.rs index 3d48b99..8955888 100644 --- a/src/endpoints/mod_versions.rs +++ b/src/endpoints/mod_versions.rs @@ -241,7 +241,7 @@ pub async fn download_version( let url = mod_version .managed_download_link .as_deref() - .take_if(|_| data.mod_storage().is_some()) + .take_if(|_| data.cdn_storage().is_some()) .unwrap_or(&mod_version.download_link); if data.disable_downloads() || mod_version.status != ModVersionStatusEnum::Accepted { diff --git a/src/main.rs b/src/main.rs index 96d8d64..8db3b40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,8 +39,8 @@ async fn main() -> anyhow::Result<()> { let app_data = config::build_config().await?; app_data.static_storage().init().await?; app_data.private_storage().init().await?; - if let Some(mod_storage) = app_data.mod_storage() { - mod_storage.init().await?; + if let Some(cdn_storage) = app_data.cdn_storage() { + cdn_storage.init().await?; } if cli::maybe_cli(&app_data).await? { diff --git a/src/s3_worker.rs b/src/s3_worker.rs index 7a34963..750e04a 100644 --- a/src/s3_worker.rs +++ b/src/s3_worker.rs @@ -2,10 +2,19 @@ use std::time::Duration; use actix_web::web; use bytes::Bytes; +use reqwest::StatusCode; use crate::{ - config::AppData, database::repository::mod_versions::update_managed_download_link, mod_zip, - types::models::mod_gd_version::GDVersionEnum, + config::AppData, + database::repository::{ + geode_versions::{update_resources_download, upsert_download}, + mod_versions::update_managed_download_link, + }, + mod_zip, + types::models::{ + loader_version::{LoaderDownload, LoaderDownloads}, + mod_gd_version::GDVersionEnum, + }, }; pub enum S3WorkerTask { @@ -15,14 +24,90 @@ pub enum S3WorkerTask { version: String, version_id: i32, }, + + UploadLoader { + tag: String, + }, } fn path_for_mod(mod_id: &str, version: &str) -> String { format!("mods/{mod_id}/{version}/{mod_id}.geode") } +fn path_for_loader(tag: &str, platform: &str) -> String { + format!("geode/{tag}/geode-v{tag}-{platform}.zip") +} + +fn path_for_resources(tag: &str) -> String { + format!("geode/{tag}/resources.zip") +} + +fn github_url_for_loader(tag: &str, platform: &str) -> String { + format!( + "https://github.com/geode-sdk/geode/releases/download/v{tag}/geode-v{tag}-{platform}.zip" + ) +} + +fn github_url_for_resources(tag: &str) -> String { + format!("https://github.com/geode-sdk/geode/releases/download/v{tag}/resources.zip") +} + +async fn migrate_geode_version_opt( + data: &AppData, + db: &mut sqlx::Transaction<'_, sqlx::Postgres>, + tag: &str, + platform: &str, +) -> anyhow::Result> { + let storage = data.cdn_storage().expect("mod storage must be set by now"); + let (github_url, new_path) = match platform { + "resources" => (github_url_for_resources(tag), path_for_resources(tag)), + _ => ( + github_url_for_loader(tag, platform), + path_for_loader(tag, platform), + ), + }; + + let resp = data.http_client().get(&github_url).send().await?; + + if resp.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + + let bytes = resp.error_for_status()?.bytes().await?; + let public_url = storage.asset_url(&new_path); + let hash = sha256::digest(&bytes[..]); + storage.store(&new_path, &bytes).await?; + + if platform == "resources" { + update_resources_download(tag, &public_url, &hash, db).await?; + } else { + upsert_download(tag, platform, &public_url, &hash, db).await?; + } + + Ok(Some(LoaderDownload { + url: public_url, + hash, + })) +} + +async fn migrate_geode_version( + data: &AppData, + db: &mut sqlx::Transaction<'_, sqlx::Postgres>, + tag: &str, + platform: &str, +) -> anyhow::Result { + match migrate_geode_version_opt(data, db, tag, platform).await? { + Some(download) => Ok(download), + None => Err(anyhow::anyhow!( + "Geode version {} for platform '{}' not found on GitHub", + tag, + platform + )), + } +} + async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> { - let storage = data.mod_storage().expect("mod storage must be set by now"); + let storage = data.cdn_storage().expect("mod storage must be set by now"); match task { S3WorkerTask::UploadMod { @@ -47,6 +132,36 @@ async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> public_url ); } + + S3WorkerTask::UploadLoader { tag } => { + tracing::info!("Preparing to upload Geode v{tag} to S3"); + + let mut tx = data.db().begin().await?; + + let ios = match migrate_geode_version_opt(data, &mut tx, &tag, "ios").await? { + Some(download) => download, + None => { + tracing::warn!( + "Geode version {} for iOS not found on GitHub, skipping iOS", + tag + ); + LoaderDownload::default() + } + }; + + let downloads = LoaderDownloads { + win: migrate_geode_version(data, &mut tx, &tag, "win").await?, + mac: migrate_geode_version(data, &mut tx, &tag, "mac").await?, + android32: migrate_geode_version(data, &mut tx, &tag, "android32").await?, + android64: migrate_geode_version(data, &mut tx, &tag, "android64").await?, + ios, + resources: migrate_geode_version(data, &mut tx, &tag, "resources").await?, + }; + + tx.commit().await?; + + tracing::info!("Uploaded new loader release to S3: {downloads:?}"); + } } Ok(()) @@ -54,7 +169,7 @@ async fn process_task(data: &AppData, task: S3WorkerTask) -> anyhow::Result<()> async fn cleanup_old_s3_files(data: &AppData) -> anyhow::Result<()> { let supported_gd = GDVersionEnum::supported_for_storage(); - let storage = data.mod_storage().expect("mod storage must be set by now"); + let storage = data.cdn_storage().expect("mod storage must be set by now"); let mut db = data.db().acquire().await?; @@ -177,8 +292,31 @@ async fn migrate_existing_mods_to_s3(data: &AppData) -> anyhow::Result<()> { Ok(()) } +async fn migrate_loader_versions_to_s3(data: &AppData) -> anyhow::Result<()> { + let mut db = data.db().acquire().await?; + + let versions: Vec = sqlx::query_scalar( + "SELECT gv.tag FROM geode_versions gv WHERE NOT EXISTS ( + SELECT 1 FROM geode_version_download gvd WHERE gvd.tag = gv.tag + )", + ) + .fetch_all(&mut *db) + .await + .map_err(|e| anyhow::anyhow!("{e}"))?; + + tracing::info!("Migrating {} Geode releases to S3", versions.len()); + + for tag in versions { + if let Err(e) = process_task(data, S3WorkerTask::UploadLoader { tag: tag.clone() }).await { + tracing::error!("error migrating Geode release {} to S3: {e:?}", tag); + } + } + + Ok(()) +} + pub async fn run_s3_worker(data: web::Data) { - if data.mod_storage().is_none() { + if data.cdn_storage().is_none() { return; } @@ -192,6 +330,13 @@ pub async fn run_s3_worker(data: web::Data) { } }); + let s_data2 = data.clone(); + tokio::spawn(async move { + if let Err(e) = migrate_loader_versions_to_s3(&s_data2).await { + tracing::error!("Error migrating loader versions to S3: {:?}", e); + } + }); + let mut interval = tokio::time::interval(Duration::from_mins(30)); loop { diff --git a/src/types/models/loader_version.rs b/src/types/models/loader_version.rs index 66883ef..43c71a3 100644 --- a/src/types/models/loader_version.rs +++ b/src/types/models/loader_version.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use crate::{ database::DatabaseError, types::{ @@ -6,7 +8,7 @@ use crate::{ }, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use sqlx::{ @@ -25,6 +27,22 @@ pub struct LoaderVersionCreate { pub ios: Option, } +#[derive(Serialize, Deserialize, Default, Debug, ToSchema)] +pub struct LoaderDownload { + pub url: String, + pub hash: String, +} + +#[derive(Serialize, Deserialize, Default, Debug, ToSchema)] +pub struct LoaderDownloads { + pub win: LoaderDownload, + pub mac: LoaderDownload, + pub android32: LoaderDownload, + pub android64: LoaderDownload, + pub ios: LoaderDownload, + pub resources: LoaderDownload, +} + #[derive(Serialize, Debug, ToSchema)] pub struct LoaderVersion { pub version: String, @@ -34,6 +52,7 @@ pub struct LoaderVersion { pub commit_hash: String, #[serde(with = "chrono_dt_secs")] pub created_at: DateTime, + pub downloads: LoaderDownloads, } #[derive(sqlx::FromRow, Debug)] @@ -46,6 +65,17 @@ pub struct LoaderVersionGetOne { pub win: Option, pub android: Option, pub ios: Option, + + pub resources_url: Option, + pub resources_hash: Option, +} + +#[derive(sqlx::FromRow, Debug)] +pub struct GeodeVersionDownload { + pub tag: String, + pub platform: String, + pub url: String, + pub hash: String, } pub struct GetVersionsQuery { @@ -54,14 +84,86 @@ pub struct GetVersionsQuery { pub prerelease: bool, } +fn github_url(tag: &str, platform: &str) -> String { + format!( + "https://github.com/geode-sdk/geode/releases/download/v{tag}/geode-v{tag}-{platform}.zip" + ) +} + +fn github_resources_url(tag: &str) -> String { + format!("https://github.com/geode-sdk/geode/releases/download/v{tag}/resources.zip") +} + +impl LoaderDownload { + pub fn new_github(tag: &str, platform: &str) -> Self { + // this should only be called for versions that weren't migrated to S3 yet, + // temporarily serve the GitHub URLs and tell the client to not verify hashes + LoaderDownload { + url: github_url(tag, platform), + hash: String::new(), + } + } + + pub fn new_github_resources(tag: &str) -> Self { + LoaderDownload { + url: github_resources_url(tag), + hash: String::new(), + } + } +} + +fn build_downloads( + version: &LoaderVersionGetOne, + managed: Vec, +) -> LoaderDownloads { + let mut out = LoaderDownloads { + win: LoaderDownload::new_github(&version.tag, "windows"), + mac: LoaderDownload::new_github(&version.tag, "macos"), + android32: LoaderDownload::new_github(&version.tag, "android32"), + android64: LoaderDownload::new_github(&version.tag, "android64"), + ios: LoaderDownload::new_github(&version.tag, "ios"), + resources: LoaderDownload::new_github_resources(&version.tag), + }; + + for d in managed { + let download = LoaderDownload { + url: d.url, + hash: d.hash, + }; + + match d.platform.as_str() { + "win" => out.win = download, + "mac" => out.mac = download, + "android32" => out.android32 = download, + "android64" => out.android64 = download, + "ios" => out.ios = download, + _ => {} + } + } + + if let Some(url) = version.resources_url.clone() + && let Some(hash) = version.resources_hash.clone() + { + out.resources = LoaderDownload { url, hash }; + } + + out +} + impl LoaderVersionGetOne { - pub fn into_loader_version(self) -> LoaderVersion { + pub fn into_loader_version( + self, + managed_downloads: Vec, + ) -> LoaderVersion { + let downloads = build_downloads(&self, managed_downloads); + LoaderVersion { tag: format!("v{}", self.tag), version: self.tag, prerelease: self.prerelease, created_at: self.created_at, commit_hash: self.commit_hash, + downloads, gd: DetailedGDVersion { win: self.win, mac: self.mac, @@ -77,6 +179,40 @@ impl LoaderVersionGetOne { } impl LoaderVersion { + pub async fn get_downloads_for_tag( + tag: &str, + pool: &mut PgConnection, + ) -> Result, DatabaseError> { + Ok(sqlx::query_as::<_, GeodeVersionDownload>( + "SELECT tag, platform, url, hash FROM geode_version_download WHERE tag = $1", + ) + .bind(tag) + .fetch_all(&mut *pool) + .await?) + } + + pub async fn get_downloads_for_tags( + tags: &[String], + pool: &mut PgConnection, + ) -> Result>, DatabaseError> { + if tags.is_empty() { + return Ok(HashMap::new()); + } + + let rows = sqlx::query_as::<_, GeodeVersionDownload>( + "SELECT tag, platform, url, hash FROM geode_version_download WHERE tag = ANY($1)", + ) + .bind(tags) + .fetch_all(&mut *pool) + .await?; + + let mut map = HashMap::<_, Vec<_>>::with_capacity(rows.len()); + for row in rows { + map.entry(row.tag.clone()).or_default().push(row); + } + Ok(map) + } + #[tracing::instrument(skip_all, fields(gd = ?gd, platform = ?platform, accept_prereleases = %accept_prereleases))] pub async fn get_latest( gd: Option, @@ -86,7 +222,7 @@ impl LoaderVersion { ) -> Result, DatabaseError> { let mut query_builder: QueryBuilder = QueryBuilder::new( r#"SELECT - mac, win, android, ios, tag, commit_hash, created_at, prerelease + mac, win, android, ios, tag, commit_hash, created_at, prerelease, resources_url, resources_hash FROM geode_versions "#, ); @@ -161,13 +297,17 @@ impl LoaderVersion { query_builder.push(" created_at DESC LIMIT 1;"); - query_builder + let Some(row) = query_builder .build_query_as::() .fetch_optional(&mut *pool) .await - .inspect_err(|e| tracing::error!("{:?}", e)) - .map_err(|e| e.into()) - .map(|x| x.map(|y| y.into_loader_version())) + .inspect_err(|e| tracing::error!("{:?}", e))? + else { + return Ok(None); + }; + + let downloads = LoaderVersion::get_downloads_for_tag(&row.tag, &mut *pool).await?; + Ok(Some(row.into_loader_version(downloads))) } #[tracing::instrument(skip_all, fields(tag = %tag))] @@ -175,20 +315,24 @@ impl LoaderVersion { tag: &str, pool: &mut PgConnection, ) -> Result, DatabaseError> { - sqlx::query_as!( + let Some(row) = sqlx::query_as!( LoaderVersionGetOne, r#"SELECT mac as "mac: _", win as "win: _", android as "android: _", ios as "ios: _", - tag, created_at, commit_hash, prerelease + tag, created_at, commit_hash, prerelease, resources_url, resources_hash FROM geode_versions WHERE tag = $1"#, tag ) .fetch_optional(&mut *pool) .await - .inspect_err(|e| tracing::error!("{:?}", e)) - .map_err(|e| e.into()) - .map(|x| x.map(|y| y.into_loader_version())) + .inspect_err(|e| tracing::error!("{:?}", e))? + else { + return Ok(None); + }; + + let downloads = LoaderVersion::get_downloads_for_tag(&row.tag, &mut *pool).await?; + Ok(Some(row.into_loader_version(downloads))) } #[tracing::instrument(skip_all, fields(tag = %version.tag))] @@ -229,7 +373,7 @@ impl LoaderVersion { let mut query_builder = QueryBuilder::new( r#" SELECT - mac, win, android, ios, tag, created_at, commit_hash, prerelease + mac, win, android, ios, tag, created_at, commit_hash, prerelease, resources_url, resources_hash FROM geode_versions "#, ); @@ -290,12 +434,22 @@ impl LoaderVersion { query_builder.push(" OFFSET "); query_builder.push_bind(offset); - query_builder + let rows = query_builder .build_query_as::() .fetch_all(&mut *pool) .await .inspect_err(|e| tracing::error!("{:?}", e)) - .map(|x| x.into_iter().map(|y| y.into_loader_version()).collect()) - .map_err(|e| e.into()) + .map_err(DatabaseError::from)?; + + let tags: Vec = rows.iter().map(|r| r.tag.clone()).collect(); + let mut downloads_map = LoaderVersion::get_downloads_for_tags(&tags, &mut *pool).await?; + + Ok(rows + .into_iter() + .map(|row| { + let downloads = downloads_map.remove(&row.tag).unwrap_or_default(); + row.into_loader_version(downloads) + }) + .collect()) } }