diff --git a/editor/Assets/AssetDatabase.cs b/editor/Assets/AssetDatabase.cs index c67ccf169..9b482a687 100644 --- a/editor/Assets/AssetDatabase.cs +++ b/editor/Assets/AssetDatabase.cs @@ -3,8 +3,6 @@ using System.IO; using System.Linq; using System.Security.Cryptography; -using System.Text.Json; -using System.Text.Json.Nodes; using System.Threading.Tasks; using Avalonia; using Avalonia.Controls; @@ -19,16 +17,7 @@ namespace editor.Assets; -// two databases because they serve different purposes and have different lifetimes: -// artwork_database.json -> maps source artwork paths to output UIDs + hash -// used to skip re-importing files that havent changed -// database.json -> maps UID to virtual path + asset type -// full rebuild on every import, used by the engine at runtime public static class AssetDatabase { - private static readonly JsonSerializerOptions s_prettyJson = new() { WriteIndented = true }; - - private static Dictionary? s_uidLookup; - // extension -> engine asset type, derived from the registry private static readonly Dictionary s_assetTypes = AssetTypeRegistry.All.ToDictionary(a => a.Extension, a => a.Type, StringComparer.OrdinalIgnoreCase); @@ -42,118 +31,43 @@ public static string ComputeHash(string realPath) { } public static bool IsUpToDate(string sourceVirtualPath, string hash) { - var db = LoadArtworkDatabase(); - return db[sourceVirtualPath] is JsonObject entry - && entry["last_hash"]?.GetValue() == hash; + AssetStore.Sync(); + return AssetStore.ArtworkByUri(sourceVirtualPath)?.Hash == hash; } public static void UpdateArtworkDatabase( string sourceVirtualPath, string hash, IEnumerable artworkOutputs) { - var db = LoadArtworkDatabase(); - - var outputs = new JsonArray(); + AssetStore.Sync(); + var id = AssetStore.UpsertArtwork(sourceVirtualPath); foreach (var output in artworkOutputs) - outputs.Add(new JsonObject { - ["uid"] = output.Uid, - ["name"] = output.Name, - ["hash"] = output.Hash, - ["uri"] = output.Uri - }); - - db[sourceVirtualPath] = new JsonObject { - ["last_hash"] = hash, - ["outputs"] = outputs - }; - - SaveArtworkDatabase(db); - } - - public static IReadOnlyList GetArtworkOutputs(string sourceVirtualPath) { - if (LoadArtworkDatabase()[sourceVirtualPath] is not JsonObject entry) return []; - return ReadOutputs(entry); - } - - private static List ReadOutputs(JsonObject entry) { - var result = new List(); - if (entry["outputs"] is not JsonArray outputs) return result; - - foreach (var node in outputs) - if (node is JsonObject obj && obj["uid"]?.GetValue() is { } uid) - result.Add(new ArtworkOutput(uid, obj["name"]?.GetValue() ?? "", obj["hash"]?.GetValue(), - obj["uri"]?.GetValue() ?? "")); - else if (node?.GetValue() is { } legacyUid) { - var uri = TryResolve(legacyUid, out var path, out _) ? path : ""; - result.Add(new ArtworkOutput(legacyUid, Path.GetFileName(uri), null, uri)); - } - - return result; - } - - private static List OutputsFromMeta(string sourceVirtualPath) { - return OutputsBySource().GetValueOrDefault(sourceVirtualPath) ?? []; - } - - private static Dictionary> OutputsBySource() { - var result = new Dictionary>(); - if (!Directory.Exists(ProjectContext.AssetsPath)) return result; - foreach (var metaPath in MetaFile.FindAll(ProjectContext.AssetsPath)) { - var header = MetaFile.ReadHeader(metaPath); - if (header?.Source is not { Length: > 0 } source) continue; - var real = metaPath[..^5]; - if (!result.TryGetValue(source, out var list)) result[source] = list = []; - list.Add(new ArtworkOutput(header.Uid, Path.GetFileName(real), null, ProjectContext.ToVirtual(real) ?? real)); + AssetStore.SetOutputInfo(output.Uid, output.Name, output.Hash, id); + AssetStore.SetArtworkHash(sourceVirtualPath, hash); + } + + public static void SyncArtworkHashes(Action? log = null) { + AssetStore.Sync(); + var count = 0; + foreach (var artwork in AssetStore.AllArtwork().Where(a => a.Hash is null)) { + var real = ProjectContext.Resolve(artwork.Uri); + if (!File.Exists(real)) continue; // external imports name a source that is not on the project + AssetStore.SetArtworkHash(artwork.Uri, ComputeHash(real)); + count++; } - return result; + if (count > 0) log?.Invoke($"Hashed {count} artwork file(s)"); } - public static void SyncArtworkDatabase() { - var db = LoadArtworkDatabase(); - var changed = false; - foreach (var (source, outputs) in OutputsBySource()) { - if (db[source] is not null) continue; - var real = ProjectContext.Resolve(source); - if (!File.Exists(real)) continue; - db[source] = new JsonObject { - ["last_hash"] = ComputeHash(real), - ["outputs"] = new JsonArray(outputs.Select(o => (JsonNode)new JsonObject { - ["uid"] = o.Uid, - ["name"] = o.Name, - ["hash"] = o.Hash, - ["uri"] = o.Uri - }).ToArray()) - }; - changed = true; - } - - if (changed) SaveArtworkDatabase(db); + public static IReadOnlyList GetArtworkOutputs(string sourceVirtualPath) { + AssetStore.Sync(); + return AssetStore.OutputsOf(sourceVirtualPath) + .Select(a => new ArtworkOutput(a.Uid, a.OutputName ?? Path.GetFileName(a.Path), a.OutputHash, a.Virtual)) + .ToList(); } - // full re-scan of all .meta files; writes one .json per content database + core.json + // syncs the cache from the .meta files + // if theres a bug it's most likely here public static void RebuildAssetDatabase(bool notifyListeners = true) { - var now = DateTime.UtcNow.ToString("o"); - - // One manifest file per content database - foreach (var db in ProjectContext.Databases) { - var obj = new JsonObject { ["version"] = 2, ["generated_at"] = now }; - ScanDirectory(Path.Combine(ProjectContext.ProjectPath, db), obj); - var path = ProjectContext.Resolve($"cache://{db}.json"); - File.WriteAllText(path, obj.ToJsonString(s_prettyJson)); - } - - // core:// always gets its own manifest - { - var obj = new JsonObject { ["version"] = 2, ["generated_at"] = now }; - ScanDirectory(ProjectContext.CorePath, obj); - var path = ProjectContext.Resolve("cache://core.json"); - File.WriteAllText(path, obj.ToJsonString(s_prettyJson)); - } - - // Remove the legacy combined file if it still exists - var legacyPath = ProjectContext.Resolve("cache://database.json"); - if (File.Exists(legacyPath)) File.Delete(legacyPath); - - s_uidLookup = null; // the on-disk database changed, drop the cached lookup + AssetStore.Sync(); // Refresh UI bind stubs so Lua completion tracks .rml edits UIBindStubGenerator.Generate(); @@ -166,15 +80,12 @@ public static void RebuildAssetDatabase(bool notifyListeners = true) { if (notifyListeners) ReloadedDatabase?.Invoke(); } - public static void Reset() { - s_uidLookup = null; - } + public static void Reset() { } public static bool TryResolve(string uid, out string virtualPath, out string type) { - s_uidLookup ??= BuildUidLookup(); - if (s_uidLookup.TryGetValue(uid, out var entry)) { - virtualPath = entry.Path; - type = entry.Type; + if (AssetStore.ByUid(uid) is { } record) { + virtualPath = record.Virtual; + type = record.Type; return true; } @@ -183,30 +94,13 @@ public static bool TryResolve(string uid, out string virtualPath, out string typ return false; } - private static Dictionary BuildUidLookup() { - var lookup = new Dictionary(); - if (LoadAssetDatabase() is not { } db) return lookup; - - foreach (var (type, collectionNode) in db) { - if (type is "version" or "generated_at") continue; - if (collectionNode is not JsonObject collection) continue; - foreach (var (uid, pathNode) in collection) { - var path = pathNode?.GetValue(); - if (path is null) continue; - lookup[uid] = (path, type); - } - } - - return lookup; - } - public static async Task ValidateProject(Action? log = null) { log ??= _ => { }; GenerateMissingMetas(log); RebuildAssetDatabase(); - SyncArtworkDatabase(); await RelocateMissingAssets(log); await RelocateMissingArtwork(log); + SyncArtworkHashes(log); await CheckArtworkChanges(log); RebuildAssetDatabase(); } @@ -256,60 +150,55 @@ public static void GenerateMissingThumbnails(Action? log = null) { } public static async Task RelocateMissingAssets(Action log) { - if (LoadAssetDatabase() is not { } db) return; - var changed = false; var discardAll = false; var current = ScanCurrentAssets(); - foreach (var (type, collectionNode) in db.ToList()) { - if (type is "version" or "generated_at") continue; - if (collectionNode is not JsonObject collection) continue; - foreach (var (uid, pathNode) in collection.ToList()) { - var virtualPath = pathNode?.GetValue(); - if (virtualPath is null) continue; - - var realPath = ProjectContext.Resolve(virtualPath); - if (File.Exists(realPath)) continue; - - if (current.TryGetValue(uid, out var moved) && moved != virtualPath && - File.Exists(ProjectContext.Resolve(moved))) { - log($"Moved file: {virtualPath} -> {moved}"); - changed = true; // rebuild rewrites the database - continue; - } + foreach (var row in AssetStore.ListAll()) { + var uid = row.Uid; + var virtualPath = row.Virtual; + if (!AssetStore.IsTrackedScheme(row.Db)) continue; - log($"Missing file: {virtualPath}"); + var realPath = ProjectContext.Resolve(virtualPath); + if (File.Exists(realPath)) continue; - if (discardAll) { - TryDelete(realPath + ".meta"); - changed = true; - log($"Discarded {virtualPath}"); - continue; - } + if (current.TryGetValue(uid, out var moved) && moved != virtualPath && + File.Exists(ProjectContext.Resolve(moved))) { + log($"Moved file: {virtualPath} -> {moved}"); + changed = true; // rebuild rewrites the database + continue; + } - var result = await ShowRelocateModal( - "Missing file", - $"File {virtualPath} could not be found, do you wish to relocate?", - "assets://", - Path.GetExtension(realPath)); + log($"Missing file: {virtualPath}"); - switch (result.Decision) { - case RelocateDecision.Relocate: - if (RelocateMeta(realPath, result.PickedPath!, uid, type, log)) - changed = true; - break; - case RelocateDecision.Discard: - if (result.AppliedToAll) discardAll = true; - TryDelete(realPath + ".meta"); + if (discardAll) { + TryDelete(realPath + ".meta"); + changed = true; + log($"Discarded {virtualPath}"); + continue; + } + + var result = await ShowRelocateModal( + "Missing file", + $"File {virtualPath} could not be found, do you wish to relocate?", + "assets://", + Path.GetExtension(realPath)); + + switch (result.Decision) { + case RelocateDecision.Relocate: + if (RelocateMeta(realPath, result.PickedPath!, uid, row.Type, log)) changed = true; - log($"Discarded {virtualPath}"); - break; - case RelocateDecision.Skip: - log($"Skipped {virtualPath}"); - break; - } + break; + case RelocateDecision.Discard: + if (result.AppliedToAll) discardAll = true; + TryDelete(realPath + ".meta"); + changed = true; + log($"Discarded {virtualPath}"); + break; + case RelocateDecision.Skip: + log($"Skipped {virtualPath}"); + break; } } @@ -317,13 +206,12 @@ public static async Task RelocateMissingAssets(Action log) { } public static async Task RelocateMissingArtwork(Action log) { - var db = LoadArtworkDatabase(); var changed = false; var discardAll = false; - foreach (var (sourceVirtual, node) in db.ToList()) { - if (sourceVirtual is "version" or "type") continue; - + // artwork without a hash counts as unchanged + foreach (var artwork in AssetStore.AllArtwork().Where(a => a.Hash is not null)) { + var sourceVirtual = artwork.Uri; var realPath = ProjectContext.Resolve(sourceVirtual); if (File.Exists(realPath)) continue; @@ -331,7 +219,7 @@ public static async Task RelocateMissingArtwork(Action log) { // See RelocateMissingAssets() if (discardAll) { - db.Remove(sourceVirtual); + AssetStore.DeleteArtwork(sourceVirtual); changed = true; log($"Removed link {sourceVirtual}"); continue; @@ -352,16 +240,15 @@ public static async Task RelocateMissingArtwork(Action log) { } // repoint every asset generated from this artwork at the new source, then move the artwork - // database entry (outputs + hash) onto the new key + // onto the new uri RepointArtworkSource(sourceVirtual, newVirtual, log); - db.Remove(sourceVirtual); // detaches the node so it can be re-keyed - db[newVirtual] = node; + AssetStore.RenameArtwork(sourceVirtual, newVirtual); changed = true; log($"Relocated artwork {sourceVirtual} -> {newVirtual}"); break; case RelocateDecision.Discard: if (result.AppliedToAll) discardAll = true; - db.Remove(sourceVirtual); + AssetStore.DeleteArtwork(sourceVirtual); changed = true; log($"Removed link {sourceVirtual}"); break; @@ -371,40 +258,23 @@ public static async Task RelocateMissingArtwork(Action log) { } } - if (changed) SaveArtworkDatabase(db); + if (changed) RebuildAssetDatabase(); } public static async Task CheckArtworkChanges(Action log) { - var db = LoadArtworkDatabase(); var reimported = false; - foreach (var (sourceVirtual, node) in db.ToList()) { - if (sourceVirtual is "version" or "type") continue; - if (node is not JsonObject entry) continue; - + foreach (var artwork in AssetStore.AllArtwork().Where(a => a.Hash is not null)) { + var sourceVirtual = artwork.Uri; var realPath = ProjectContext.Resolve(sourceVirtual); if (!File.Exists(realPath)) continue; // missing files are handled by RelocateMissingArtwork var hash = ComputeHash(realPath); - if (entry["last_hash"]?.GetValue() == hash) continue; + if (artwork.Hash == hash) continue; log($"Artwork changed: {sourceVirtual}"); - var reimport = await ShowChoice(new ModalConfig( - "Artwork changed", - $"Artwork file {sourceVirtual} has changed, do you wish to reimport the assets generated from it?", - ModalButtons.OkCancel, - LucideIconKind.RefreshCw, - new SolidColorBrush(Color.Parse("#4a9eff")), - "Reimport", - CancelLabel: "Skip", - OkIcon: LucideIconKind.FileInput)); - - if (reimport == true) { - await Reimport(sourceVirtual, log); - reimported = true; - } else { - log($"Skipped {sourceVirtual}"); - } + await Reimport(sourceVirtual, log); + reimported = true; } if (reimported) RebuildAssetDatabase(); @@ -413,10 +283,7 @@ public static async Task CheckArtworkChanges(Action log) { public static async Task Reimport( string sourceVirtualPath, Action log, Action? progress = null) { - var db = LoadArtworkDatabase(); - var previous = db[sourceVirtualPath] is JsonObject entry - ? ReadOutputs(entry) - : OutputsFromMeta(sourceVirtualPath); + var previous = GetArtworkOutputs(sourceVirtualPath); if (previous.Count == 0) { log($"No import record for {sourceVirtualPath}"); return; @@ -428,10 +295,8 @@ public static async Task Reimport( return; } - var uids = previous.Select(o => o.Uid).ToList(); - // find where the existing outputs live + the settings they were imported with - var (destDir, textureSection, psdSection, gltfSection, voxSection) = LocateOutputs(uids); + var (destDir, textureSection, psdSection, gltfSection, voxSection) = LocateOutputs(previous, realSource); var textureSettings = SettingsFromMeta(textureSection); var ext = Path.GetExtension(realSource).ToLowerInvariant(); @@ -449,9 +314,11 @@ public static async Task Reimport( }; log($"Reimporting {sourceVirtualPath}..."); + var sourceHash = ComputeHash(realSource); var outputs = await ArtworkImport.RunAsync(importer, realSource, sourceVirtualPath, destDir ?? ProjectContext.AssetsPath, previous, log, progress); - UpdateArtworkDatabase(sourceVirtualPath, ComputeHash(realSource), outputs); + if (outputs is null) return; + UpdateArtworkDatabase(sourceVirtualPath, sourceHash, outputs); log("Reimport done."); } @@ -489,62 +356,32 @@ private static void RepointArtworkSource(string oldSource, string newSource, Act } private static Dictionary ScanCurrentAssets() { - var scanned = new JsonObject(); - foreach (var db in ProjectContext.Databases) - ScanDirectory(Path.Combine(ProjectContext.ProjectPath, db), scanned); - ScanDirectory(ProjectContext.CorePath, scanned); - var current = new Dictionary(); - foreach (var (_, collectionNode) in scanned) - if (collectionNode is JsonObject collection) - foreach (var (uid, pathNode) in collection) - if (pathNode?.GetValue() is { } path) - current[uid] = path; - - return current; - } - - // collections are created on demand - private static void ScanDirectory(string directory, JsonObject db) { - if (!Directory.Exists(directory)) return; - foreach (var metaPath in MetaFile.FindAll(directory)) { - var header = MetaFile.ReadHeader(metaPath); - if (header is null) continue; - - var assetRealPath = metaPath[..^5]; // strip ".meta" - var virtualPath = ProjectContext.ToVirtual(assetRealPath); - if (virtualPath is null) continue; - - if (db[header.Type] is not JsonObject collection) { - collection = new JsonObject(); - db[header.Type] = collection; + var roots = ProjectContext.DatabaseRoots.Append(ProjectContext.CorePath); + foreach (var root in roots) { + if (!Directory.Exists(root)) continue; + foreach (var metaPath in MetaFile.FindAll(root)) { + var header = MetaFile.ReadHeader(metaPath); + if (header is null) continue; + var virtualPath = ProjectContext.ToVirtual(metaPath[..^5]); // strip ".meta" + if (virtualPath is not null) current[header.Uid] = virtualPath; } - - collection[header.Uid] = virtualPath; } + + return current; } - // finds the directory + import settings for a set of output UIDs by scanning their .meta sidecars private static (string? destDir, TextureMetaSection? texture, PsdMetaSection? psd, GltfMetaSection? gltf, - VoxMetaSection? vox) - LocateOutputs(IReadOnlyCollection uids) { - if (uids.Count == 0 || !Directory.Exists(ProjectContext.AssetsPath)) - return (null, null, null, null, null); - - var wanted = new HashSet(uids); - foreach (var metaPath in MetaFile.FindAll(ProjectContext.AssetsPath)) { - var header = MetaFile.ReadHeader(metaPath); - if (header is null || !wanted.Contains(header.Uid)) continue; - - var assetReal = metaPath[..^5]; - return (Path.GetDirectoryName(assetReal), - MetaFile.ReadTextureSection(metaPath), - MetaFile.ReadPsdSection(metaPath), - MetaFile.ReadGltfSection(metaPath), - MetaFile.ReadVoxSection(metaPath)); - } - - return (null, null, null, null, null); + VoxMetaSection? vox) LocateOutputs(IReadOnlyList outputs, string sourcePath) { + var sourceName = Path.GetFileName(sourcePath); + var paths = outputs.OrderByDescending(output => + string.Equals(output.Name, sourceName, StringComparison.OrdinalIgnoreCase)) + .Select(output => ProjectContext.Resolve(output.Uri)).ToList(); + var texture = paths.Select(MetaFile.ReadTextureSection).FirstOrDefault(section => section is not null); + var psd = paths.Select(MetaFile.ReadPsdSection).FirstOrDefault(section => section is not null); + var gltf = paths.Select(MetaFile.ReadGltfSection).FirstOrDefault(section => section is not null); + var vox = paths.Select(MetaFile.ReadVoxSection).FirstOrDefault(section => section is not null); + return (Path.GetDirectoryName(paths[0]), texture, psd, gltf, vox); } private static TextureImporter.Settings SettingsFromMeta(TextureMetaSection? s) { @@ -596,67 +433,6 @@ private static GltfImporter.Settings SettingsFromMeta(GltfMetaSection? s) { return settings; } - private static JsonObject? LoadAssetDatabase() { - // Merge per-database manifests (.json + core.json) into a single JsonObject - JsonObject? merged = null; - - var sources = ProjectContext.Databases - .Select(db => $"cache://{db}.json") - .Append("cache://core.json"); - - foreach (var virtualPath in sources) { - var path = ProjectContext.Resolve(virtualPath); - if (!File.Exists(path)) continue; - JsonObject? obj; - try { - obj = JsonNode.Parse(File.ReadAllText(path)) as JsonObject; - } catch { - continue; - } - - if (obj is null) continue; - - if (merged is null) merged = new JsonObject { ["version"] = 2 }; - - // Union the per-type collections (skip metadata keys) - foreach (var (type, collectionNode) in obj) { - if (type is "version" or "generated_at") continue; - if (collectionNode is not JsonObject collection) continue; - - if (merged[type] is not JsonObject target) { - target = new JsonObject(); - merged[type] = target; - } - - foreach (var (uid, value) in collection) - target[uid] = value?.GetValue() is { } s ? JsonValue.Create(s) : null; - } - } - - return merged; - } - - private static JsonObject LoadArtworkDatabase() { - var path = ProjectContext.Resolve("cache://artwork_database.json"); - if (!File.Exists(path)) - return new JsonObject { - ["version"] = 1, - ["type"] = "artwork_database" - }; - try { - return JsonNode.Parse(File.ReadAllText(path)) as JsonObject - ?? new JsonObject { ["version"] = 1, ["type"] = "artwork_database" }; - } catch { - return new JsonObject { ["version"] = 1, ["type"] = "artwork_database" }; - } - } - - private static void SaveArtworkDatabase(JsonObject db) { - var path = ProjectContext.Resolve("cache://artwork_database.json"); - Directory.CreateDirectory(Path.GetDirectoryName(path)!); - File.WriteAllText(path, db.ToJsonString(s_prettyJson)); - } - public static void TryDelete(string path) { try { if (!File.Exists(path)) return; @@ -680,27 +456,7 @@ public static void RemoveArtworkOutputs(string uid) { } public static void RemoveArtworkOutputs(IEnumerable uids) { - var targets = new HashSet(uids, StringComparer.Ordinal); - if (targets.Count == 0) return; - - var db = LoadArtworkDatabase(); - var changed = false; - foreach (var key in db.Select(kv => kv.Key).ToList()) { - if (key is "version" or "type") continue; - if (db[key] is not JsonObject entry) continue; - if (entry["outputs"] is not JsonArray outputs) continue; - for (var i = outputs.Count - 1; i >= 0; i--) { - var output = outputs[i] is JsonObject record ? record["uid"]?.GetValue() : outputs[i]?.GetValue(); - if (output is null || !targets.Contains(output)) continue; - outputs.RemoveAt(i); - changed = true; - } - - if (outputs.Count == 0) - db.Remove(key); - } - - if (changed) SaveArtworkDatabase(db); + AssetStore.UnlinkArtwork(uids); } private static async Task ShowRelocateModal( @@ -733,10 +489,6 @@ private static async Task ShowRelocateModal( } } - private static async Task ShowChoice(ModalConfig cfg) { - return (await ShowChoiceEx(cfg)).Choice; - } - private static async Task<(bool? Choice, bool AppliedToAll)> ShowChoiceEx(ModalConfig cfg) { var owner = ActiveWindow(); if (owner is null) return (null, false); diff --git a/editor/Assets/AssetFolder.cs b/editor/Assets/AssetFolder.cs index 7ce27951f..0e8d341e9 100644 --- a/editor/Assets/AssetFolder.cs +++ b/editor/Assets/AssetFolder.cs @@ -39,22 +39,32 @@ public AssetFolder(string path, AssetFolder? parent = null, bool listRawFiles = SubFolders.Add(new AssetFolder(sub.FullName, this, listRawFiles)); } - foreach (var file in dirInfo.EnumerateFiles() - .OrderBy(f => f.Name, StringComparer.OrdinalIgnoreCase) - .ThenBy(f => f.Name, StringComparer.Ordinal)) { - var assetName = file.Extension == ".meta" ? Path.GetFileNameWithoutExtension(file.Name) : file.Name; - if (AssetBrowserSettings.IsHidden(assetName)) continue; - - if (listRawFiles) { - if (file.Extension == ".meta") - continue; + if (listRawFiles) { + foreach (var file in dirInfo.EnumerateFiles() + .OrderBy(f => f.Name, StringComparer.OrdinalIgnoreCase) + .ThenBy(f => f.Name, StringComparer.Ordinal)) { + if (file.Extension == ".meta" || AssetBrowserSettings.IsHidden(file.Name)) continue; Files.Add(AssetFile.Raw(file.FullName)); - continue; } - if (file.Extension != ".meta") continue; // assets are tracked by their .meta sidecar - Files.Add(new AssetFile(file.FullName)); + return; } + + // assets come from the asset database + if (parent is null) AssetStore.Sync(); + var virtualDir = VirtualDirectory(dirInfo.FullName); + if (virtualDir is null) return; + foreach (var record in AssetStore.ListDirectory(virtualDir)) { + var real = ProjectContext.Resolve(record.Virtual); + if (AssetBrowserSettings.IsHidden(Path.GetFileName(real))) continue; + var meta = real + ".meta"; // we still get the .meta tho + if (File.Exists(meta)) Files.Add(new AssetFile(meta)); + } + } + + private static string? VirtualDirectory(string realDir) { + var probe = ProjectContext.ToVirtual(Path.Combine(realDir, "_")); + return probe is null ? null : probe[..^1]; } public string Name { get; set; } = ""; diff --git a/editor/Assets/AssetStore.cs b/editor/Assets/AssetStore.cs new file mode 100644 index 000000000..e3c852dcd --- /dev/null +++ b/editor/Assets/AssetStore.cs @@ -0,0 +1,378 @@ +// +// AssetStore.cs by Xein +// 8 Nov 2026 +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using Microsoft.Data.Sqlite; + +namespace editor.Assets; + +public sealed record AssetRecord( + string Uid, string Type, string Db, string Path, + string? ArtworkId, string? OutputName, string? OutputHash, + string CreatedAt, string? ModifiedAt) { + public string Virtual => $"{Db}://{Path}"; + public bool IsCore => Db == AssetStore.CoreDb; +} + +public sealed record ArtworkRecord(string Id, string Uri, string? Hash); + +public static class AssetStore { + public const string CoreDb = "core"; + public const string RuntimeDatabaseName = "game.db"; + private const int SchemaVersion = 2; + private static readonly object s_lock = new(); + private static SqliteConnection? s_db; + public static bool IsOpen => s_db is not null; + + public static void Open() { + Close(); + var path = Path.Combine(ProjectContext.CachePath, "assets.db"); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + + lock (s_lock) { + s_db = new SqliteConnection($"Data Source={path};Pooling=False"); + s_db.Open(); + Exec("PRAGMA journal_mode = WAL"); + Exec("PRAGMA foreign_keys = ON"); + Exec("PRAGMA synchronous = NORMAL"); + EnsureSchema(); + } + } + + public static void Close() { + lock (s_lock) { + s_db?.Dispose(); + s_db = null; + } + } + + private static void EnsureSchema() { + if (Scalar("PRAGMA user_version") != SchemaVersion) { + Exec("DROP TABLE IF EXISTS asset_tags"); + Exec("DROP TABLE IF EXISTS assets"); + Exec("DROP TABLE IF EXISTS artwork"); + Exec("DROP TABLE IF EXISTS meta"); + } + + Exec(""" + CREATE TABLE IF NOT EXISTS artwork ( + id TEXT PRIMARY KEY, + uri TEXT NOT NULL UNIQUE, + hash TEXT + ) + """); + Exec(""" + CREATE TABLE IF NOT EXISTS assets ( + uid TEXT PRIMARY KEY, + type TEXT NOT NULL, + db TEXT NOT NULL, + path TEXT NOT NULL, + artwork_id TEXT REFERENCES artwork(id) ON DELETE SET NULL, + output_name TEXT, + output_hash TEXT, + created_at TEXT NOT NULL, + modified_at TEXT, + meta_mtime INTEGER NOT NULL DEFAULT 0, + UNIQUE (db, path) + ) + """); + Exec("CREATE INDEX IF NOT EXISTS assets_artwork ON assets(artwork_id)"); + Exec("CREATE INDEX IF NOT EXISTS assets_type ON assets(type)"); + Exec($"PRAGMA user_version = {SchemaVersion}"); + } + + public static bool Sync() { + lock (s_lock) { + if (s_db is null) return false; + + var known = Query("SELECT uid, db, path, meta_mtime FROM assets", + r => (Uid: r.GetString(0), Virtual: $"{r.GetString(1)}://{r.GetString(2)}", Mtime: r.GetInt64(3))); + var byVirtual = known.ToDictionary(k => k.Virtual, k => k, StringComparer.Ordinal); + + var seen = new HashSet(StringComparer.Ordinal); + var fresh = new List<(string Meta, string Virtual, long Mtime)>(); + foreach (var root in ProjectContext.DatabaseRoots.Append(ProjectContext.CorePath)) { + if (!Directory.Exists(root)) continue; + foreach (var meta in Directory.EnumerateFiles(root, "*.meta", SearchOption.AllDirectories)) { + var virtualPath = ProjectContext.ToVirtual(meta[..^".meta".Length]); + if (virtualPath is null || !TrySplit(virtualPath, out var scheme, out _) || !IsTrackedScheme(scheme)) continue; + + var mtime = File.GetLastWriteTimeUtc(meta).Ticks; + if (byVirtual.TryGetValue(virtualPath, out var row) && row.Mtime == mtime) { + seen.Add(row.Uid); + continue; + } + + fresh.Add((meta, virtualPath, mtime)); + } + } + + var changed = false; + using var tx = s_db.BeginTransaction(); + foreach (var (meta, virtualPath, mtime) in fresh) { + var header = MetaFile.ReadHeader(meta); + if (header is null || !TrySplit(virtualPath, out var db, out var path)) continue; + + string? artworkId = null; + if (header.Source is { Length: > 0 } source) artworkId = UpsertArtworkRow(source); + + Exec("DELETE FROM assets WHERE db = @db AND path = @path AND uid <> @uid", ("@db", db), ("@path", path), ("@uid", header.Uid)); + Exec(""" + INSERT INTO assets (uid, type, db, path, artwork_id, created_at, modified_at, meta_mtime) + VALUES (@uid, @type, @db, @path, @artwork, @created, @modified, @mtime) + ON CONFLICT(uid) DO UPDATE SET + type = excluded.type, db = excluded.db, path = excluded.path, artwork_id = excluded.artwork_id, + created_at = excluded.created_at, modified_at = excluded.modified_at, meta_mtime = excluded.meta_mtime + """, + ("@uid", header.Uid), ("@type", header.Type), ("@db", db), ("@path", path), ("@artwork", artworkId), + ("@created", header.CreatedAt), ("@modified", header.ModifiedAt), ("@mtime", mtime)); + seen.Add(header.Uid); + changed = true; + } + + foreach (var (uid, _, _) in known.Select(k => (k.Uid, k.Virtual, k.Mtime))) + if (!seen.Contains(uid)) { + Exec("DELETE FROM assets WHERE uid = @uid", ("@uid", uid)); + changed = true; + } + + if (changed) Exec("DELETE FROM artwork WHERE id NOT IN (SELECT artwork_id FROM assets WHERE artwork_id IS NOT NULL)"); + tx.Commit(); + return changed; + } + } + + public static AssetRecord? ByUid(string uid) { + lock (s_lock) { + return s_db is null ? null : Query(AssetSelect + " WHERE uid = @uid", ReadAsset, ("@uid", uid)).FirstOrDefault(); + } + } + + public static AssetRecord? ByVirtual(string virtualPath) { + if (!TrySplit(virtualPath, out var db, out var path)) return null; + lock (s_lock) { + return s_db is null + ? null + : Query(AssetSelect + " WHERE db = @db AND path = @path", ReadAsset, ("@db", db), ("@path", path)).FirstOrDefault(); + } + } + + public static IReadOnlyList ListAll() { + lock (s_lock) { + return s_db is null ? [] : Query(AssetSelect + " ORDER BY db, path", ReadAsset); + } + } + + public static IReadOnlyList ListByType(string type) { + lock (s_lock) { + return s_db is null ? [] : Query(AssetSelect + " WHERE type = @type ORDER BY db, path", ReadAsset, ("@type", type)); + } + } + + public static IReadOnlyList ListDirectory(string virtualDir) { + if (!TrySplit(virtualDir, out var db, out var dir)) return []; + dir = dir.Trim('/'); + lock (s_lock) { + if (s_db is null) return []; + var prefix = dir.Length == 0 ? "" : dir + "/"; + var rows = Query(AssetSelect + " WHERE db = @db AND path LIKE @like ESCAPE '\\'", ReadAsset, + ("@db", db), ("@like", EscapeLike(prefix) + "%")); + return rows.Where(r => r.Path.IndexOf('/', prefix.Length) < 0).OrderBy(r => r.Path, StringComparer.OrdinalIgnoreCase).ToList(); + } + } + + public static IReadOnlyList OutputsOf(string artworkUri) { + lock (s_lock) { + return s_db is null + ? [] + : Query(AssetSelect + " WHERE artwork_id = (SELECT id FROM artwork WHERE uri = @uri) ORDER BY path", ReadAsset, ("@uri", artworkUri)); + } + } + + public static ArtworkRecord? ArtworkByUri(string uri) { + lock (s_lock) { + return s_db is null ? null : Query(ArtworkSelect + " WHERE uri = @uri", ReadArtwork, ("@uri", uri)).FirstOrDefault(); + } + } + + public static IReadOnlyList AllArtwork() { + lock (s_lock) { + return s_db is null ? [] : Query(ArtworkSelect + " ORDER BY uri", ReadArtwork); + } + } + + public static string UpsertArtwork(string uri) { + lock (s_lock) { + return s_db is null ? "" : UpsertArtworkRow(uri); + } + } + + public static void SetArtworkHash(string uri, string? hash) { + lock (s_lock) { + if (s_db is null) return; + Exec("UPDATE artwork SET hash = @hash WHERE uri = @uri", ("@hash", hash), ("@uri", uri)); + } + } + + public static void SetOutputInfo(string uid, string? name, string? hash, string? artworkId = null) { + lock (s_lock) { + if (s_db is null) return; + Exec("UPDATE assets SET output_name = @name, output_hash = @hash, artwork_id = COALESCE(@artwork, artwork_id) WHERE uid = @uid", + ("@name", name), ("@hash", hash), ("@artwork", artworkId), ("@uid", uid)); + } + } + + public static void RenameArtwork(string oldUri, string newUri) { + lock (s_lock) { + if (s_db is null) return; + var from = Query(ArtworkSelect + " WHERE uri = @uri", ReadArtwork, ("@uri", oldUri)).FirstOrDefault(); + if (from is null) return; + if (Query(ArtworkSelect + " WHERE uri = @uri", ReadArtwork, ("@uri", newUri)).FirstOrDefault() is { } clash && clash.Id != from.Id) { + Exec("UPDATE assets SET artwork_id = @to WHERE artwork_id = @from", ("@to", clash.Id), ("@from", from.Id)); + Exec("UPDATE artwork SET hash = COALESCE(hash, @hash) WHERE id = @id", ("@hash", from.Hash), ("@id", clash.Id)); + Exec("DELETE FROM artwork WHERE id = @id", ("@id", from.Id)); + } else { + Exec("UPDATE artwork SET uri = @uri WHERE id = @id", ("@uri", newUri), ("@id", from.Id)); + } + } + } + + public static void DeleteArtwork(string uri) { + lock (s_lock) { + if (s_db is null) return; + Exec("DELETE FROM artwork WHERE uri = @uri", ("@uri", uri)); + } + } + + public static void UnlinkArtwork(IEnumerable uids) { + lock (s_lock) { + if (s_db is null) return; + foreach (var uid in uids) + Exec("UPDATE assets SET artwork_id = NULL, output_name = NULL, output_hash = NULL WHERE uid = @uid", ("@uid", uid)); + Exec("DELETE FROM artwork WHERE id NOT IN (SELECT artwork_id FROM assets WHERE artwork_id IS NOT NULL)"); + } + } + + private static string UpsertArtworkRow(string uri) { + var existing = Query(ArtworkSelect + " WHERE uri = @uri", ReadArtwork, ("@uri", uri)).FirstOrDefault(); + if (existing is not null) return existing.Id; + var id = UidGenerator.Generate(); + Exec("INSERT INTO artwork (id, uri, hash) VALUES (@id, @uri, NULL)", ("@id", id), ("@uri", uri)); + return id; + } + + public static void ExportRuntimeDatabase(string path, IReadOnlyList<(string Db, string File)> paks) { + var folder = Path.GetDirectoryName(Path.GetFullPath(path))!; + Directory.CreateDirectory(folder); + foreach (var suffix in new[] { "", "-wal", "-shm", "-journal" }) + if (File.Exists(path + suffix)) + File.Delete(path + suffix); + + var packed = paks.Select(p => p.Db).ToHashSet(StringComparer.Ordinal); + var rows = ListAll().Where(a => packed.Contains(a.Db)).ToList(); + + using (var db = new SqliteConnection($"Data Source={path};Pooling=False")) { + db.Open(); + using var tx = db.BeginTransaction(); + + void Run(string sql, params (string Name, object? Value)[] args) { + using var cmd = db.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = sql; + foreach (var (name, value) in args) cmd.Parameters.AddWithValue(name, value ?? DBNull.Value); + cmd.ExecuteNonQuery(); + } + + Run("CREATE TABLE assets (uid TEXT PRIMARY KEY, type TEXT NOT NULL, db TEXT NOT NULL, path TEXT NOT NULL)"); + Run("CREATE TABLE paks (db TEXT PRIMARY KEY, file TEXT NOT NULL, size INTEGER, hash TEXT)"); + foreach (var row in rows) + Run("INSERT INTO assets (uid, type, db, path) VALUES (@uid, @type, @db, @path)", + ("@uid", row.Uid), ("@type", row.Type), ("@db", row.Db), ("@path", row.Path)); + + foreach (var (name, file) in paks) { + var pakPath = Path.Combine(folder, file); + long? size = null; + string? hash = null; + if (File.Exists(pakPath)) { + size = new FileInfo(pakPath).Length; + using var stream = File.OpenRead(pakPath); + hash = Convert.ToHexString(SHA256.HashData(stream)); + } + + Run("INSERT INTO paks (db, file, size, hash) VALUES (@db, @file, @size, @hash)", + ("@db", name), ("@file", file.Replace('\\', '/')), ("@size", size), ("@hash", hash)); + } + + tx.Commit(); + } + + SqliteConnection.ClearAllPools(); + } + + private const string AssetSelect = + "SELECT uid, type, db, path, artwork_id, output_name, output_hash, created_at, modified_at FROM assets"; + + private const string ArtworkSelect = "SELECT id, uri, hash FROM artwork"; + + private static AssetRecord ReadAsset(SqliteDataReader r) { + return new AssetRecord(r.GetString(0), r.GetString(1), r.GetString(2), r.GetString(3), + NullableString(r, 4), NullableString(r, 5), NullableString(r, 6), r.GetString(7), NullableString(r, 8)); + } + + private static ArtworkRecord ReadArtwork(SqliteDataReader r) { + return new ArtworkRecord(r.GetString(0), r.GetString(1), NullableString(r, 2)); + } + + private static string? NullableString(SqliteDataReader r, int index) { + return r.IsDBNull(index) ? null : r.GetString(index); + } + + public static bool TrySplit(string virtualPath, out string db, out string path) { + db = path = ""; + var sep = virtualPath.IndexOf("://", StringComparison.Ordinal); + if (sep <= 0) return false; + db = virtualPath[..sep]; + path = virtualPath[(sep + 3)..].Replace('\\', '/'); + return true; + } + + public static bool IsTrackedScheme(string db) { + return db == CoreDb || ProjectContext.Databases.Contains(db); + } + + private static string EscapeLike(string text) { + return text.Replace("\\", "\\\\").Replace("%", "\\%").Replace("_", "\\_"); + } + + private static SqliteCommand Command(string sql, (string Name, object? Value)[] args) { + var cmd = s_db!.CreateCommand(); + cmd.CommandText = sql; + foreach (var (name, value) in args) cmd.Parameters.AddWithValue(name, value ?? DBNull.Value); + return cmd; + } + + private static int Exec(string sql, params (string Name, object? Value)[] args) { + using var cmd = Command(sql, args); + return cmd.ExecuteNonQuery(); + } + + private static T? Scalar(string sql, params (string Name, object? Value)[] args) { + using var cmd = Command(sql, args); + var value = cmd.ExecuteScalar(); + return value is null or DBNull ? default : (T)Convert.ChangeType(value, typeof(T)); + } + + private static List Query(string sql, Func map, params (string Name, object? Value)[] args) { + using var cmd = Command(sql, args); + using var reader = cmd.ExecuteReader(); + var rows = new List(); + while (reader.Read()) rows.Add(map(reader)); + return rows; + } +} diff --git a/editor/Assets/Importers/ArtworkImport.cs b/editor/Assets/Importers/ArtworkImport.cs index 33ff12cb8..c9a8c4a5f 100644 --- a/editor/Assets/Importers/ArtworkImport.cs +++ b/editor/Assets/Importers/ArtworkImport.cs @@ -1,143 +1,97 @@ // // ArtworkImport.cs by Xein -// +// using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; +using editor.Components.Modals; namespace editor.Assets.Importers; public sealed record ArtworkOutput(string Uid, string Name, string? Hash, string Uri); public static class ArtworkImport { - public static async Task> RunAsync( + public static async Task?> RunAsync( IAssetImporter importer, string realSource, string sourceVirtual, string destDir, IReadOnlyList previous, Action log, Action? progress = null) { - var writes = new List<(string Uid, string Path)>(); - using var scope = MetaFile.Capture(writes); - if (previous.Count == 0) { - var ctx = new ImportContext { DestDir = destDir, SourceVirtualPath = sourceVirtual }; - await importer.Import(realSource, ctx, log, progress); - return Describe(Produced(writes, null)); + var produced = await Generate(importer, realSource, new ImportContext { + DestDir = destDir, SourceVirtualPath = sourceVirtual + }, log, progress); + return produced.Select(p => Describe(p.Uid, p.Path)).ToList(); } var stageRoot = Path.Combine(ProjectContext.CachePath, "import_staging", Guid.NewGuid().ToString("N")); - // Named like the real folder + // Match the live directory name so importers don't add a duplicate subfolder. var stageDest = Path.Combine(stageRoot, new DirectoryInfo(destDir).Name); Directory.CreateDirectory(stageDest); + IReadOnlyList<(string Uid, string Path)> producedInStage = []; + IReadOnlyList? applied = null; + var preserveStage = false; try { - var ctx = new ImportContext { DestDir = stageDest, SourceVirtualPath = sourceVirtual }; - await importer.Import(realSource, ctx, log, progress); - return Reconcile(Produced(writes, stageRoot), previous, stageDest, destDir, log); - } finally { - try { - Directory.Delete(stageRoot, true); - } catch { - /* the cache is disposable */ + producedInStage = await Generate(importer, realSource, new ImportContext { + DestDir = stageDest, SourceVirtualPath = sourceVirtual, IsStaging = true + }, log, progress); + var plan = new ArtworkReimport(previous, producedInStage, stageDest, destDir); + IReadOnlyDictionary decisions = new Dictionary(); + if (plan.Missing.Count > 0) { + var owner = ModalService.FindActiveWindow(); + var reviewed = owner is null ? null + : await new ReimportReviewWindow(sourceVirtual, plan.ModifiedCount, plan.Added, plan.Missing) + .ShowDialog?>(owner); + if (reviewed is null) { + log($"Skipped {sourceVirtual}"); + return null; + } + decisions = reviewed; } - } - } - - private static List<(string Uid, string Path)> Produced(List<(string Uid, string Path)> writes, string? root) { - var byUid = new Dictionary(); - foreach (var (uid, path) in writes) byUid[uid] = path; - return byUid - .Where(kv => File.Exists(kv.Value) && (root is null || kv.Value.StartsWith(root, StringComparison.OrdinalIgnoreCase))) - .Select(kv => (kv.Key, kv.Value)) - .ToList(); - } - private static IReadOnlyList Describe(IEnumerable<(string Uid, string Path)> produced) { - return produced.Select(p => Describe(p.Uid, p.Path)).ToList(); - } - - private static ArtworkOutput Describe(string uid, string realPath) { - return new ArtworkOutput(uid, Path.GetFileName(realPath), AssetDatabase.ComputeHash(realPath), - ProjectContext.ToVirtual(realPath) ?? realPath); - } - - private static IReadOnlyList Reconcile( - List<(string Uid, string Path)> produced, IReadOnlyList previous, - string stageDest, string destDir, Action log) { - var fresh = produced.Select(p => (p.Uid, p.Path, Name: Path.GetFileName(p.Path), - Hash: AssetDatabase.ComputeHash(p.Path))).ToList(); - - var free = previous.ToList(); - var matches = new Dictionary(); - - void Match(Func same) { - foreach (var f in fresh) { - if (matches.ContainsKey(f.Uid)) continue; - var old = free.FirstOrDefault(o => same(o, f)); - if (old is null) continue; - matches[f.Uid] = old; - free.Remove(old); - } - } - - Match((o, f) => o.Hash == f.Hash); - Match((o, f) => string.Equals(o.Name, f.Name, StringComparison.OrdinalIgnoreCase)); - - foreach (var f in fresh) { - if (matches.ContainsKey(f.Uid)) continue; - var old = free.FirstOrDefault(o => o.Hash is null); - if (old is null) break; - matches[f.Uid] = old; - free.Remove(old); - } - - var uidMap = matches.ToDictionary(m => m.Key, m => m.Value.Uid); - if (uidMap.Count > 0) - foreach (var file in Directory.EnumerateFiles(stageDest, "*", SearchOption.AllDirectories)) - ReplaceUids(file, uidMap); - - var outputs = new List(); - var moved = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var f in fresh) { - string target; - if (matches.TryGetValue(f.Uid, out var old)) { - var uri = AssetDatabase.TryResolve(old.Uid, out var current, out _) ? current : old.Uri; - target = ProjectContext.Resolve(uri); - } else { - target = Path.Combine(destDir, Path.GetRelativePath(stageDest, f.Path)); + applied = plan.Apply(decisions, stageRoot, log); + return applied; + } catch (AggregateException) { + // Keep original files available if failed + preserveStage = true; + throw; + } finally { + var previousUids = previous.Select(p => p.Uid).ToHashSet(); + if (applied is not null) previousUids.UnionWith(applied.Select(p => p.Uid)); + foreach (var (uid, _) in producedInStage) + if (!previousUids.Contains(uid)) + AssetDatabase.TryDelete(ThumbnailPath(uid)); + + if (!preserveStage) { + try { + AssetDatabase.DeleteDirectory(stageRoot); + } catch (IOException ex) { + log($"Could not clean up reimport staging files: {ex.Message}"); + } catch (UnauthorizedAccessException ex) { + log($"Could not clean up reimport staging files: {ex.Message}"); + } } - - var tags = MetaFile.ReadHeader(target)?.Tags; - Place(f.Path, target); - Place(f.Path + ".meta", target + ".meta"); - if (tags is { Count: > 0 }) MetaFile.SetTags(target, tags); - - moved.Add(f.Path); - moved.Add(f.Path + ".meta"); - var uid = matches.TryGetValue(f.Uid, out var kept) ? kept.Uid : f.Uid; - outputs.Add(Describe(uid, target)); - log($"{(kept is null ? "New" : "Replaced")} {ProjectContext.ToVirtual(target) ?? target}"); } - - foreach (var file in Directory.EnumerateFiles(stageDest, "*", SearchOption.AllDirectories)) - if (!moved.Contains(file)) - Place(file, Path.Combine(destDir, Path.GetRelativePath(stageDest, file))); - - return outputs; } - private static void Place(string from, string to) { - if (!File.Exists(from)) return; - Directory.CreateDirectory(Path.GetDirectoryName(to)!); - File.Move(from, to, true); + private static async Task> Generate( + IAssetImporter importer, string realSource, ImportContext context, Action log, + Action? progress) { + var writes = new List<(string Uid, string Path)>(); + using (MetaFile.Capture(writes)) + await importer.Import(realSource, context, log, progress); + + var produced = writes.GroupBy(w => w.Path, ArtworkReimport.PathComparer).Select(group => group.Last()).ToList(); + foreach (var output in produced) + if (!File.Exists(output.Path)) + throw new IOException($"Import did not generate {Path.GetFileName(output.Path)}."); + return produced; } - private static void ReplaceUids(string file, IReadOnlyDictionary uidMap) { - var bytes = File.ReadAllBytes(file); - if (Array.IndexOf(bytes, (byte)0, 0, Math.Min(bytes.Length, 8192)) >= 0) return; - - var text = System.Text.Encoding.UTF8.GetString(bytes); - var changed = text; - foreach (var (from, to) in uidMap) changed = changed.Replace(from, to, StringComparison.Ordinal); - if (changed != text) File.WriteAllText(file, changed); + private static ArtworkOutput Describe(string uid, string path) { + return new ArtworkOutput(uid, Path.GetFileName(path), AssetDatabase.ComputeHash(path), + ProjectContext.ToVirtual(path) ?? path); } + + internal static string ThumbnailPath(string uid) => ProjectContext.Resolve($"cache://thumbnails/{uid}.png"); } diff --git a/editor/Assets/Importers/ArtworkReimport.cs b/editor/Assets/Importers/ArtworkReimport.cs new file mode 100644 index 000000000..902aa61de --- /dev/null +++ b/editor/Assets/Importers/ArtworkReimport.cs @@ -0,0 +1,284 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; + +namespace editor.Assets.Importers; + +internal enum MissingAssetAction { Keep, Replace, Delete } +internal sealed record MissingAssetResolution(MissingAssetAction Action, ReimportAsset? Replacement = null); +internal sealed record ReimportAsset(string Uid, string Path, string Name, string Type, string Label); + +internal sealed class ArtworkReimport { + internal static readonly StringComparer PathComparer = + OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + + private static readonly Regex s_nodeUid = new( + @"^(m_uid|m_parent) @\S+ = ([A-Za-z0-9_-]{11})(?=\r?$)", RegexOptions.Multiline); + private readonly List m_matches = []; + private readonly Dictionary m_originalTimes; + private readonly string m_destDir; + + private sealed record Match(ReimportAsset Old, ReimportAsset New, bool SameContent, string Destination); + + public ArtworkReimport(IReadOnlyList previous, IReadOnlyList<(string Uid, string Path)> generated, + string stageDest, string destDir) { + m_destDir = destDir; + var old = previous.Select(output => ReadAsset(output.Uid, ProjectContext.Resolve(output.Uri), output.Name, + output.Uri)).ToList(); + var fresh = generated.Select(output => ReadAsset(output.Uid, output.Path, Path.GetFileName(output.Path), + Path.GetRelativePath(stageDest, output.Path))).OrderBy(asset => asset.Label, PathComparer).ToList(); + var storedHashes = previous.ToDictionary(asset => asset.Uid, asset => asset.Hash); + m_originalTimes = old.ToDictionary(asset => asset, asset => LastWriteTimes(asset.Path)); + var oldHashes = old.ToDictionary(asset => asset, asset => File.Exists(asset.Path) ? CompareHash(asset.Path) : null); + var uidMap = new Dictionary(); + + // Resolve references to earlier outputs before comparing + foreach (var group in fresh.GroupBy(asset => DependencyOrder(asset.Path)).OrderBy(group => group.Key)) { + foreach (var asset in group) RemapReferences(asset.Path, uidMap); + var hashes = group.ToDictionary(asset => asset, asset => CompareHash(asset.Path)); + + bool SameName(ReimportAsset previousAsset, ReimportAsset generatedAsset) => + string.Equals(previousAsset.Name, generatedAsset.Name, StringComparison.OrdinalIgnoreCase); + + bool SamePath(ReimportAsset previousAsset, ReimportAsset generatedAsset) => + PathComparer.Equals(previousAsset.Path, GeneratedDestination(generatedAsset)); + + bool SameHash(ReimportAsset previousAsset, ReimportAsset generatedAsset) => + oldHashes[previousAsset] == hashes[generatedAsset] || + storedHashes[previousAsset.Uid] == hashes[generatedAsset]; + + void MatchAssets(Func predicate) { + foreach (var asset in group) { + if (uidMap.ContainsKey(asset.Uid)) continue; + var candidates = old.Where(candidate => candidate.Type == asset.Type && predicate(candidate, asset)); + var match = candidates.FirstOrDefault(candidate => SamePath(candidate, asset)) ?? + candidates.FirstOrDefault(); + if (match is null) continue; + old.Remove(match); + uidMap.Add(asset.Uid, match.Uid); + + var sameContent = SameHash(match, asset) && File.Exists(match.Path); + var rename = SameHash(match, asset) && + !string.Equals(match.Name, asset.Name, StringComparison.Ordinal); + var destination = rename ? Path.Combine(Path.GetDirectoryName(match.Path)!, asset.Name) : match.Path; + m_matches.Add(new Match(match, asset, sameContent, destination)); + } + } + + // Prefer a matching name among identical + MatchAssets((previousAsset, generatedAsset) => SameHash(previousAsset, generatedAsset) && + SamePath(previousAsset, generatedAsset)); + MatchAssets((previousAsset, generatedAsset) => SameHash(previousAsset, generatedAsset) && + SameName(previousAsset, generatedAsset)); + MatchAssets(SameHash); + MatchAssets(SamePath); + MatchAssets(SameName); + } + + Added = fresh.Where(asset => !uidMap.ContainsKey(asset.Uid)).ToList(); + Missing = old; + } + + public IReadOnlyList Added { get; } + public IReadOnlyList Missing { get; } + public int ModifiedCount => m_matches.Count(match => !match.SameContent || + !string.Equals(match.Old.Path, match.Destination, StringComparison.Ordinal)); + + public IReadOnlyList Apply(IReadOnlyDictionary decisions, + string stageRoot, Action log) { + var matches = new List(m_matches); + var added = new List(Added); + var removed = new List(); + var replacements = new Dictionary(); + foreach (var asset in Missing) { + var resolution = decisions[asset.Uid]; + if (resolution.Action == MissingAssetAction.Keep) continue; + if (resolution.Action == MissingAssetAction.Delete) { + removed.Add(asset); + continue; + } + + var replacement = resolution.Replacement!; + if (asset.Type != replacement.Type || !added.Remove(replacement)) + throw new InvalidOperationException($"Invalid replacement for {asset.Label}."); + replacements.Add(replacement.Uid, asset.Uid); + var destination = Path.Combine(Path.GetDirectoryName(asset.Path)!, replacement.Name); + matches.Add(new Match(asset, replacement, false, destination)); + } + + // keep the missing asset's UID + foreach (var asset in matches.Select(match => match.New).Concat(added)) + RemapReferences(asset.Path, replacements); + + var placements = new List<(string From, string To)>(); + var outputs = new List(); + var thumbnails = new List<(string From, string To)>(); + foreach (var match in matches) { + var previous = VerifyUnchanged(match.Old); + + var sameContent = match.SameContent && (replacements.Count == 0 || + CompareHash(match.New.Path) == CompareHash(match.Old.Path)); + if (sameContent && string.Equals(match.Old.Path, match.Destination, StringComparison.Ordinal)) { + outputs.Add(DescribeDestination(match.Old.Uid, match.New, match.Destination)); + continue; + } + + var source = match.New.Path; + if (sameContent) { + placements.Add((match.Old.Path, match.Destination)); + placements.Add((match.Old.Path + ".meta", match.Destination + ".meta")); + } else { + MetaFile.ReuseIdentity(source, previous); + placements.Add((source, match.Destination)); + placements.Add((source + ".meta", match.Destination + ".meta")); + thumbnails.Add((match.New.Uid, match.Old.Uid)); + } + + removed.Add(match.Old); + outputs.Add(DescribeDestination(match.Old.Uid, match.New, match.Destination)); + } + + foreach (var asset in added) { + var target = GeneratedDestination(asset); + placements.Add((asset.Path, target)); + placements.Add((asset.Path + ".meta", target + ".meta")); + outputs.Add(DescribeDestination(asset.Uid, asset, target)); + } + + foreach (var asset in removed) VerifyUnchanged(asset); + + PlaceFiles(placements, removed.SelectMany(asset => new[] { asset.Path, asset.Path + ".meta" }), stageRoot); + foreach (var (from, to) in thumbnails) { + if (from == to) continue; + var thumbnail = ArtworkImport.ThumbnailPath(from); + var target = ArtworkImport.ThumbnailPath(to); + try { + AssetDatabase.TryDelete(target); + if (File.Exists(thumbnail)) File.Move(thumbnail, target); + } catch (IOException ex) { + log($"Could not update thumbnail for {to}: {ex.Message}"); + } catch (UnauthorizedAccessException ex) { + log($"Could not update thumbnail for {to}: {ex.Message}"); + } + } + + foreach (var asset in Missing.Where(asset => decisions[asset.Uid].Action == MissingAssetAction.Delete)) + AssetDatabase.TryDelete(ArtworkImport.ThumbnailPath(asset.Uid)); + return outputs; + } + + private static void PlaceFiles(IReadOnlyList<(string From, string To)> placements, + IEnumerable removals, string stageRoot) { + var existing = removals.Where(File.Exists).ToHashSet(PathComparer); + var destinations = new HashSet(PathComparer); + foreach (var (_, to) in placements) { + if (!destinations.Add(to) || (File.Exists(to) && !existing.Contains(to)) || Directory.Exists(to)) + throw new IOException($"Reimport would overwrite an asset at {ProjectContext.ToVirtual(to) ?? to}."); + } + + var backups = new Dictionary(PathComparer); + var installed = new List(); + try { + foreach (var path in existing) { + var backup = Path.Combine(stageRoot, $"backup-{backups.Count}"); + File.Move(path, backup); + backups.Add(path, backup); + } + + foreach (var (from, to) in placements) { + Directory.CreateDirectory(Path.GetDirectoryName(to)!); + using var input = File.OpenRead(backups.GetValueOrDefault(from, from)); + using var output = File.Open(to, FileMode.CreateNew, FileAccess.Write); + installed.Add(to); + input.CopyTo(output); + } + } catch (Exception failure) { + try { + foreach (var path in installed) { + File.SetAttributes(path, File.GetAttributes(path) & ~FileAttributes.ReadOnly); + File.Delete(path); + } + foreach (var (path, backup) in backups) File.Move(backup, path); + } catch (Exception restoreFailure) { + throw new AggregateException($"Could not restore reimport files. Backups are in {stageRoot}.", + failure, restoreFailure); + } + + throw; + } + } + + private string GeneratedDestination(ReimportAsset asset) => Path.Combine(m_destDir, asset.Label); + + private static (DateTime Asset, DateTime Meta) LastWriteTimes(string path) => + (File.GetLastWriteTimeUtc(path), File.GetLastWriteTimeUtc(path + ".meta")); + + private MetaHeader VerifyUnchanged(ReimportAsset asset) { + var header = MetaFile.ReadHeader(asset.Path); + if (header?.Uid != asset.Uid || LastWriteTimes(asset.Path) != m_originalTimes[asset]) + throw new IOException($"Asset changed during reimport: {asset.Label}."); + return header; + } + + private static ReimportAsset ReadAsset(string uid, string path, string name, string label) => + new(uid, path, name, MetaFile.ReadHeader(path)?.Type ?? + throw new IOException($"Missing asset metadata for {label}."), label); + + private static ArtworkOutput DescribeDestination(string uid, ReimportAsset generated, string destination) => + new(uid, generated.Name, AssetDatabase.ComputeHash(generated.Path), + ProjectContext.ToVirtual(destination) ?? destination); + + private static int DependencyOrder(string path) => Path.GetExtension(path).ToLowerInvariant() switch { + ".tnode" => 2, + ".tmat" or ".tvox" => 1, + _ => 0 + }; + + private static string CompareHash(string path) { + if (!path.EndsWith(".tnode", StringComparison.OrdinalIgnoreCase)) return AssetDatabase.ComputeHash(path); + + // Compare their topology + var text = File.ReadAllText(path).ReplaceLineEndings("\n"); + var ids = new Dictionary(); + foreach (System.Text.RegularExpressions.Match match in s_nodeUid.Matches(text)) + if (match.Groups[1].Value == "m_uid") ids.TryAdd(match.Groups[2].Value, ids.Count); + text = s_nodeUid.Replace(text, match => ids.TryGetValue(match.Groups[2].Value, out var index) + ? match.Groups[1].Value + " = " + index + : match.Value); + return "sha256:" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text))).ToLowerInvariant(); + } + + private static void RemapReferences(string path, IReadOnlyDictionary uidMap) { + if (uidMap.Count == 0) return; + switch (Path.GetExtension(path).ToLowerInvariant()) { + case ".tnode": + case ".tmat": { + var original = File.ReadAllText(path); + var updated = original; + foreach (var (from, to) in uidMap) + updated = updated.Replace(from, to, StringComparison.Ordinal); + if (updated != original) File.WriteAllText(path, updated); + break; + } + case ".tvox": { + const int paletteOffset = 16; + using var stream = File.Open(path, FileMode.Open, FileAccess.ReadWrite); + Span uidBytes = stackalloc byte[8]; + stream.Position = paletteOffset; + stream.ReadExactly(uidBytes); + var uid = UidGenerator.EncodeNative(BinaryPrimitives.ReadUInt64LittleEndian(uidBytes)); + if (!uidMap.TryGetValue(uid, out var target) || uid == target) break; + var nativeBytes = Convert.FromBase64String(target.Replace('-', '+').Replace('_', '/') + "="); + BinaryPrimitives.WriteUInt64LittleEndian(uidBytes, BinaryPrimitives.ReadUInt64BigEndian(nativeBytes)); + stream.Position = paletteOffset; + stream.Write(uidBytes); + break; + } + } + } +} diff --git a/editor/Assets/Importers/AudioBankImporter.cs b/editor/Assets/Importers/AudioBankImporter.cs index 4c27f5ecf..1fb4c3d2a 100644 --- a/editor/Assets/Importers/AudioBankImporter.cs +++ b/editor/Assets/Importers/AudioBankImporter.cs @@ -30,7 +30,7 @@ public Task> Import( File.Copy(realSourcePath, destPath, true); log("Writing .meta sidecar..."); - var uid = ctx.UidFor(0); + var uid = UidGenerator.Generate(); var header = new MetaHeader { Uid = uid, Type = PrimaryOutputType.Type, Source = ctx.SourceVirtualPath }; MetaFile.Write(destPath, header); return Task.FromResult>([uid]); diff --git a/editor/Assets/Importers/AudioStringImporter.cs b/editor/Assets/Importers/AudioStringImporter.cs index 80d6047c3..27dd12653 100644 --- a/editor/Assets/Importers/AudioStringImporter.cs +++ b/editor/Assets/Importers/AudioStringImporter.cs @@ -87,7 +87,7 @@ public async Task> Import( log("Importing strings bank..."); var destPath = Path.Combine(ctx.DestDir, Path.GetFileName(realSourcePath)); File.Copy(realSourcePath, destPath, true); - var uid = ctx.UidFor(current); + var uid = UidGenerator.Generate(); uids.Add(uid); var header = new MetaHeader { Uid = uid, Type = AssetTypeRegistry.ByType("audio_strings")!.Type, Source = ctx.SourceVirtualPath }; @@ -121,7 +121,7 @@ async Task ImportObjects(JsonObject obj, string type, string ext) { destPath = Path.Combine(ctx.DestDir, $"{name}{ext}"); } - uid = ctx.UidFor(current); + uid = UidGenerator.Generate(); uids.Add(uid); header = new MetaHeader { Uid = uid, diff --git a/editor/Assets/Importers/CompactImportWindowViewModel.cs b/editor/Assets/Importers/CompactImportWindowViewModel.cs index e04550f98..b300e38ae 100644 --- a/editor/Assets/Importers/CompactImportWindowViewModel.cs +++ b/editor/Assets/Importers/CompactImportWindowViewModel.cs @@ -129,6 +129,7 @@ private async Task ImportFile(string filePath, string? artworkDir, Action> Import( File.Copy(realSourcePath, destPath, true); log("Writing .meta sidecar..."); - var uid = ctx.UidFor(0); + var uid = UidGenerator.Generate(); var header = new MetaHeader { Uid = uid, Type = PrimaryOutputType.Type, Source = ctx.SourceVirtualPath }; MetaFile.Write(destPath, header); return Task.FromResult>([uid]); diff --git a/editor/Assets/Importers/GltfImporter.cs b/editor/Assets/Importers/GltfImporter.cs index 1325a38a2..d28f3dceb 100644 --- a/editor/Assets/Importers/GltfImporter.cs +++ b/editor/Assets/Importers/GltfImporter.cs @@ -301,8 +301,7 @@ void ReportProgress() { } } - log("Rebuilding asset database..."); - AssetDatabase.RebuildAssetDatabase(); + if (!ctx.IsStaging) AssetDatabase.RebuildAssetDatabase(); // Scenes log("Updating scene intermediates with UIDs..."); @@ -365,8 +364,7 @@ void PatchNode(JsonNode node) { ReportProgress(); } - log("Rebuilding asset database..."); - AssetDatabase.RebuildAssetDatabase(); + if (!ctx.IsStaging) AssetDatabase.RebuildAssetDatabase(); log("Removing intermediates..."); if (Directory.Exists(tempDir.FullName)) Directory.Delete(tempDir.FullName, true); diff --git a/editor/Assets/Importers/IAssetImporter.cs b/editor/Assets/Importers/IAssetImporter.cs index 22b24480b..0079abacf 100644 --- a/editor/Assets/Importers/IAssetImporter.cs +++ b/editor/Assets/Importers/IAssetImporter.cs @@ -21,15 +21,7 @@ public static bool AlreadyNamed(string dir, string name) { } public required string SourceVirtualPath { get; init; } - // On a reimport we want the regenerated assets to keep the UIDs they had before - public IReadOnlyList? ReuseUids { get; init; } - - // nth output keeps its old UID if we have one, otherwise a new UID is generated - public string UidFor(int outputIndex) { - return ReuseUids is { } uids && outputIndex < uids.Count - ? uids[outputIndex] - : UidGenerator.Generate(); - } + public bool IsStaging { get; init; } } /// Interface for file format importers diff --git a/editor/Assets/Importers/ImportWindowViewModel.cs b/editor/Assets/Importers/ImportWindowViewModel.cs index bde1a3536..19b290753 100644 --- a/editor/Assets/Importers/ImportWindowViewModel.cs +++ b/editor/Assets/Importers/ImportWindowViewModel.cs @@ -574,6 +574,7 @@ private async Task ImportSingleFile( var importer = m_importers.First(i => i.CanHandle(realSourcePath)); var outputs = await ArtworkImport.RunAsync(importer, realSourcePath, sourceVirtual, destDir, AssetDatabase.GetArtworkOutputs(sourceVirtual), log, progress); + if (outputs is null) return; AssetDatabase.UpdateArtworkDatabase(sourceVirtual, hash, outputs); log("Done."); } catch (Exception ex) { diff --git a/editor/Assets/Importers/PsdImporter.cs b/editor/Assets/Importers/PsdImporter.cs index 1d3199044..570d1c751 100644 --- a/editor/Assets/Importers/PsdImporter.cs +++ b/editor/Assets/Importers/PsdImporter.cs @@ -76,7 +76,7 @@ await Task.Run(() => { collection[0].Write(tempPng); tempFiles.Add(tempPng); outputs.Add((tempPng, Path.Combine(destDir, baseName + ".ktx2"), - ctx.UidFor(outputs.Count), baseName)); + UidGenerator.Generate(), baseName)); break; } case PsdImportMode.Layers: { @@ -89,7 +89,7 @@ await Task.Run(() => { img.Write(tempPng); tempFiles.Add(tempPng); outputs.Add((tempPng, Path.Combine(destDir, safeName + ".ktx2"), - ctx.UidFor(outputs.Count), label)); + UidGenerator.Generate(), label)); } break; @@ -115,7 +115,7 @@ await Task.Run(() => { folderCollection[0].Write(tempPng); tempFiles.Add(tempPng); outputs.Add((tempPng, Path.Combine(destDir, $"{baseName}_{folderName}.ktx2"), - ctx.UidFor(outputs.Count), folderName)); + UidGenerator.Generate(), folderName)); } break; diff --git a/editor/Assets/Importers/TextureImporter.cs b/editor/Assets/Importers/TextureImporter.cs index 17cdc9032..dca513a51 100644 --- a/editor/Assets/Importers/TextureImporter.cs +++ b/editor/Assets/Importers/TextureImporter.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -83,7 +83,7 @@ public IReadOnlyList GetSettings() { public async Task> Import( string realSourcePath, ImportContext ctx, Action log, Action? progress = null) { - var uid = ctx.UidFor(0); + var uid = UidGenerator.Generate(); var name = Path.GetFileNameWithoutExtension(realSourcePath); var destPath = Path.Combine(ctx.DestDir, name + ".ktx2"); Directory.CreateDirectory(ctx.DestDir); diff --git a/editor/Assets/Importers/UIImageImporter.cs b/editor/Assets/Importers/UIImageImporter.cs index 60716dec0..d5bd6ca68 100644 --- a/editor/Assets/Importers/UIImageImporter.cs +++ b/editor/Assets/Importers/UIImageImporter.cs @@ -30,7 +30,7 @@ public Task> Import( File.Copy(realSourcePath, destPath, true); log("Writing .meta sidecar..."); - var uid = ctx.UidFor(0); + var uid = UidGenerator.Generate(); var header = new MetaHeader { Uid = uid, Type = PrimaryOutputType.Type, Source = ctx.SourceVirtualPath }; MetaFile.Write(destPath, header); diff --git a/editor/Assets/Importers/VoxImporter.cs b/editor/Assets/Importers/VoxImporter.cs index 8aa53dde8..4526b741b 100644 --- a/editor/Assets/Importers/VoxImporter.cs +++ b/editor/Assets/Importers/VoxImporter.cs @@ -80,16 +80,11 @@ string UidOf(string destPath) { log($"Reading {Path.GetFileName(realSourcePath)}..."); await Task.Run(() => vox_generate_intermediates(realSourcePath, tempDir, paletteUid)); - // An empty directory means the native parse failed var temp = new DirectoryInfo(tempDir); - var models = temp.Exists ? temp.GetFiles("*.tvox").OrderBy(f => f.Name).ToList() : []; - // we need to allow when we only have a palette and no models - //if (models.Count == 0) { - // throw new Exception( - // $"Voxel import produced no models in {tempDir} - check the editor log for the underlying error"); - //} - - var manifest = temp.GetFiles("*.json").FirstOrDefault(); + var manifest = temp.Exists ? temp.GetFiles("*.json").FirstOrDefault() : null; + if (manifest is null) + throw new InvalidOperationException($"Voxel import produced no manifest for {realSourcePath}."); + var models = temp.GetFiles("*.tvox").OrderBy(f => f.Name).ToList(); var importedUids = new List(); var totalItems = models.Count + 2; var doneItems = 0; @@ -143,7 +138,7 @@ void ReportProgress() { doneItems++; ReportProgress(); - if (m_settings.GeneratePrefab && manifest is not null) { + if (m_settings.GeneratePrefab) { log("Updating the scene manifest with UIDs..."); var json = await File.ReadAllTextAsync(manifest.FullName); foreach (var (fileName, uid) in modelUids) { @@ -168,8 +163,7 @@ void ReportProgress() { doneItems++; ReportProgress(); - log("Rebuilding asset database..."); - AssetDatabase.RebuildAssetDatabase(); + if (!ctx.IsStaging) AssetDatabase.RebuildAssetDatabase(); log("Removing intermediates..."); if (Directory.Exists(tempDir)) Directory.Delete(tempDir, true); diff --git a/editor/Assets/MetaFile.cs b/editor/Assets/MetaFile.cs index bb45ed18c..c2a8a7a53 100644 --- a/editor/Assets/MetaFile.cs +++ b/editor/Assets/MetaFile.cs @@ -260,6 +260,15 @@ public static bool UpdateSource(string path, string? newSource) { } } + internal static void ReuseIdentity(string assetPath, MetaHeader previous) { + var metaPath = assetPath + ".meta"; + var dto = TomlSerializer.Deserialize(File.ReadAllText(metaPath))!; + dto.Uid = previous.Uid; + dto.CreatedAt = previous.CreatedAt; + dto.Tags = previous.Tags?.ToList(); + File.WriteAllText(metaPath, TomlSerializer.Serialize(dto)); + } + /// Rewrites the modified_at of an existing .meta in place; public static bool Touch(string virtualPath) { var metaPath = ProjectContext.Resolve(virtualPath); diff --git a/editor/Assets/ProjectContext.cs b/editor/Assets/ProjectContext.cs index 3b3f411b8..e1f9157eb 100644 --- a/editor/Assets/ProjectContext.cs +++ b/editor/Assets/ProjectContext.cs @@ -43,6 +43,7 @@ public static void Reset() { DataSchemaStubGenerator.StopWatching(); SchemaEditorState.Reset(); AssetDatabase.Reset(); + AssetStore.Close(); IsInitialized = false; ProjectPath = ArtworkPath = AssetsPath = CachePath = CorePath = SavedPath = ""; Databases = ["assets"]; @@ -66,6 +67,7 @@ public static void Initialize(string projectPath, string corePath) { RegisterSchemes(); EnsureDirectories(); IsInitialized = true; + AssetStore.Open(); AssetBrowserSettings.Load(); UIBindStubGenerator.Generate(); UIBindStubGenerator.StartWatching(); diff --git a/editor/Components/Modals/ReimportReviewWindow.axaml b/editor/Components/Modals/ReimportReviewWindow.axaml new file mode 100644 index 000000000..f86971b5e --- /dev/null +++ b/editor/Components/Modals/ReimportReviewWindow.axaml @@ -0,0 +1,147 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/editor/Components/Modals/ReimportReviewWindow.axaml.cs b/editor/Components/Modals/ReimportReviewWindow.axaml.cs new file mode 100644 index 000000000..8b54e93a8 --- /dev/null +++ b/editor/Components/Modals/ReimportReviewWindow.axaml.cs @@ -0,0 +1,127 @@ +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using Avalonia; +using Avalonia.Controls; +using Avalonia.Interactivity; +using Avalonia.Media; +using CommunityToolkit.Mvvm.ComponentModel; +using editor.Assets.Importers; +using editor.Assets.Types; +using Lucide.Avalonia; + +namespace editor.Components.Modals; + +internal sealed record ReimportReviewViewModel(string Source, int AddedCount, int ModifiedCount, + IReadOnlyList Missing) { + public int MissingCount => Missing.Count; +} + +internal sealed partial class ReimportMissingRow(ReimportAsset asset) : ObservableObject { + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(ShowKeep), nameof(ShowReplace), nameof(ShowDelete))] + [NotifyPropertyChangedFor(nameof(KeepTip), nameof(ReplaceTip), nameof(DeleteTip))] + [NotifyPropertyChangedFor(nameof(HasResolution), nameof(HasReplacement), nameof(ReplacementLabel))] + private MissingAssetResolution? m_resolution; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(ReplaceTip))] + private bool m_canReplace; + + public ReimportAsset Asset { get; } = asset; + public string Label => Asset.Label; + public bool HasResolution => Resolution is not null; + public bool ShowKeep => Resolution is null or { Action: MissingAssetAction.Keep }; + public bool ShowReplace => Resolution is null or { Action: MissingAssetAction.Replace }; + public bool ShowDelete => Resolution is null or { Action: MissingAssetAction.Delete }; + public string KeepTip => Resolution is null ? "Keep asset" : "Undo keep"; + public string ReplaceTip => Resolution is not null ? "Undo replacement" + : CanReplace ? "Choose replacement" : "No new assets of the same type are available"; + public string DeleteTip => Resolution is null ? "Delete asset" : "Undo delete"; + public bool HasReplacement => Resolution?.Replacement is not null; + public string? ReplacementLabel => Resolution?.Replacement is { } replacement ? $"Replace with {replacement.Label}" : null; +} + +public partial class ReimportReviewWindow : Window { + private readonly IReadOnlyList m_added = []; + private readonly IReadOnlyList m_rows = []; + + public ReimportReviewWindow() { + InitializeComponent(); + } + + internal ReimportReviewWindow(string source, int modifiedCount, + IReadOnlyList added, IReadOnlyList missing) : this() { + m_added = added; + m_rows = missing.Select(asset => new ReimportMissingRow(asset)).ToList(); + DataContext = new ReimportReviewViewModel(source, added.Count, modifiedCount, m_rows); + UpdateActions(); + } + + private void OnKeep(object? sender, RoutedEventArgs e) => ToggleAction(sender, MissingAssetAction.Keep); + private void OnDelete(object? sender, RoutedEventArgs e) => ToggleAction(sender, MissingAssetAction.Delete); + + private void ToggleAction(object? sender, MissingAssetAction action) { + if (sender is not Button { DataContext: ReimportMissingRow row }) return; + row.Resolution = row.Resolution is null ? new MissingAssetResolution(action) : null; + UpdateActions(); + } + + private async void OnReplace(object? sender, RoutedEventArgs e) { + if (sender is not Button { DataContext: ReimportMissingRow row }) return; + if (row.Resolution is not null) { + row.Resolution = null; + } else { + var available = AvailableReplacements(row).ToList(); + var picker = new ReplacementPicker(new ReplacementPickerViewModel(row.Asset, available)); + var uid = await picker.ShowDialog(this); + if (available.FirstOrDefault(asset => asset.Uid == uid) is not { } replacement) return; + row.Resolution = new MissingAssetResolution(MissingAssetAction.Replace, replacement); + } + UpdateActions(); + } + + private IEnumerable AvailableReplacements(ReimportMissingRow row) => + m_added.Where(asset => asset.Type == row.Asset.Type && + m_rows.All(other => other.Resolution?.Replacement != asset)); + + private void UpdateActions() { + foreach (var row in m_rows) + row.CanReplace = row.HasReplacement || AvailableReplacements(row).Any(); + ApplyButton.IsEnabled = m_rows.All(row => row.Resolution is not null); + } + + private void OnCancel(object? sender, RoutedEventArgs e) => Close(); + + private void OnApply(object? sender, RoutedEventArgs e) { + if (ApplyButton.IsEnabled) + Close(m_rows.ToDictionary(row => row.Asset.Uid, row => row.Resolution!)); + } + + private sealed class ReplacementPicker(ReplacementPickerViewModel vm) : PickerWindow(vm); + + private sealed class ReplacementPickerViewModel : PickerViewModel { + private readonly List m_all; + private IReadOnlyList m_filtered; + + public ReplacementPickerViewModel(ReimportAsset old, IReadOnlyList replacements) { + WindowTitle = $"Replace {old.Name}"; + var definition = AssetTypeRegistry.ByType(old.Type); + var color = Application.Current!.TryGetResource(definition?.ChipColor ?? "Blue", null, out var value) + ? value as IBrush ?? Brushes.DodgerBlue : Brushes.DodgerBlue; + m_all = replacements.Select(asset => new AssetPickerItem(asset.Uid, asset.Label, asset.Label, color, + definition?.ChipText ?? old.Type, definition?.HasThumbnail == true, showPath: false)).ToList(); + m_filtered = m_all; + } + + public override string WindowTitle { get; } + public override string AcceptLabel => "Replace"; + public override LucideIconKind AcceptIconKind => LucideIconKind.Replace; + public override IEnumerable Items => m_filtered; + + public override void UpdateFilter(string query, bool caseSensitive) => + m_filtered = m_all.Where(item => item.UpdateSegments(query, caseSensitive)).ToList(); + + public override string? GetResult(object? selected) => (selected as AssetPickerItem)?.Uid; + } +} diff --git a/editor/StartWindow/StartWindowViewModel.cs b/editor/StartWindow/StartWindowViewModel.cs index a9cb40f64..6f1b0708f 100644 --- a/editor/StartWindow/StartWindowViewModel.cs +++ b/editor/StartWindow/StartWindowViewModel.cs @@ -91,8 +91,6 @@ private void LaunchProject(string projectPath, Window parentWindow) { } tasks.Add(LoaderTask.Do("Check for missing files", AssetDatabase.RelocateMissingAssets)); - tasks.Add(LoaderTask.Do("Sync artwork database", - async log => { await Task.Run(AssetDatabase.SyncArtworkDatabase); })); tasks.Add(LoaderTask.Do("Check for missing artwork", AssetDatabase.RelocateMissingArtwork)); tasks.Add(LoaderTask.Do("Generate missing metadata", @@ -101,9 +99,12 @@ private void LaunchProject(string projectPath, Window parentWindow) { tasks.Add(LoaderTask.Do("Generate asset database", async log => { await Task.Run(() => AssetDatabase.RebuildAssetDatabase(false)); log( - $"Rebuilt asset database ({string.Join(", ", ProjectContext.Databases.Select(db => db + "://"))}, core://)"); + $"Synced asset database from .meta files ({string.Join(", ", ProjectContext.Databases.Select(db => db + "://"))}, core://)"); })); + tasks.Add(LoaderTask.Do("Sync artwork database", + async log => { await Task.Run(() => AssetDatabase.SyncArtworkHashes(log)); })); + tasks.Add(LoaderTask.Do("Check for artwork changes", AssetDatabase.CheckArtworkChanges)); tasks.Add(LoaderTask.Do("Generate missing thumbnails", diff --git a/editor/Workspace/MainWindowViewModel.cs b/editor/Workspace/MainWindowViewModel.cs index a89caf1c9..1a5370f0f 100644 --- a/editor/Workspace/MainWindowViewModel.cs +++ b/editor/Workspace/MainWindowViewModel.cs @@ -3,7 +3,6 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Avalonia; @@ -1191,7 +1190,7 @@ async Task CopyBakedLighting(Action log) { } } - async Task BakeAndPack(Action log, string dbName, string dbSourceDir, string manifestJsonPath) { + async Task BakeAndPack(Action log, string dbName, string dbSourceDir) { var stageDir = Path.Combine(stageRoot, dbName); if (Directory.Exists(stageDir)) { foreach (var f in Directory.EnumerateFiles(stageDir, "*", SearchOption.AllDirectories)) @@ -1205,6 +1204,8 @@ async Task BakeAndPack(Action log, string dbName, string dbSourceDir, st await Task.Run(() => { foreach (var srcFile in Directory.EnumerateFiles(dbSourceDir, "*", SearchOption.AllDirectories)) { var rel = Path.GetRelativePath(dbSourceDir, srcFile); + if (rel.EndsWith(".meta", StringComparison.OrdinalIgnoreCase)) + continue; var dest = Path.Combine(stageDir, rel); Directory.CreateDirectory(Path.GetDirectoryName(dest)!); File.Copy(srcFile, dest, true); @@ -1212,32 +1213,13 @@ await Task.Run(() => { } }); - // Read the manifest JSON to find all node UIDs, then overwrite their staged files with binary log($" baking nodes in {dbName}/ ..."); - if (File.Exists(manifestJsonPath)) { - var manifestJson = JsonNode.Parse(File.ReadAllText(manifestJsonPath)); - if (manifestJson?["node"] is JsonObject nodeCollection) - foreach (var (uid, pathNode) in nodeCollection) { - var virtualPath = pathNode?.GetValue(); - if (virtualPath is null) continue; - var sep = virtualPath.IndexOf("://", StringComparison.Ordinal); - if (sep < 0) continue; - var rel = virtualPath[(sep + 3)..]; - var dest = Path.Combine(stageDir, rel.Replace('/', Path.DirectorySeparatorChar)); - Directory.CreateDirectory(Path.GetDirectoryName(dest)!); - log($" bake {rel}"); - if (!await Task.Run(() => ToastEngine.BakeAsset(uid, dest))) - throw new Exception($"could not bake node {virtualPath}"); - } - } - - // Copy the manifest JSON into the stage folder root - var manifestDest = Path.Combine(stageDir, $"{dbName}.json"); - if (File.Exists(manifestJsonPath)) { - log($" copying manifest {dbName}.json"); - await Task.Run(() => File.Copy(manifestJsonPath, manifestDest, true)); - } else { - log($" warning: manifest not found at {manifestJsonPath}"); + foreach (var node in AssetStore.ListByType("node").Where(n => n.Db == dbName)) { + var dest = Path.Combine(stageDir, node.Path.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(dest)!); + log($" bake {node.Path}"); + if (!await Task.Run(() => ToastEngine.BakeAsset(node.Uid, dest))) + throw new Exception($"could not bake node {node.Virtual}"); } // Run the Rust packer on the staging directory @@ -1298,17 +1280,24 @@ await Task.Run(() => { }; // Bake assets + var paks = new List<(string Db, string File)>(); foreach (var db in ProjectContext.Databases) { var dbSource = Path.Combine(ProjectContext.ProjectPath, db); - var manifestJs = Path.Combine(ProjectContext.CachePath, $"{db}.json"); var dbCapture = db; - tasks.Add(LoaderTask.Do($"bake & pack {db}://", log => BakeAndPack(log, dbCapture, dbSource, manifestJs))); + paks.Add((db, $"{db}.pak")); + tasks.Add(LoaderTask.Do($"bake & pack {db}://", log => BakeAndPack(log, dbCapture, dbSource))); } // Bake core - var coreManifest = Path.Combine(ProjectContext.CachePath, "core.json"); + paks.Add((AssetStore.CoreDb, $"{AssetStore.CoreDb}.pak")); tasks.Add(LoaderTask.Do("bake & pack core://", - log => BakeAndPack(log, "core", ProjectContext.CorePath, coreManifest))); + log => BakeAndPack(log, AssetStore.CoreDb, ProjectContext.CorePath))); + + tasks.Add(LoaderTask.Do("write game.db", async log => { + var path = Path.Combine(outputDir, AssetStore.RuntimeDatabaseName); + await Task.Run(() => AssetStore.ExportRuntimeDatabase(path, paks)); + log($" wrote {AssetStore.RuntimeDatabaseName} ({paks.Count} pak(s))"); + })); var vm = new LoaderViewModel(tasks); await new SimpleLoaderWindow(vm).ShowDialog(owner); diff --git a/editor/editor.csproj b/editor/editor.csproj index 9a90e00ba..3b8e86899 100644 --- a/editor/editor.csproj +++ b/editor/editor.csproj @@ -62,6 +62,7 @@ + diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 6b336d73c..83f1ece93 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -37,6 +37,7 @@ find_package(slang CONFIG REQUIRED) find_package(protobuf CONFIG REQUIRED) find_package(tinysplinecxx CONFIG REQUIRED) find_package(lz4 CONFIG REQUIRED) +find_package(unofficial-sqlite3 CONFIG REQUIRED) find_package(tomlplusplus CONFIG REQUIRED) find_package(RmlUi CONFIG REQUIRED) find_package(Lua REQUIRED) @@ -106,6 +107,7 @@ target_link_libraries(toast_engine unofficial::VulkanMemoryAllocator-Hpp::VulkanMemoryAllocator-Hpp slang::gfx slang::slang protobuf::libprotobuf tinysplinecxx::tinysplinecxx LZ4::lz4 + unofficial::sqlite3::sqlite3 toast_pack $<$,$>:imgui::imgui> RmlUi::RmlUi diff --git a/engine/ffi/engine.h b/engine/ffi/engine.h index 1901819aa..f8d8075b5 100644 --- a/engine/ffi/engine.h +++ b/engine/ffi/engine.h @@ -58,6 +58,8 @@ TOAST_C_API void toast_set_load_mode(int mode) NOEXCEPT; */ TOAST_C_API void toast_mount_pack(const char* scheme, const char* pak_path) NOEXCEPT; +TOAST_C_API int toast_open_asset_database(const char* db_path) NOEXCEPT; + /** * @brief Creates the World and loads the init_scene from project settings */ diff --git a/engine/src/toast/assets/asset_manager.cpp b/engine/src/toast/assets/asset_manager.cpp index 99f552f85..d65f4ce7d 100644 --- a/engine/src/toast/assets/asset_manager.cpp +++ b/engine/src/toast/assets/asset_manager.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -290,6 +291,9 @@ auto AssetManager::saveBytes(std::string_view uri, const std::vector& d auto AssetManager::loadBytes(std::string_view uri) -> std::optional> { std::lock_guard lock(mutex); + if (const auto sep = uri.find("://"); sep != std::string_view::npos && mounts.contains(std::string(uri.substr(0, sep)))) { + return readVirtualPath(uri); + } auto real_path = resolveVirtualPath(uri); if (!real_path) { TOAST_ERROR("AssetManager", "Cannot load bytes: could not resolve path {}", uri); @@ -320,69 +324,160 @@ auto AssetManager::tryLoadBytes(std::string_view uri) -> std::optional; + +struct StmtFinalizer { + void operator()(sqlite3_stmt* stmt) const { sqlite3_finalize(stmt); } +}; + +using SqliteStmt = std::unique_ptr; + +auto columnText(sqlite3_stmt* stmt, int col) -> std::string { + const auto* text = sqlite3_column_text(stmt, col); + return text ? std::string(reinterpret_cast(text)) : std::string(); +} + +auto openReadOnly(const std::filesystem::path& file) -> SqliteDb { + // we open the WAL database so the editor can write at the same time + // it will be read only tho but that shouldn't be a problem + const auto u8 = file.generic_u8string(); + const std::string utf8(reinterpret_cast(u8.data()), u8.size()); + sqlite3* raw = nullptr; + const int rc = sqlite3_open_v2(utf8.c_str(), &raw, SQLITE_OPEN_READONLY, nullptr); + SqliteDb db(raw); + if (rc != SQLITE_OK) { + TOAST_ERROR( + "AssetManager", "Could not open asset database {}: {}", file.string(), raw ? sqlite3_errmsg(raw) : "out of memory" + ); + return nullptr; + } + sqlite3_busy_timeout(db.get(), 2000); + return db; +} + +auto prepare(sqlite3* db, const char* sql) -> SqliteStmt { + sqlite3_stmt* raw = nullptr; + if (sqlite3_prepare_v2(db, sql, -1, &raw, nullptr) != SQLITE_OK) { + TOAST_ERROR("AssetManager", "SQL error: {}", sqlite3_errmsg(db)); + return nullptr; + } + return SqliteStmt(raw); +} + +auto hasTable(sqlite3* db, const char* name) -> bool { + auto stmt = prepare(db, "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1"); + if (!stmt) { + return false; + } + sqlite3_bind_text(stmt.get(), 1, name, -1, SQLITE_STATIC); + return sqlite3_step(stmt.get()) == SQLITE_ROW; +} + +} + void AssetManager::reloadManifest() { ZoneScoped; clearUnusedAssets(); std::lock_guard lock(mutex); manifest.clear(); + uri_index.clear(); - // Build the list of per-database manifest paths. - // Editor mode: manifests live under cache:// - // Game mode: manifests are packed inside each db:// pack - std::vector manifest_uris; - if (toast::ProjectSettings* ps = toast::ProjectSettings::get()) { - for (const auto& db : toast::ProjectSettings::databases()) { - if (load_mode == SaveMode::game) { - manifest_uris.push_back(db + "://" + db + ".json"); - } else { - manifest_uris.push_back("cache://" + db + ".json"); - } - } - } else { - manifest_uris.emplace_back("cache://database.json"); + const auto file = activeDatabasePath(); + if (file.empty()) { + TOAST_WARN("AssetManager", "No asset database configured, manifest is empty"); + return; } - if (load_mode == SaveMode::game) { - manifest_uris.emplace_back("core://core.json"); - } else { - manifest_uris.emplace_back("cache://core.json"); + std::error_code ec; + if (!std::filesystem::exists(file, ec)) { + TOAST_WARN("AssetManager", "Asset database not found: {}", file.string()); + return; + } + + auto db = openReadOnly(file); + if (!db) { + return; + } + auto stmt = prepare(db.get(), "SELECT uid, type, db, path FROM assets"); + if (!stmt) { + return; } - // Load and merge every manifest file - auto load_manifest = [&](const std::string& uri) { - auto raw_json = readVirtualPath(uri); - if (!raw_json) { - TOAST_WARN("AssetManager", "Manifest not found: {}", uri); - return; + int rc = SQLITE_OK; + while ((rc = sqlite3_step(stmt.get())) == SQLITE_ROW) { + const uint64_t uid = toast::UID::fromString(columnText(stmt.get(), 0)); + std::string uri = columnText(stmt.get(), 2) + "://" + columnText(stmt.get(), 3); + uri_index[uri] = uid; + manifest[uid] = {std::move(uri), columnText(stmt.get(), 1)}; + } + if (rc != SQLITE_DONE) { + TOAST_ERROR("AssetManager", "Reading asset database failed: {}", sqlite3_errmsg(db.get())); + } + + TOAST_INFO("AssetManager", "Manifest reloaded from {}: {} assets tracked", file.string(), manifest.size()); +} + +auto AssetManager::activeDatabasePath() -> std::filesystem::path { + if (!db_path.empty()) { + return db_path; + } + if (load_mode == SaveMode::editor) { + if (auto cached = resolveVirtualPath("cache://assets.db")) { + return *cached; } + } + return {}; +} - try { - auto json = nlohmann::json::parse(raw_json->begin(), raw_json->end()); - - // an entry is a "uid": "virtual path" pair - // Every top-level object is a type collection keyed by uid -> virtual path; `version` and - // `generated_at` are scalars and skip themselves. This used to be a hardcoded list of type names, - // which meant a new asset type loaded fine, imported fine, wrote a correct manifest entry - and was - // then invisible here, surfacing as "not found in manifest" with nothing pointing at the cause - for (const auto& [type, collection] : json.items()) { - if (!collection.is_object()) { - continue; - } - for (const auto& [key, value] : collection.items()) { - if (!value.is_string()) { - continue; - } - manifest[toast::UID::fromString(key)] = {value.get(), type}; - } - } - } catch (const std::exception& e) { TOAST_ERROR("AssetManager", "Failed to parse manifest {}: {}", uri, e.what()); } - }; +auto AssetManager::openDatabase(const std::filesystem::path& db_file) -> bool { + std::error_code ec; + if (!std::filesystem::exists(db_file, ec)) { + TOAST_ERROR("AssetManager", "Asset database not found: {}", db_file.string()); + return false; + } - for (const auto& uri : manifest_uris) { - load_manifest(uri); + auto db = openReadOnly(db_file); + if (!db) { + return false; + } + if (!hasTable(db.get(), "assets")) { + TOAST_ERROR("AssetManager", "{} is not an asset database (no 'assets' table)", db_file.string()); + return false; } - TOAST_INFO("AssetManager", "Manifest reloaded: {} assets tracked", manifest.size()); + // A release game.db lists the paks that hold its assets + if (!hasTable(db.get(), "paks")) { + db_path = db_file; + return true; + } + auto stmt = prepare(db.get(), "SELECT db, file FROM paks"); + if (!stmt) { + return false; + } + bool ok = true; + while (sqlite3_step(stmt.get()) == SQLITE_ROW) { + const auto scheme = columnText(stmt.get(), 0); + const auto file = columnText(stmt.get(), 1); + const auto pak = db_file.parent_path() / std::filesystem::path(std::u8string(file.begin(), file.end())); + if (!std::filesystem::exists(pak, ec)) { + TOAST_ERROR("AssetManager", "Pak '{}' for '{}://' is missing: {}", file, scheme, pak.string()); + ok = false; + continue; + } + mountPack(scheme, pak); + ok = ok && mounts.contains(scheme); + } + // Only use the database once we know every pack exists + if (ok) { + db_path = db_file; + } + return ok; } void AssetManager::clearUnusedAssets() { @@ -512,12 +607,11 @@ auto AssetManager::saveFile(const std::filesystem::path& path, const std::vector } auto AssetManager::resolveURI(std::string_view uri) -> std::optional { - for (const auto& [uid_val, info] : instance->manifest) { - if (info.path == uri) { - return toast::UID(uid_val); - } + const auto it = instance->uri_index.find(std::string(uri)); + if (it == instance->uri_index.end()) { + return std::nullopt; } - return std::nullopt; + return toast::UID(it->second); } auto AssetManager::getURI(toast::UID uid) -> std::string { diff --git a/engine/src/toast/assets/asset_manager.hpp b/engine/src/toast/assets/asset_manager.hpp index 8b1f1afde..791b8af0f 100644 --- a/engine/src/toast/assets/asset_manager.hpp +++ b/engine/src/toast/assets/asset_manager.hpp @@ -235,6 +235,8 @@ class AssetManager { */ static void mountPack(std::string_view scheme, const std::filesystem::path& pak_path); + static auto openDatabase(const std::filesystem::path& db_file) -> bool; + private: static inline AssetManager* instance = nullptr; @@ -242,9 +244,14 @@ class AssetManager { static inline std::unordered_map> mounts; + static inline std::filesystem::path db_path; + + auto activeDatabasePath() -> std::filesystem::path; + event::Listener listener; mutable std::mutex mutex; std::unordered_map manifest; + std::unordered_map uri_index; std::unordered_map> cache; std::unordered_map asset_mtimes; std::unordered_set pending_reloads; diff --git a/engine/src/toast/engine.cpp b/engine/src/toast/engine.cpp index f76af7f14..d82e65232 100644 --- a/engine/src/toast/engine.cpp +++ b/engine/src/toast/engine.cpp @@ -1332,6 +1332,13 @@ void toast_set_load_mode(int mode) noexcept { assets::AssetManager::setLoadMode(mode == 1 ? assets::SaveMode::game : assets::SaveMode::editor); } +auto toast_open_asset_database(const char* db_path) noexcept -> int { + if (!db_path) { + return 0; + } + return assets::AssetManager::openDatabase(std::filesystem::path(db_path)) ? 1 : 0; +} + void toast_mount_pack(const char* scheme, const char* pak_path) noexcept { if (!scheme || !pak_path) { return; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d6bcc91ac..e06258f92 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,6 +2,7 @@ include(CTest) if (BUILD_TESTING) find_package(RmlUi CONFIG REQUIRED) + find_package(unofficial-sqlite3 CONFIG REQUIRED) file(GLOB_RECURSE TEST_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") list(REMOVE_ITEM TEST_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/test_runner.cpp") @@ -12,10 +13,11 @@ if (BUILD_TESTING) # HACK DARIO SHOULD FUCKING HAVE SOME INCLUDES ON THE SOURCE FILES RATHER THAN THE HEADERS target_include_directories(toast_tests PRIVATE ${CMAKE_SOURCE_DIR}/engine) target_include_directories(toast_tests PRIVATE ${CMAKE_SOURCE_DIR}/engine/external/inc) + target_include_directories(toast_tests PRIVATE ${CMAKE_SOURCE_DIR}/tools/asset_packer/pack/include) target_precompile_headers(toast_tests PRIVATE ${CMAKE_SOURCE_DIR}/engine/pch.h) # renderer/04-import-reflection calls the Slang API directly and the engine find_package does not carry here find_package(slang CONFIG REQUIRED) - target_link_libraries(toast_tests PRIVATE toast_engine RmlUi::RmlUi slang::slang) + target_link_libraries(toast_tests PRIVATE toast_engine RmlUi::RmlUi slang::slang unofficial::sqlite3::sqlite3) target_compile_definitions(toast_tests PRIVATE UNIT_TESTING TRACY_NO_INVARIANT_CHECK=1) target_compile_definitions(toast_tests PRIVATE TOAST_CORE_ASSETS_DIR="${CMAKE_SOURCE_DIR}/engine/assets") target_compile_definitions(toast_tests PRIVATE TOAST_TEST_CONTENT_DIR="${CMAKE_SOURCE_DIR}/tools/test_content") diff --git a/tests/asset_db_fixture.hpp b/tests/asset_db_fixture.hpp new file mode 100644 index 000000000..2da77d1ac --- /dev/null +++ b/tests/asset_db_fixture.hpp @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace test_support { + +struct AssetRow { + std::string uid; + std::string type; + std::string db; + std::string path; +}; + +struct PakRow { + std::string db; + std::string file; +}; + +inline void writeAssetDb(const std::filesystem::path& file, const std::vector& assets, const std::vector& paks = {}) { + std::filesystem::create_directories(file.parent_path()); + std::filesystem::remove(file); + + sqlite3* db = nullptr; + const auto utf8 = file.generic_string(); + if (sqlite3_open(utf8.c_str(), &db) != SQLITE_OK) { + throw std::runtime_error("cannot create " + utf8); + } + + auto exec = [&](const std::string& sql) { + char* err = nullptr; + if (sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &err) != SQLITE_OK) { + const std::string message = err ? err : "unknown"; + sqlite3_free(err); + sqlite3_close(db); + throw std::runtime_error("sqlite: " + message); + } + }; + auto quote = [](const std::string& value) { + std::string out = "'"; + for (const char c : value) { + out += c == '\'' ? "''" : std::string(1, c); + } + return out + "'"; + }; + + exec("CREATE TABLE assets (uid TEXT PRIMARY KEY, type TEXT NOT NULL, db TEXT NOT NULL, path TEXT NOT NULL)"); + for (const auto& row : assets) { + exec("INSERT INTO assets VALUES (" + quote(row.uid) + "," + quote(row.type) + "," + quote(row.db) + "," + quote(row.path) + ")"); + } + if (!paks.empty()) { + exec("CREATE TABLE paks (db TEXT PRIMARY KEY, file TEXT NOT NULL)"); + for (const auto& pak : paks) { + exec("INSERT INTO paks VALUES (" + quote(pak.db) + "," + quote(pak.file) + ")"); + } + } + sqlite3_close(db); +} + +} diff --git a/tests/prefab_instancing/05-load_node.cpp b/tests/prefab_instancing/05-load_node.cpp index 008af8ab2..d94347d26 100644 --- a/tests/prefab_instancing/05-load_node.cpp +++ b/tests/prefab_instancing/05-load_node.cpp @@ -1,3 +1,4 @@ +#include "asset_db_fixture.hpp" #include "prefab_test_helpers.hpp" #include "test_registry.hpp" #include "toast/world/world.hpp" @@ -55,11 +56,9 @@ TOAST_TEST_NAMED("prefab_instancing", "prefab_instancing/05-load_node", test_pre ); // The manifest maps each asset UID to its file. - writeFile( - cache_dir / "database.json", - R"({"node":{)" - R"("Scene000000":"assets://scene.node",)" - R"("Child000000":"assets://child_prefab.node"}})" + test_support::writeAssetDb( + cache_dir / "assets.db", + {{"Scene000000", "node", "assets", "scene.node"}, {"Child000000", "node", "assets", "child_prefab.node"}} ); WorldTestAccess::initAssetManager(assets_dir.string(), cache_dir.string()); diff --git a/tests/scripting/19_prefab_scripts_build_without_waiting.cpp b/tests/scripting/19_prefab_scripts_build_without_waiting.cpp index dfaf7ced8..4bbcd5de2 100644 --- a/tests/scripting/19_prefab_scripts_build_without_waiting.cpp +++ b/tests/scripting/19_prefab_scripts_build_without_waiting.cpp @@ -1,3 +1,4 @@ +#include "asset_db_fixture.hpp" #include "prefab_instancing/prefab_test_helpers.hpp" #include "scripting_test_helpers.hpp" #include "test_registry.hpp" @@ -63,8 +64,8 @@ auto sceneWithScriptedLeaves() -> assets::Handle { ); } writeFile(assets_dir / "scene.node", scene); - writeFile( - cache_dir / "database.json", R"({"node":{"SlowScene00":"assets://scene.node"},"script":{"SlowScrpt00":"assets://slow.lua"}})" + test_support::writeAssetDb( + cache_dir / "assets.db", {{"SlowScene00", "node", "assets", "scene.node"}, {"SlowScrpt00", "script", "assets", "slow.lua"}} ); WorldTestAccess::initAssetManager(assets_dir.string(), cache_dir.string()); diff --git a/tests/scripting/90_bench_parallel_tick.cpp b/tests/scripting/90_bench_parallel_tick.cpp index b9f0c6bc0..042e0ab27 100644 --- a/tests/scripting/90_bench_parallel_tick.cpp +++ b/tests/scripting/90_bench_parallel_tick.cpp @@ -1,3 +1,4 @@ +#include "asset_db_fixture.hpp" #include "scripting_test_helpers.hpp" #include "test_registry.hpp" #include "toast/scripting/lua_state.hpp" @@ -292,9 +293,8 @@ void instantiateCase(int leaf_count, int chunk_iterations) { ); } writeFile(tmp / "assets" / "scene.node", scene); - writeFile( - tmp / "cache" / "database.json", - std::format(R"({{"node":{{"{0}":"assets://scene.node"}},"script":{{"{1}":"assets://bench.lua"}}}})", scene_uid, script_uid) + test_support::writeAssetDb( + tmp / "cache" / "assets.db", {{scene_uid, "node", "assets", "scene.node"}, {script_uid, "script", "assets", "bench.lua"}} ); toast::_detail::WorldTestAccess::initAssetManager((tmp / "assets").string(), (tmp / "cache").string()); diff --git a/tools/player/program.cs b/tools/player/program.cs index 2a1a98604..00496fdf7 100644 --- a/tools/player/program.cs +++ b/tools/player/program.cs @@ -21,9 +21,11 @@ public static void Main() { ); engine.SetLoadMode(gameMode: true); - foreach (var pakFile in Directory.EnumerateFiles(basePath, "*.pak")) { - var scheme = Path.GetFileNameWithoutExtension(pakFile); - engine.MountPack(scheme, pakFile); + var databasePath = Path.Combine(basePath, "game.db"); + if (!engine.OpenAssetDatabase(databasePath)) { + Console.Error.WriteLine($"Could not load the game's asset database: {databasePath} (or a pak it lists) is missing"); + engine.Dispose(); + return; } engine.Init(); diff --git a/tools/player/toast_engine.cs b/tools/player/toast_engine.cs index ddf631f7c..38422118a 100644 --- a/tools/player/toast_engine.cs +++ b/tools/player/toast_engine.cs @@ -54,6 +54,10 @@ public void MountPack(string scheme, string pakPath) { toast_mount_pack(scheme, pakPath); } + public bool OpenAssetDatabase(string databasePath) { + return toast_open_asset_database(databasePath) != 0; + } + public void StartGame() { toast_start_game(); } @@ -94,6 +98,9 @@ private static extern void toast_set_working_directory( [DllImport("__ENGINE_LIB__", CallingConvention = CallingConvention.Cdecl)] private static extern void toast_mount_pack(string scheme, string pakPath); + [DllImport("__ENGINE_LIB__", CallingConvention = CallingConvention.Cdecl)] + private static extern int toast_open_asset_database(string databasePath); + [DllImport("__ENGINE_LIB__", CallingConvention = CallingConvention.Cdecl)] private static extern void toast_start_game(); } diff --git a/vcpkg.json b/vcpkg.json index b12fbecda..12644b553 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -96,6 +96,9 @@ "name": "lz4", "version>=": "1.10.0" }, + { + "name": "sqlite3" + }, { "name": "indicators", "version>=": "2.3"