Preserve transparency in inline Kitty images

CHANGELOG.md

@@ -25,6 +25,8 @@ and a release's notes are its section. The procedure is in [`docs/RELEASING.md`]
 
 ### Changed
 
+- Inline game images preserve full and partial transparency through Kitty transmission, including the desktop window, instead of losing their alpha channel.
+
 - The Windows installer and the Linux AppImage are now built with Velopack, which is what lets them update. Windows
   installs per user under `%LocalAppData%\SharpMUTerm`, adds the `sharpmuterm` folder to PATH and installs the Symbols
   Nerd Font Mono font, with no questions asked. If you installed an earlier version with the old installer, uninstall

src/SharpMUTerm.Tui/ImageDecoder.cs

@@ -50,23 +50,44 @@ public static PixelBuffer Decode(byte[] bytes)
         }
     }
 
-    private static PixelBuffer DecodeWithImageSharp(byte[] bytes)
+    /// <summary>Decode without the framework's RGB-only PixelBuffer discarding alpha.</summary>
+    public static RgbaPicture DecodeRgba(byte[] bytes)
     {
-        var options = new DecoderOptions
+        ArgumentNullException.ThrowIfNull(bytes);
+        try
+        {
+            using var input = new MemoryStream(bytes, writable: false);
+            var info = StbImageSharp.ImageInfo.FromStream(input);
+            if (info is null)
+            {
+                throw new UnsupportedImageFormatException();
+            }
+            if ((long)info.Value.Width * info.Value.Height > MaxPixels)
+            {
+                throw new InvalidImageContentException("Image exceeds the decode ceiling.");
+            }
+            var image = StbImageSharp.ImageResult.FromMemory(bytes, StbImageSharp.ColorComponents.RedGreenBlueAlpha);
+            return new RgbaPicture(image.Width, image.Height, image.Data);
+        }
+        catch (UnsupportedImageFormatException)
         {
-            Configuration = WebPOnly,
-            MaxFrames = 1,
-            SkipMetadata = true,
-        };
+            return DecodeWebPRgba(bytes);
+        }
+    }
+
+    private static RgbaPicture DecodeWebPRgba(byte[] bytes)
+    {
+        var options = new DecoderOptions { Configuration = WebPOnly, MaxFrames = 1, SkipMetadata = true };
         var info = Image.Identify(options, bytes);
         if ((long)info.Width * info.Height > MaxPixels)
         {
-            throw new InvalidImageContentException($"{info.Width}x{info.Height} is past the decode ceiling.");
+            throw new InvalidImageContentException("Image exceeds the decode ceiling.");
         }
-
         using var image = Image.Load<Rgba32>(options, bytes);
         var rgba = new byte[image.Width * image.Height * 4];
         image.CopyPixelDataTo(rgba);
-        return PixelBuffer.FromRgba32Bytes(rgba, image.Width, image.Height);
+        return new RgbaPicture(image.Width, image.Height, rgba);
     }
+
+    private static PixelBuffer DecodeWithImageSharp(byte[] bytes) => DecodeWebPRgba(bytes).Pixels;
 }

src/SharpMUTerm.Tui/InlineImageHost.cs

@@ -235,7 +235,7 @@ presentation is not (InlineImagePresentation.Kitty or InlineImagePresentation.Ha
     /// fetched and decoded now. Null when it cannot be had, and a failure is not remembered, so the next
     /// mention tries again.
     /// </summary>
-    private Task<PixelBuffer?> SourceFor(string url)
+    private Task<RgbaPicture?> SourceFor(string url)
     {
         lock (_gate)
         {
@@ -252,7 +252,7 @@ presentation is not (InlineImagePresentation.Kitty or InlineImagePresentation.Ha
             _sources[url] = _sourceOrder.AddFirst(source);
             source.Picture = Task.Run(async () =>
             {
-                PixelBuffer? picture = null;
+                RgbaPicture? picture = null;
                 try
                 {
                     picture = await FetchAndDecodeAsync(url).ConfigureAwait(false);
@@ -267,7 +267,7 @@ presentation is not (InlineImagePresentation.Kitty or InlineImagePresentation.Ha
         }
     }
 
-    private async Task<PixelBuffer?> FetchAndDecodeAsync(string url)
+    private async Task<RgbaPicture?> FetchAndDecodeAsync(string url)
     {
         byte[]? bytes;
         await _fetches.WaitAsync(_cts.Token).ConfigureAwait(false);
@@ -291,7 +291,7 @@ presentation is not (InlineImagePresentation.Kitty or InlineImagePresentation.Ha
 
         try
         {
-            return ImageDecoder.Decode(bytes);
+            return ImageDecoder.DecodeRgba(bytes);
         }
         catch (Exception ex) when (ex is not OutOfMemoryException)
         {
@@ -300,7 +300,7 @@ presentation is not (InlineImagePresentation.Kitty or InlineImagePresentation.Ha
     }
 
     /// <summary>Accounts for a finished fetch: forgets a failure, and trims the cache to its budget.</summary>
-    private void Settle(Source source, PixelBuffer? picture)
+    private void Settle(Source source, RgbaPicture? picture)
     {
         lock (_gate)
         {
@@ -360,10 +360,10 @@ internal int SourcesHeld
         int availableColumns,
         int maxRows = InlineImageLayout.MaxRows)
     {
-        PixelBuffer source;
+        RgbaPicture source;
         try
         {
-            source = ImageDecoder.Decode(bytes);
+            source = ImageDecoder.DecodeRgba(bytes);
         }
         catch (Exception ex) when (ex is not OutOfMemoryException)
         {
@@ -375,7 +375,7 @@ internal int SourcesHeld
 
     /// <summary>Sizes a decoded picture to its cell box, keeping the pixels either presentation needs.</summary>
     internal static DecodedPicture? Fit(
-        PixelBuffer source,
+        RgbaPicture source,
         InlineImageRequest request,
         int availableColumns,
         int maxRows = InlineImageLayout.MaxRows)
@@ -390,8 +390,8 @@ internal int SourcesHeld
         var height = Math.Min(source.Height, box.Rows * InlineImageLayout.CellPixelHeight);
         try
         {
-            var scaled = PixelScale.Stretch(source, width, height);
-            return new DecodedPicture(box, scaled);
+            var scaled = source.Resize(width, height);
+            return new DecodedPicture(box, scaled.Pixels, scaled.Png);
         }
         catch (ArgumentOutOfRangeException)
         {
@@ -444,7 +444,7 @@ private static IReadOnlyList<string> HalfBlockRows(WebImageLayout.CellBox box, P
 
         try
         {
-            protocol.TransmitRawRgb((uint)id, Rgb(picture.Pixels), picture.Pixels.Width, picture.Pixels.Height, columns, rows);
+            protocol.TransmitImage((uint)id, picture.Png, columns, rows);
         }
         catch (NotSupportedException)
         {
@@ -454,24 +454,6 @@ private static IReadOnlyList<string> HalfBlockRows(WebImageLayout.CellBox box, P
         return InlineImageRows.Kitty(id, columns, rows);
     }
 
-    private static byte[] Rgb(PixelBuffer pixels)
-    {
-        var data = new byte[pixels.Width * pixels.Height * 3];
-        var i = 0;
-        for (var y = 0; y < pixels.Height; y++)
-        {
-            for (var x = 0; x < pixels.Width; x++)
-            {
-                var p = pixels.GetPixel(x, y);
-                data[i++] = p.R;
-                data[i++] = p.G;
-                data[i++] = p.B;
-            }
-        }
-
-        return data;
-    }
-
     private IReadOnlyList<string>? Cached(string key)
     {
         lock (_gate)
@@ -507,7 +489,7 @@ private sealed class Source(string url)
     {
         public string Url { get; } = url;
 
-        public Task<PixelBuffer?> Picture { get; set; } = Task.FromResult<PixelBuffer?>(null);
+        public Task<RgbaPicture?> Picture { get; set; } = Task.FromResult<RgbaPicture?>(null);
 
         /// <summary>Its size in the cache's budget; zero while the fetch is under way.</summary>
         public long Bytes { get; set; }
@@ -522,4 +504,4 @@ public void Dispose()
 }
 
 /// <summary>A decoded picture and the cell box it will fill.</summary>
-internal sealed record DecodedPicture(WebImageLayout.CellBox Box, PixelBuffer Pixels);
+internal sealed record DecodedPicture(WebImageLayout.CellBox Box, PixelBuffer Pixels, byte[] Png);

src/SharpMUTerm.Tui/RgbaPicture.cs

@@ -0,0 +1,32 @@
+using SharpConsoleUI.Imaging;
+using SharpMUTerm.Core.Avatars;
+using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.PixelFormats;
+using SixLabors.ImageSharp.Processing;
+
+namespace SharpMUTerm.Tui;
+
+/// <summary>Straight RGBA pixels retained until Kitty PNG transmission; RGB is only a fallback.</summary>
+internal sealed record RgbaPicture(int Width, int Height, byte[] Data)
+{
+    public PixelBuffer Pixels => PixelBuffer.FromRgba32Bytes(Data, Width, Height);
+
+    public byte[] Png => PngWriter.Rgba(Width, Height, Data);
+
+    public RgbaPicture Resize(int width, int height)
+    {
+        if (width == Width && height == Height)
+        {
+            return this;
+        }
+        using var image = Image.LoadPixelData<Rgba32>(Data, Width, Height);
+        // Premultiplied interpolation prevents hidden RGB under transparent pixels bleeding into edges.
+        image.Mutate(x => x.Resize(new ResizeOptions
+        {
+            Size = new Size(width, height), Mode = ResizeMode.Stretch, PremultiplyAlpha = true,
+        }));
+        var data = new byte[width * height * 4];
+        image.CopyPixelDataTo(data);
+        return new RgbaPicture(width, height, data);
+    }
+}

tests/SharpMUTerm.Avalonia.Tests/PictureRenderingTests.cs

@@ -14,6 +14,55 @@ public class PictureRenderingTests
 {
     private static readonly SKColor Back = new(0x20, 0x20, 0x20);
 
+    [Test]
+    [Arguments(0)]
+    [Arguments(128)]
+    [Arguments(255)]
+    public async Task InlineKittyPngAlphaCompositesOverTheCellBackground(int alpha)
+    {
+        const int side = 16;
+        var rgba = new byte[side * side * 4];
+        for (var i = 0; i < rgba.Length; i += 4)
+        {
+            rgba[i] = 255;
+            rgba[i + 3] = (byte)alpha;
+        }
+        var png = SharpMUTerm.Core.Avatars.PngWriter.Rgba(side, side, rgba);
+        using var driver = new AvaloniaConsoleDriver(4, 3);
+        using var host = new SharpMUTerm.Tui.InlineImageHost((_, _) => Task.FromResult<byte[]?>(png), run => run());
+        host.Load([new SharpMUTerm.Core.Text.InlineImageRequest("https://example.test/alpha.png", "alpha")],
+            InlineImagePresentation.Kitty, 4, () => driver, (_, _) => { });
+        await host.Pending;
+        await Assert.That(driver.Pictures.TryGet(0xC00000, out var picture)).IsTrue();
+        using var decoded = SKBitmap.FromImage(picture.Image);
+        await Assert.That(decoded.GetPixel(0, 0).Alpha).IsEqualTo((byte)alpha);
+
+        var buffer = new CharacterBuffer(4, 3, Color.Default);
+        buffer.SetCell(1, 1, ToCell(Placeholder(0xC00000, 0, 0)));
+        driver.WriteBufferRegion(0, 1, buffer, 0, 1, 4, Color.Default);
+        driver.Flush();
+        using var fonts = new CellFonts(16);
+        using var surface = SKSurface.Create(new SKImageInfo(4 * fonts.CellWidth, 3 * fonts.CellHeight));
+        new CellRenderer(fonts, SKColors.White, Back).Render(surface.Canvas, driver.Frame, driver.Pictures, caretShown: false);
+        using var pixels = SKBitmap.FromImage(surface.Snapshot());
+        var actual = pixels.GetPixel(fonts.CellWidth + fonts.CellWidth / 2, fonts.CellHeight + fonts.CellHeight / 2);
+        var expectedR = (255 * alpha + Back.Red * (255 - alpha) + 127) / 255;
+        var expectedGB = (Back.Green * (255 - alpha) + 127) / 255;
+        await Assert.That(Math.Abs(actual.Red - expectedR) <= 1).IsTrue();
+        await Assert.That(Math.Abs(actual.Green - expectedGB) <= 1).IsTrue();
+        await Assert.That(Math.Abs(actual.Blue - expectedGB) <= 1).IsTrue();
+    }
+
+    [Test]
+    public async Task ResizingTransparentPixelsDoesNotBleedTheirHiddenColour()
+    {
+        var picture = new SharpMUTerm.Tui.RgbaPicture(2, 1, [255, 0, 0, 255, 0, 0, 255, 0]);
+        var resized = picture.Resize(1, 1);
+        await Assert.That(resized.Data[0]).IsEqualTo((byte)255);
+        await Assert.That(resized.Data[2]).IsEqualTo((byte)0);
+        await Assert.That(Math.Abs(resized.Data[3] - 128) <= 1).IsTrue();
+    }
+
     [Test]
     public async Task APlaceholderNamesItsPictureRowAndColumn()
     {

tests/SharpMUTerm.Tui.Tests/ImageDecoderTests.cs

@@ -7,6 +7,22 @@ namespace SharpMUTerm.Tui.Tests;
 /// </summary>
 public class ImageDecoderTests
 {
+    [Test]
+    public async Task KittyDecodePreservesPngAndWebPAlpha()
+    {
+        var png = ImageDecoder.DecodeRgba(TestImages.RgbaPng(16, 12));
+        var webp = ImageDecoder.DecodeRgba(TestImages.WebP(16, 12));
+        await Assert.That(png.Data.SequenceEqual(webp.Data)).IsTrue();
+        await Assert.That(Enumerable.Range(0, 16 * 12).Any(i => png.Data[i * 4 + 3] != 255)).IsTrue();
+    }
+
+    [Test]
+    public async Task KittyDecodeStillRejectsTiffAndInvalidBytes()
+    {
+        await Assert.That(() => ImageDecoder.DecodeRgba(TestImages.Tiff(8, 8))).ThrowsException();
+        await Assert.That(() => ImageDecoder.DecodeRgba([1, 2, 3, 4])).ThrowsException();
+    }
+
     [Test]
     public async Task TheFrameworkStillRefusesWebP()
     {