Copy displayed Android frame to PC clipboard

apps/web-client/src/main.ts

@@ -5,6 +5,7 @@ import { TransferController } from "./transfer-controller.js";
 import { RunningAppController } from "./running-app-controller.js";
 import { AuthController } from "./auth-controller.js";
 import { NetworkAccessController } from "./network-controller.js";
+import { screenCopyShortcut, writeCanvasPngToClipboard } from "./screen-copy.js";
 
 function required<T extends Element>(selector: string): T {
   const value = document.querySelector<T>(selector);
@@ -20,19 +21,32 @@ function bindAndroidCopyWriteThrough(): void {
   const details = required<HTMLElement>("#details");
   let generation = 0;
 
-  const finishCopy = async (): Promise<void> => {
+  const copyDisplayImage = async (request: number, reason: string): Promise<void> => {
+    try {
+      await writeCanvasPngToClipboard(canvas);
+      if (request !== generation) return;
+      status.textContent = "Image copied";
+      details.textContent = reason;
+    } catch (error) {
+      if (request !== generation) return;
+      status.textContent = "Image copy unavailable";
+      details.textContent = error instanceof Error ? error.message : String(error);
+    }
+  };
+
+  const finishCopy = async (fallbackToImage = false): Promise<void> => {
     const request = ++generation;
     const initialText = clipboardText.value;
     const initialStatus = status.textContent?.trim() ?? "";
-    const deadline = performance.now() + 1200;
+    const deadline = performance.now() + (fallbackToImage ? 700 : 1200);
     let responseObserved = false;
 
     while (performance.now() < deadline) {
       if (request !== generation) return;
       const currentStatus = status.textContent?.trim() ?? "";
       const textChanged = clipboardText.value !== initialText;
       const statusChanged = currentStatus !== initialStatus;
-      if ((textChanged || statusChanged) && currentStatus === "Clipboard copied") return;
+      if (currentStatus === "Clipboard copied" && (textChanged || (!fallbackToImage && statusChanged))) return;
       if (textChanged || (statusChanged && currentStatus === "Clipboard received")) {
         responseObserved = true;
         break;
@@ -42,6 +56,10 @@ function bindAndroidCopyWriteThrough(): void {
 
     if (request !== generation) return;
     const text = clipboardText.value;
+    if (fallbackToImage && (!responseObserved || text === initialText || !text)) {
+      await copyDisplayImage(request, "Ctrl+C found no new Android text selection, so the displayed Android frame was copied to the PC clipboard as PNG.");
+      return;
+    }
     if (!text) {
       status.textContent = "Copy not confirmed";
       details.textContent = "Android did not return clipboard text.";
@@ -76,8 +94,15 @@ function bindAndroidCopyWriteThrough(): void {
 
   button.addEventListener("click", () => void finishCopy());
   canvas.addEventListener("keydown", (event) => {
-    if (event.altKey || (!event.ctrlKey && !event.metaKey) || event.key.toLowerCase() !== "c") return;
-    void finishCopy();
+    if (screenCopyShortcut(event) !== "image") return;
+    event.preventDefault();
+    event.stopImmediatePropagation();
+    const request = ++generation;
+    void copyDisplayImage(request, "Ctrl+Shift+C copied the displayed Android frame to the PC clipboard as PNG.");
+  }, { capture: true });
+  canvas.addEventListener("keydown", (event) => {
+    if (screenCopyShortcut(event) !== "smart") return;
+    void finishCopy(true);
   });
 }
 

apps/web-client/src/screen-copy.ts

@@ -0,0 +1,28 @@
+export type ScreenCopyShortcut = "smart" | "image" | null;
+
+export function screenCopyShortcut(
+  event: Pick<KeyboardEvent, "key" | "ctrlKey" | "metaKey" | "altKey" | "shiftKey">,
+): ScreenCopyShortcut {
+  if (event.altKey || (!event.ctrlKey && !event.metaKey) || event.key.toLowerCase() !== "c") return null;
+  return event.shiftKey ? "image" : "smart";
+}
+
+export async function canvasPngBlob(canvas: HTMLCanvasElement): Promise<Blob> {
+  if (canvas.width <= 0 || canvas.height <= 0) throw new Error("Android display frame is not available yet");
+  return await new Promise<Blob>((resolve, reject) => {
+    canvas.toBlob((blob) => {
+      if (blob) resolve(blob);
+      else reject(new Error("Could not encode the Android display frame as PNG"));
+    }, "image/png");
+  });
+}
+
+export async function writeCanvasPngToClipboard(canvas: HTMLCanvasElement): Promise<void> {
+  if (!navigator.clipboard?.write || typeof ClipboardItem === "undefined") {
+    throw new Error("This browser does not support writing images to the PC clipboard");
+  }
+  const blob = await canvasPngBlob(canvas);
+  await navigator.clipboard.write([
+    new ClipboardItem({ "image/png": blob }),
+  ]);
+}

apps/web-client/tests/screen-copy.test.mjs

@@ -0,0 +1,11 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { screenCopyShortcut } from "../dist/assets/screen-copy.js";
+
+test("screen copy shortcuts distinguish smart and forced image copy", () => {
+  assert.equal(screenCopyShortcut({ key: "c", ctrlKey: true, metaKey: false, altKey: false, shiftKey: false }), "smart");
+  assert.equal(screenCopyShortcut({ key: "C", ctrlKey: true, metaKey: false, altKey: false, shiftKey: true }), "image");
+  assert.equal(screenCopyShortcut({ key: "c", ctrlKey: false, metaKey: true, altKey: false, shiftKey: true }), "image");
+  assert.equal(screenCopyShortcut({ key: "c", ctrlKey: true, metaKey: false, altKey: true, shiftKey: false }), null);
+  assert.equal(screenCopyShortcut({ key: "v", ctrlKey: true, metaKey: false, altKey: false, shiftKey: false }), null);
+});