fix: retry attachment uploads and heal content missing from storage

src/CAS.ts

@@ -10,6 +10,7 @@ import {
 	s3ApiErrorFromResponse,
 	s3ApiErrorFromUnknown,
 	s3NetworkFailureFromUnknown,
+	s3TransferFailureFromUnknown,
 } from "./S3Error";
 
 // In-attempt retry schedule for transient transfer failures: one retry per
@@ -120,7 +121,9 @@ export class ContentAddressedStore extends HasLogging {
 				return await request();
 			} catch (error) {
 				const classified =
-					s3NetworkFailureFromUnknown(error, operation) ?? error;
+					s3NetworkFailureFromUnknown(error, operation) ??
+					s3TransferFailureFromUnknown(error, operation) ??
+					error;
 				const delayCapMs = this.transferRetryDelaysMs[attempt];
 				if (!isRetryableS3Error(classified) || delayCapMs === undefined) {
 					throw classified;
@@ -143,36 +146,37 @@ export class ContentAddressedStore extends HasLogging {
 		if (!(content && hash)) {
 			throw new Error("invalid caf");
 		}
-		const token = await this.tokenStore.getFileToken(
-			S3RN.encode(syncFile.s3rn),
-			hash,
-			syncFile.mimetype,
-			content.byteLength,
-		);
-		const response = await customFetch(token.baseUrl + "/upload-url", {
-			method: "POST",
-			headers: { Authorization: `Bearer ${token.token}` },
-			relayNetworkDomain: "relay",
+		return this.withTransientRetry("upload attachment", async () => {
+			const token = await this.tokenStore.getFileToken(
+				S3RN.encode(syncFile.s3rn),
+				hash,
+				syncFile.mimetype,
+				content.byteLength,
+			);
+			const response = await customFetch(token.baseUrl + "/upload-url", {
+				method: "POST",
+				headers: { Authorization: `Bearer ${token.token}` },
+				relayNetworkDomain: "relay",
+			});
+			if (response.status !== 200) {
+				throw await this.s3ResponseError(response, "upload attachment url");
+			}
+			const responseJson = await response.json();
+			const presignedUrl = responseJson.uploadUrl;
+			const uploadResponse = await this.s3Request(
+				() =>
+					customFetch(presignedUrl, {
+						method: "PUT",
+						headers: { "Content-Type": syncFile.mimetype },
+						body: content,
+						relayNetworkDomain: "external",
+					}),
+				"upload attachment",
+			);
+			if (!uploadResponse.ok) {
+				throw await this.s3ResponseError(uploadResponse, "upload attachment");
+			}
 		});
-		if (response.status !== 200) {
-			throw await this.s3ResponseError(response, "upload attachment url");
-		}
-		const responseJson = await response.json();
-		const presignedUrl = responseJson.uploadUrl;
-		const uploadResponse = await this.s3Request(
-			() =>
-				customFetch(presignedUrl, {
-					method: "PUT",
-					headers: { "Content-Type": syncFile.mimetype },
-					body: content,
-					relayNetworkDomain: "external",
-				}),
-			"upload attachment",
-		);
-		if (!uploadResponse.ok) {
-			throw await this.s3ResponseError(uploadResponse, "upload attachment");
-		}
-		return;
 	}
 
 	private async s3Request(

src/S3Error.ts

@@ -85,6 +85,31 @@ export function s3NetworkFailureFromUnknown(
 const NETWORK_FAILURE_PATTERN =
 	/net::ERR_|Failed to fetch|fetch failed|\bLoad failed\b|NetworkError|network error|socket hang up|ECONNRESET|ECONNREFUSED|ECONNABORTED|ETIMEDOUT|ENOTFOUND|EPIPE|EAI_AGAIN/i;
 
+/**
+ * Classify a failure thrown mid-transfer when no HTTP response was produced.
+ * Unlike the pattern-matched s3NetworkFailureFromUnknown, any non-abort
+ * failure is treated as transport-level and retryable: a transfer that
+ * produced no response cannot have been refused by policy, and parking it
+ * strands the file (the background queue only re-drives retryable failures).
+ * Aborts stay null so a cancelled transfer is not re-driven by the retry
+ * loop that was just cancelled.
+ */
+export function s3TransferFailureFromUnknown(
+	error: unknown,
+	operation?: string,
+): S3ApiError | null {
+	if (error instanceof S3ApiError) return null;
+	if (error instanceof DOMException && error.name === "AbortError") {
+		return null;
+	}
+	const text = errorText(error) ?? "";
+	if (/abort/i.test(text)) return null;
+	return new S3ApiError(
+		{ code: "NetworkingError", operation, message: text || undefined },
+		error,
+	);
+}
+
 function jsonErrorMessage(body: string): string | undefined {
 	try {
 		const parsed = JSON.parse(body) as { error?: unknown };
@@ -159,6 +184,9 @@ function userMessageForS3Error(details: S3ErrorDetails): string {
 	if (details.status === 429) {
 		return "Attachment storage is busy. Relay will retry the upload.";
 	}
+	if (details.status === 404) {
+		return "Attachment content not found in storage.";
+	}
 	if (details.status !== undefined && details.status >= 500) {
 		return "Attachment storage is temporarily unavailable. Relay will retry the upload.";
 	}

src/SharedFolder.ts

@@ -241,6 +241,7 @@ export class SharedFolder extends HasProvider {
 	private syncRequestedDuringSync: boolean = false;
 	private authoritative: boolean;
 	private pendingUpload: LocalStorage<string>;
+	private casVerified: LocalStorage<string>;
 	private unsubscribes: Unsubscriber[] = [];
 	private storageQuota?: number;
 	/**
@@ -386,6 +387,9 @@ export class SharedFolder extends HasProvider {
 		this.pendingUpload = new LocalStorage<string>(
 			`${appId}-system3-relay/folders/${this.guid}/pendingUploads`,
 		);
+		this.casVerified = new LocalStorage<string>(
+			`${appId}-system3-relay/folders/${this.guid}/casVerified`,
+		);
 		this.pendingUpload.forEach((guid, vpath) => {
 			if (!this.existsSync(vpath)) {
 				this.warn(
@@ -3287,6 +3291,27 @@ export class SharedFolder extends HasProvider {
 		this.warn("dropped stale pending-upload holds", stale);
 	}
 
+	/**
+	 * Enqueue a sync for every attachment version not yet in the casVerified
+	 * ledger. The tree sync noops files whose local state matches committed
+	 * metadata, so their sync() — where remote-content verification lives —
+	 * otherwise never runs for at-rest files. The ledger bounds this to one
+	 * verification per (file, hash) per machine.
+	 */
+	private sweepUnverifiedCasContent(): void {
+		for (const file of this.files.values()) {
+			if (!isSyncFile(file) || file.destroyed) continue;
+
+			const meta = this.syncStore.getMeta(file.path);
+			if (!meta || !isFileMetas(meta) || !meta.hash) continue;
+			if (this.isCasVerified(file.guid, meta.hash)) continue;
+
+			this.backgroundSync.enqueueSync(file).catch((error) => {
+				this.warn(`cas verification sync failed for ${file.path}`, error);
+			});
+		}
+	}
+
 	syncFileTree(): Promise<void> {
 		// If a sync is already running, mark that we want another sync after
 		if (this.syncFileTreePromise) {
@@ -3375,6 +3400,7 @@ export class SharedFolder extends HasProvider {
 					this.log("syncFileTree diff:\n" + diffLog.join("\n"));
 				}
 				this.sweepStalePendingUploads();
+				this.sweepUnverifiedCasContent();
 			} finally {
 				// Reset the promise after completion (success or failure)
 				this.syncFileTreePromise = null;
@@ -3566,6 +3592,20 @@ export class SharedFolder extends HasProvider {
 		this.pendingUpload.clear();
 	}
 
+	/**
+	 * Durable per-machine ledger of attachment versions whose content is
+	 * known to exist in storage, so remote-existence verification runs at
+	 * most once per (file, hash) instead of on every sync pass. Content is
+	 * immutable under its hash, so an entry never needs re-checking.
+	 */
+	public isCasVerified(guid: string, hash: string): boolean {
+		return this.casVerified.get(guid) === hash;
+	}
+
+	public markCasVerified(guid: string, hash: string): void {
+		this.casVerified.set(guid, hash);
+	}
+
 	async markUploaded(
 		file: IFile,
 		outcome: SyncCompletionOutcome = "completed",

src/SyncFile.ts

@@ -13,7 +13,6 @@ import { Observable, type Unsubscriber } from "./observable/Observable";
 import { generateHash } from "./hashing";
 import type { HasMimeType, IFile } from "./IFile";
 import { getMimeType } from "./mimetypes";
-import { flags } from "./flagManager";
 import { errorFromUnknown, formatUserFacingError } from "./UserFacingError";
 
 export function isSyncFile(file: IFile | undefined): file is SyncFile {
@@ -557,6 +556,9 @@ export class SyncFile
 			try {
 				await this.sharedFolder.cas.writeFile(this);
 				await this.sharedFolder.markUploaded(this);
+				if (hash) {
+					this.sharedFolder.markCasVerified(this.guid, hash);
+				}
 				this.uploadError = undefined;
 				this.notifyListeners();
 				this.debug("push complete", {
@@ -648,7 +650,10 @@ export class SyncFile
 			return;
 		}
 
-		if (this.isCleanLastServerEdit(this.meta as FileMetas, this.stat)) {
+		if (
+			this.isCleanLastServerEdit(this.meta as FileMetas, this.stat) &&
+			this.sharedFolder.isCasVerified(this.guid, this.meta.hash)
+		) {
 			this.debug("sync decision", {
 				path: this.path,
 				guid: this.guid,
@@ -672,17 +677,7 @@ export class SyncFile
 				statMtime: this.stat.mtime,
 				metaSynctime: this.meta.synctime,
 			});
-			if (flags().enableVerifyUploads) {
-				// Not remote
-				try {
-					if (!(await this.verifyUpload())) {
-						this.warn("file in metadata, but not on the server!");
-						await this.push();
-					}
-				} catch (err) {
-					// pass
-				}
-			}
+			await this.reconcileRemoteContent(hash);
 			if (hash === this.meta.hash) {
 				this.clearCurrentUserEdit();
 				this.debug("sync decision", {
@@ -795,6 +790,53 @@ export class SyncFile
 		return this.sharedFolder.cas.verify(this);
 	}
 
+	/**
+	 * Confirm the version named by metadata actually exists in storage,
+	 * at most once per (file, hash). A hole is healable only from a machine
+	 * whose local content IS the missing version; a machine holding a
+	 * different version must not clobber the claim - the missing bytes may
+	 * still exist on the authoring machine.
+	 */
+	private async reconcileRemoteContent(localHash: string | null) {
+		if (!this.meta) {
+			return;
+		}
+		const metaHash = this.meta.hash;
+		if (this.sharedFolder.isCasVerified(this.guid, metaHash)) {
+			return;
+		}
+		let exists: boolean;
+		try {
+			exists = await this.verifyUpload();
+		} catch (err) {
+			this.debug(
+				"remote content verification failed; retrying on a later sync",
+				err,
+			);
+			return;
+		}
+		if (exists) {
+			this.sharedFolder.markCasVerified(this.guid, metaHash);
+			return;
+		}
+		if (localHash && localHash === metaHash) {
+			this.warn(
+				`[${this.path}] content missing from storage; re-uploading local copy`,
+			);
+			await this.push(true);
+			return;
+		}
+		this.warn(
+			`[${this.path}] content missing from storage and the local copy is a different version; only the authoring machine can heal this`,
+		);
+		const message =
+			"Attachment content was never uploaded; awaiting its author's machine.";
+		if (this.uploadError !== message) {
+			this.uploadError = message;
+			this.notifyListeners();
+		}
+	}
+
 	public async pull() {
 		this.log("pull");
 		this._refreshMeta();
@@ -843,6 +885,7 @@ export class SyncFile
 				.catch((error) => {
 					this.warn("Failed to save pulled hash:", error);
 				});
+			this.sharedFolder.markCasVerified(this.guid, this.meta.hash);
 			if (this.uploadError) {
 				this.uploadError = undefined;
 				this.notifyListeners();

src/flags.ts

@@ -4,7 +4,6 @@ export interface FeatureFlags {
 	enableDiffLinkStatus: boolean;
 	enableDeltaLogging: boolean;
 	enableNetworkLogging: boolean;
-	enableVerifyUploads: boolean;
 	enableDiscordLogin: boolean;
 	enableDeviceManagement: boolean;
 	enableHSMRecording: boolean;
@@ -104,13 +103,6 @@ export const FeatureFlagSchema: {
 		description:
 			"Log HTTP status, method, URL, and response bodies from Relay network requests.",
 	},
-	enableVerifyUploads: {
-		default: false,
-		category: "debugging",
-		title: "Verify uploaded attachments",
-		description:
-			"After attachment sync, confirm the remote object exists and re-upload if it is missing.",
-	},
 	enableHSMRecording: {
 		default: false,
 		category: "debugging",