diff --git a/.changeset/you_can_now_chose_to_generate_bundled_link_previews_for_your_message_msc4095_they_get_downloaded_and_attached_to_your_message_and_dont_rely_on_your_server_supporting_server_side_embeds.md b/.changeset/you_can_now_chose_to_generate_bundled_link_previews_for_your_message_msc4095_they_get_downloaded_and_attached_to_your_message_and_dont_rely_on_your_server_supporting_server_side_embeds.md new file mode 100644 index 0000000000..1da6b560e8 --- /dev/null +++ b/.changeset/you_can_now_chose_to_generate_bundled_link_previews_for_your_message_msc4095_they_get_downloaded_and_attached_to_your_message_and_dont_rely_on_your_server_supporting_server_side_embeds.md @@ -0,0 +1,5 @@ +--- +default: minor +--- + +# You can now chose to generate bundled link previews for your message (MSC4095). They get downloaded and attached to your message and dont rely on your server supporting server-side embeds. diff --git a/package.json b/package.json index 44e8aada8a..eb62558f5c 100644 --- a/package.json +++ b/package.json @@ -164,7 +164,8 @@ "vite-plugin-svgr": "4.5.0", "vite-plugin-top-level-await": "^1.6.0", "vitest": "^4.1.9", - "wrangler": "^4.106.0" + "wrangler": "^4.106.0", + "htmlparser2": "^12.0.0" }, "devEngines": { "runtime": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eea31c8f31..2df9b12132 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -341,6 +341,9 @@ importers: cloudflared: specifier: ^0.7.1 version: 0.7.1 + htmlparser2: + specifier: ^12.0.0 + version: 12.0.0 jsdom: specifier: ^29.1.1 version: 29.1.1(@noble/hashes@2.2.0) diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json index 4717beb3b4..f1c7eb6d1a 100644 --- a/src-tauri/capabilities/default.json +++ b/src-tauri/capabilities/default.json @@ -30,7 +30,7 @@ "clipboard-manager:allow-write-image", { "identifier": "http:default", - "allow": [{ "url": "https://*" }] + "allow": [{ "url": "http://*:*" }, { "url": "https://*" }] }, { "identifier": "opener:allow-open-url", diff --git a/src/app/components/RenderMessageContent.tsx b/src/app/components/RenderMessageContent.tsx index e8e79190ff..f5c6c339cc 100644 --- a/src/app/components/RenderMessageContent.tsx +++ b/src/app/components/RenderMessageContent.tsx @@ -217,7 +217,7 @@ function RenderMessageContentInternal({ ), [urlPreview] ); - const messageUrlsPreview = urlPreview || themeChatSableWidgets ? renderUrlsPreview : undefined; + const messageUrlsPreview = urlPreview ? renderUrlsPreview : undefined; const messageBundlePreview = bundledPreview ? renderBundledPreviews : undefined; const renderCaption = () => { diff --git a/src/app/components/icons/phosphor.tsx b/src/app/components/icons/phosphor.tsx index e88273dd15..71407fb9d4 100644 --- a/src/app/components/icons/phosphor.tsx +++ b/src/app/components/icons/phosphor.tsx @@ -126,6 +126,7 @@ import { VideoCameraSlashIcon, WarningIcon, XIcon, + CircleNotchIcon, } from '@phosphor-icons/react'; import type { ComponentType, ReactNode } from 'react'; import { useLayoutEffect } from 'react'; @@ -436,4 +437,5 @@ export { VideoCameraSlashIcon as VideoCameraSlash, WarningIcon as Warning, XIcon as X, + CircleNotchIcon as CircleNotch, }; diff --git a/src/app/components/overlay-stack/OverlayStack.tsx b/src/app/components/overlay-stack/OverlayStack.tsx index 5ba40c97e5..7e1b4f5a74 100644 --- a/src/app/components/overlay-stack/OverlayStack.tsx +++ b/src/app/components/overlay-stack/OverlayStack.tsx @@ -31,7 +31,7 @@ export function OverlayStackProvider({ children }: { children: ReactNode }) { setClaims((prev) => prev.some((claim) => claim.id === id) ? prev - : [...prev, { id, seq }].sort((a, b) => a.seq - b.seq) + : [...prev, { id, seq }].toSorted((a, b) => a.seq - b.seq) ), release: (id) => setClaims((prev) => prev.filter((claim) => claim.id !== id)), }), diff --git a/src/app/components/upload-card/EmbedCardRenderer.tsx b/src/app/components/upload-card/EmbedCardRenderer.tsx new file mode 100644 index 0000000000..e6a552e1fc --- /dev/null +++ b/src/app/components/upload-card/EmbedCardRenderer.tsx @@ -0,0 +1,146 @@ +import { useMatrixClient } from '$hooks/useMatrixClient.ts'; +import { roomEmbedAtomFamily } from '$state/room/roomInputDrafts.ts'; +import { useSetting } from '$state/hooks/settings.ts'; +import { settingsAtom } from '$state/settings.ts'; +import { UploadCard } from '$components/upload-card/UploadCard.tsx'; +import { Box, color, Text, toRem } from 'folds'; +import { Check, CircleNotch, File as FileIcon, sizedIcon, X } from '$components/icons/phosphor.tsx'; +import { + EmbedErrorReason, + EmbedPreviewType, + EmbedStatus, + type FixedPreviewUrlResponse, + useBindEmbedAtom, +} from '$state/bundle.ts'; +import { Image as MediaImage } from '$components/media'; +import { useObjectURL } from '$hooks/useObjectURL.ts'; +import { isImageMimeType } from '$utils/mimeTypes.ts'; + +type EmbedCardRendererProps = { + url: string; + successCallback: (result: FixedPreviewUrlResponse) => void; + encrypt: boolean; +}; + +function prettyError(error: EmbedErrorReason) { + switch (error) { + //TODO localize + case EmbedErrorReason.UploadFailed: + return 'Failed to upload'; + case EmbedErrorReason.EncryptFailed: + return 'Failed to encrypt'; + case EmbedErrorReason.RequestFailed: + return 'Failed to fetch url'; + case EmbedErrorReason.NoOgData: + return 'Url doesnt provide previews'; + } +} + +interface MediaProps { + data: Blob; + type: string; +} + +function Media({ data, type }: Readonly) { + const fileUrl = useObjectURL(data); + if (isImageMimeType(type)) { + return ( + + ); + } + + return ; +} + +export function EmbedCardRenderer({ + url, + successCallback, + encrypt, +}: Readonly) { + const mx = useMatrixClient(); + const [bundleUseHomeserver] = useSetting(settingsAtom, 'useHomeserverForBundles'); + const embedAtom = roomEmbedAtomFamily(url); + const { embed, startEmbed } = useBindEmbedAtom(mx, embedAtom, encrypt, bundleUseHomeserver); + + if (embed.status === EmbedStatus.Idle) { + startEmbed(successCallback).then(() => {}); + } + + return ( + } + bottom={ + <> + {embed.status == EmbedStatus.Idle && <>{'Preview here'}} + {(embed.status == EmbedStatus.Loading || embed.status == EmbedStatus.Success) && ( + <> + + {embed.preview?.type == EmbedPreviewType.TitleDescriptionMedia && ( + + {embed.preview?.media && ( + + )} + {!embed.preview?.media && ( + + )} + + )} + + {embed.preview?.title} + {embed.preview?.description || 'No Description'} + + + + )} + {embed.status == EmbedStatus.Error && ( + {prettyError(embed.reason)} + )} + + } + > + + {url} + + {embed.status == EmbedStatus.Success && + sizedIcon(Check, '100', { style: { color: color.Success.Main } })} + + {embed.status == EmbedStatus.Error && + sizedIcon(X, '100', { style: { color: color.Critical.Main } })} + + {embed.status == EmbedStatus.Loading && ( + + {embed.progress} + + )} + {(embed.status == EmbedStatus.Loading || embed.status == EmbedStatus.Idle) && + sizedIcon(CircleNotch, '100', { + style: { + color: color.Primary.Main, + animation: 'spin 1s infinite linear', + }, + })} + + ); +} diff --git a/src/app/components/upload-card/index.ts b/src/app/components/upload-card/index.ts index e014ab8a8e..d93108fae8 100644 --- a/src/app/components/upload-card/index.ts +++ b/src/app/components/upload-card/index.ts @@ -1,3 +1,4 @@ export * from './UploadCard'; export * from './UploadCardRenderer'; export * from './CompactUploadCardRenderer'; +export * from './EmbedCardRenderer'; diff --git a/src/app/features/room/RoomInput.test.tsx b/src/app/features/room/RoomInput.test.tsx index f61c9674c9..761668ed6b 100644 --- a/src/app/features/room/RoomInput.test.tsx +++ b/src/app/features/room/RoomInput.test.tsx @@ -88,6 +88,7 @@ vi.mock('$state/room/roomInputDrafts', async () => { const { atom } = await import('jotai'); const msgDraftAtom = atom([]); const replyDraftAtom = atom(undefined); + const embedItemsAtom = atom([]); const uploadItemsAtom = atom([], (get, set, action) => { const update = action as any; if (Array.isArray(update)) { @@ -115,6 +116,7 @@ vi.mock('$state/room/roomInputDrafts', async () => { roomIdToMsgDraftAtomFamily: () => msgDraftAtom, roomIdToReplyDraftAtomFamily: () => replyDraftAtom, roomIdToUploadItemsAtomFamily: () => uploadItemsAtom, + roomIdToEmbeddItemsAtomFamily: () => embedItemsAtom, roomUploadAtomFamily: Object.assign(() => atom(undefined), { remove: vi.fn() }), roomIdToScheduledTimeAtomFamily: () => scheduledTimeAtom, roomIdToEditingScheduledDelayIdAtomFamily: () => editingScheduledDelayIdAtom, diff --git a/src/app/features/room/RoomInput.tsx b/src/app/features/room/RoomInput.tsx index 7c75880b68..4fcf73e3d9 100644 --- a/src/app/features/room/RoomInput.tsx +++ b/src/app/features/room/RoomInput.tsx @@ -12,28 +12,26 @@ import { Suspense, useCallback, useEffect, + useMemo, useRef, useState, - useMemo, } from 'react'; -import { useAtom, useAtomValue, useSetAtom } from 'jotai'; +import { useAtom, useAtomValue, useSetAtom, useStore } from 'jotai'; import { isKeyHotkey } from 'is-hotkey'; import type { IContent, + IEventRelation, MatrixEvent, Room, - IEventRelation, RoomMessageEventContent, StickerEventContent, TimelineEvents, } from '$types/matrix-sdk'; -import { MatrixError } from '$types/matrix-sdk'; -import { EventType, RelationType } from '$types/matrix-sdk'; +import { EventType, MatrixError, RelationType } from '$types/matrix-sdk'; import { M_POLL_START } from 'matrix-js-sdk'; import { ReactEditor } from 'slate-react'; import { Editor, Point, Range, Transforms } from 'slate'; -import type { RectCords } from 'folds'; import { Box, color, @@ -44,15 +42,29 @@ import { MenuItem, OverlayBackdrop, OverlayCenter, + type RectCords, Scroll, Spinner, + Switch, Text, toRem, } from 'folds'; import { Overlay, PopOut } from '$components/overlay-stack'; import { useMatrixClient } from '$hooks/useMatrixClient'; -import type { AutocompleteQuery } from '$components/editor'; +import { + ANYWHERE_AUTOCOMPLETE_PREFIXES, + type AutocompleteQuery, + BEGINNING_AUTOCOMPLETE_PREFIXES, + EmoticonAutocomplete, + focusEditor, + isEmptyEditor, + MarkdownFormattingToolbarBottom, + MarkdownFormattingToolbarToggle, + moveCursor, + replaceWithElement, + resetEditorHistory, +} from '$components/editor'; import { AutocompletePrefix, createEmoticonElement, @@ -66,16 +78,6 @@ import { toPlainText, trimCustomHtml, UserMentionAutocomplete, - EmoticonAutocomplete, - moveCursor, - resetEditorHistory, - isEmptyEditor, - ANYWHERE_AUTOCOMPLETE_PREFIXES, - BEGINNING_AUTOCOMPLETE_PREFIXES, - MarkdownFormattingToolbarBottom, - MarkdownFormattingToolbarToggle, - focusEditor, - replaceWithElement, } from '$components/editor'; import { stripMarkdownEscapesForHiddenPreviews } from './message/hiddenLinkPreviews'; import { plainToEditorInput } from '$components/editor/input'; @@ -94,18 +96,25 @@ import { useTypingStatusUpdater } from '$hooks/useTypingStatusUpdater'; import { useFilePicker } from '$hooks/useFilePicker'; import { useFilePasteHandler } from '$hooks/useFilePasteHandler'; import { useFileDropZone } from '$hooks/useFileDrop'; -import type { TUploadItem, TUploadMetadata, IReplyDraft } from '$state/room/roomInputDrafts'; import { + roomEmbedAtomFamily, + roomIdToEmbeddItemsAtomFamily, roomIdToMsgDraftAtomFamily, roomIdToReplyDraftAtomFamily, roomIdToUploadItemsAtomFamily, roomUploadAtomFamily, } from '$state/room/roomInputDrafts'; -import { UploadCardRenderer } from '$components/upload-card'; +import type { + IReplyDraft, + TEmbeddItem, + TUploadItem, + TUploadMetadata, +} from '$state/room/roomInputDrafts'; +import { EmbedCardRenderer, UploadCard, UploadCardRenderer } from '$components/upload-card'; import type { UploadBoardImperativeHandlers } from '$components/upload-board'; import { UploadBoard, UploadBoardContent, UploadBoardHeader } from '$components/upload-board'; import type { Upload, UploadSuccess } from '$state/upload'; -import { UploadStatus, createUploadFamilyObserverAtom } from '$state/upload'; +import { createUploadFamilyObserverAtom, UploadStatus } from '$state/upload'; import { loadImageElementFromMediaUrl } from '$utils/dom'; import { isImageMimeType, safeUploadFile } from '$utils/mimeTypes'; import { useSetting } from '$state/hooks/settings'; @@ -130,8 +139,8 @@ import * as Sentry from '@sentry/react'; import { delayedEventsSupportedAtom, getScheduledMessageStateKey, - roomIdToScheduledTimeAtomFamily, roomIdToEditingScheduledDelayIdAtomFamily, + roomIdToScheduledTimeAtomFamily, serverMaxDelayMsAtom, } from '$state/scheduledMessages'; import { @@ -162,6 +171,7 @@ import { composerIcon, dropzoneIcon, File as FileIcon, + getPhosphorIconSize, Gif, ListBullets, MapPinPlusIcon, @@ -169,8 +179,8 @@ import { Microphone, PaperPlaneTilt, PencilSimple, - getPhosphorIconSize, PlusCircle, + sizedIcon, Smiley, Sticker, Stop, @@ -179,6 +189,7 @@ import { import { getSupportedAudioExtension } from '$plugins/voice-recorder-kit/supportedCodec'; import { ErrorCode } from '../../cs-errorcode'; import { PKitCommandMessageHandler } from '$plugins/pluralkit-handler/PKitCommandMessageHandler'; + import type { IGenericMSC4459, MSC4459ImagePackReference } from '$types/matrix/common'; import { getImagePackReferencesForMxc, @@ -194,14 +205,15 @@ import { MobileSwipeDownModal } from '$components/MobileSwipeDownModal'; import { SchedulePickerDialog } from './schedule-send'; import * as css from './schedule-send/SchedulePickerDialog.css'; import { + buildGalleryContent, getAudioMsgContent, getFileMsgContent, + getGalleryItemContent, + getGifMsgContent, getImageMsgContent, getVideoMsgContent, - getGifMsgContent, - buildGalleryContent, - getGalleryItemContent, } from './msgContent'; + import { getSendableKlipyMxcUrl } from '$utils/klipy'; import { CommandAutocomplete } from './CommandAutocomplete'; import type { @@ -216,7 +228,6 @@ import { PersistentPersonaPicker, type PersonaPickerTab } from './persona-picker import { createComposerController, type ComposerController } from './composerController'; import { buildEditReplacement, buildOutgoingMessage } from './composerMessage'; import { pickNativeFile } from './nativeFilePicker'; - const LocationDialog = lazy(() => import('./location-modal').then((module) => ({ default: module.LocationDialog })) ); @@ -387,6 +398,9 @@ export const RoomInput = forwardRef( const replyDraftRef = useRef(replyDraft); replyDraftRef.current = replyDraft; + const [embedLinks, setEmbedLinks] = useAtom(roomIdToEmbeddItemsAtomFamily(roomId)); + const [embedsEnabled, setEmbedsEnabled] = useState(false); + const store = useStore(); const [uploadBoard, setUploadBoard] = useState(true); const [uploadSending, setUploadSending] = useState(false); const [uploadBusy, setUploadBusy] = useState(false); @@ -665,8 +679,56 @@ export const RoomInput = forwardRef( setAutocompleteQuery(query); }, [editor]); + const [generateBundles] = useSetting(settingsAtom, 'generateBundles'); + const [encryptBundledMedia] = useSetting(settingsAtom, 'encryptBundledMedia'); + + const checkForEmbedables = useCallback( + (text: string) => { + if (!generateBundles) { + return; + } + //from https://regex101.com/r/3fYy3x/1 + const URL_REGEX = RegExp( + /http[s]?:\/\/.(?:www\.)?[-a-zA-Z0-9@%._+~#=]{2,256}\.[a-z]{2,10}\b(?:[-a-zA-Z0-9@:%_+.~#?&/=]*)/gm + ); + const urls = Array.from( + text + .matchAll(URL_REGEX) + //dont include the url if its in angled braces like cant do this in regex because braces are context free (yay computer science) + .filter( + (e) => + !(e.index > 0 && text[e.index - 1] == '<' && text[e.index + e[0].length] == '>') + ) + .map((e) => e[0]) + ); + if (urls.length > 0) { + setUploadBoard(true); + + const embedItems = urls.map( + (url): TEmbeddItem => ({ + url: url, + }) + ); + setEmbedLinks({ + type: 'CLEAR', + }); + setEmbedLinks({ + type: 'PUT', + item: embedItems, + }); + } else { + setEmbedLinks({ + type: 'CLEAR', + }); + setEmbedsEnabled(false); + } + }, + [setEmbedLinks, generateBundles] + ); + const handleEditorChange = useCallback(() => { setHasText(!isEmptyEditor(editor)); + checkForEmbedables(Editor.string(editor, [])); detectAutocomplete(); if (!room.hasEncryptionStateEvent()) return; @@ -675,7 +737,7 @@ export const RoomInput = forwardRef( lastEncryptionPreparationAt.current = now; mx.getCrypto()?.prepareToEncrypt(room); - }, [editor, detectAutocomplete, mx, room]); + }, [editor, detectAutocomplete, mx, room, checkForEmbedables]); const hasContent = hasText || selectedFiles.length > 0; const isComposing = useComposingCheck(); @@ -1486,6 +1548,7 @@ export const RoomInput = forwardRef( } if (selectedFiles.length > 0) { const uploads = uploadBoardHandlers.current?.getSendableUploads() ?? []; + const sendUpload = handleSendUploadRef.current; setUploadSending(true); try { @@ -1501,7 +1564,6 @@ export const RoomInput = forwardRef( if (isLive()) setUploadSending(false); } } - const outgoing = await buildOutgoingMessage(submission.children, { mx, room, @@ -1519,9 +1581,20 @@ export const RoomInput = forwardRef( latchedPersona, isPKCommand: (text) => PKitCommandMessageHandler.isPKCommand(text), imagePacksUsed: imagePacksUsedRef.current, + embedLinks, + embedsEnabled, + generateBundles, + store, }); if (outgoing.kind === 'empty') return; + if (outgoing.kind === 'failed') { + if (isLive()) { + setSendError(outgoing.reason); + } + restoreSubmission(submission); + return; + } if (outgoing.kind === 'quickReact') { handleQuickReact(outgoing.key); return; @@ -1710,6 +1783,10 @@ export const RoomInput = forwardRef( pmpLatchingEnable, pmpNoFallback, latchedPersona, + embedsEnabled, + store, + generateBundles, + embedLinks, ] ); @@ -1836,6 +1913,13 @@ export const RoomInput = forwardRef( [editor, sendTypingStatus, hideActivity, detectAutocomplete] ); + const handleEmbedsToggle = (state: boolean) => { + setEmbedsEnabled(state); + for (const param of roomEmbedAtomFamily.getParams()) { + roomEmbedAtomFamily.remove(param); + } + }; + const handleEmoticonSelect = (key: string, shortcode: string) => { const emoticonEl = createEmoticonElement(key, shortcode); if (autocompleteQuery) { @@ -2038,7 +2122,7 @@ export const RoomInput = forwardRef( forceMultilineLayout={showAudioRecorder} top={ <> - {selectedFiles.length > 0 && ( + {(selectedFiles.length > 0 || (embedLinks.length > 0 && generateBundles)) && ( ( } /> ))} + + {embedLinks.length > 0 && generateBundles && ( + <> + {selectedFiles.length > 0 && ( + + )} + } + after={ + //fake close button to keep box size +
+ + {sizedIcon(X, '200')} + +
+ } + bottom={ + <> + + + Bundled
Embeds + + + + + + } + > + + {'Links found'} + + + + )} + {embedsEnabled && + Array.from(embedLinks).map((link) => ( + {}} + /> + ))} )} diff --git a/src/app/features/room/composerMessage.test.ts b/src/app/features/room/composerMessage.test.ts index aee137f90c..bb283a3a8c 100644 --- a/src/app/features/room/composerMessage.test.ts +++ b/src/app/features/room/composerMessage.test.ts @@ -5,6 +5,10 @@ import type { MatrixClient, Room } from '$types/matrix-sdk'; import { SerializableMap } from '$types/wrapper/SerializableMap'; import type { MSC4459ImagePackReference } from '$types/matrix/common'; import type { PerMessageProfileMsc4461 } from '$app/persona'; +import { createStore } from 'jotai'; +import { type Embed, EmbedPreviewType, EmbedStatus } from '$state/bundle.ts'; +import { roomEmbedAtomFamily } from '$state/room/roomInputDrafts.ts'; +import type { Store } from 'jotai/vanilla/store'; const { profiles } = vi.hoisted(() => ({ profiles: { @@ -56,6 +60,8 @@ const commandInput = (command: Command, rest = '') => [ }, ]; +const store: Store = createStore(); + const build = ( input: string | ReturnType, overrides: Partial[1]> = {} @@ -77,6 +83,10 @@ const build = ( latchedPersona: undefined, isPKCommand: () => false, imagePacksUsed: new SerializableMap(), + embedLinks: [], + embedsEnabled: false, + generateBundles: false, + store: store, ...overrides, }); @@ -200,6 +210,35 @@ describe('buildOutgoingMessage', () => { expect(previews?.map((preview) => preview.matched_url)).toContain('https://example.com/page'); }); + it('generate bundled embeds', async () => { + let embedatom = roomEmbedAtomFamily('https://example.com/page'); + + let embed: Embed = { + url: 'https://example.com/page', + status: EmbedStatus.Success, + preview: { title: 'Example Title', type: EmbedPreviewType.TitleDescription }, + data: { + 'og:title': 'Example Title', + 'og:type': 'website', + 'og:url': 'https://example.com/page', + }, + }; + + store.set(embedatom, embed); + + const result = await build('see https://example.com/page', { + embedLinks: [{ url: 'https://example.com/page' }], + embedsEnabled: true, + generateBundles: true, + }); + if (result.kind !== 'message') throw new Error('expected a message'); + const previews = result.content['com.beeper.linkpreviews'] as + | { matched_url: string; 'og:title': string }[] + | undefined; + expect(previews?.map((preview) => preview.matched_url)).toContain('https://example.com/page'); + expect(previews?.map((preview) => preview['og:title'])).toContain('Example Title'); + }); + it('preserves the original per-message profile when editing', () => { const originalProfile = { id: 'original', displayname: 'Original' }; const edited = buildEditReplacement(plainToEditorInput('updated'), { diff --git a/src/app/features/room/composerMessage.ts b/src/app/features/room/composerMessage.ts index 77d156cd3c..e646766b9e 100644 --- a/src/app/features/room/composerMessage.ts +++ b/src/app/features/room/composerMessage.ts @@ -16,7 +16,11 @@ import { } from '$components/editor'; import { sanitizeText } from '$utils/sanitize'; import { getMentionContent } from '$utils/room/relations'; -import type { IReplyDraft } from '$state/room/roomInputDrafts'; +import { + type IReplyDraft, + roomEmbedAtomFamily, + type TEmbeddItem, +} from '$state/room/roomInputDrafts'; import { ProfileCatalog } from '$app/persona/catalog'; import type { PerMessageProfileMsc4461 } from '$app/persona'; import { convertPerMessageProfileToBeeperFormat } from '$app/persona/projection'; @@ -28,6 +32,8 @@ import { buildReplacementContent } from './buildReplacementContent'; import { Command, SHRUG, TABLEFLIP, UNFLIP } from '$hooks/useCommands'; import type { MSC4459ImagePackReference } from '$types/matrix/common'; import type { SerializableMap } from '$types/wrapper/SerializableMap'; +import { createEmbedFamilyObserverAtom, EmbedStatus } from '$state/bundle.ts'; +import type { Store } from 'jotai/vanilla/store'; export type MessageContent = IContent & Pick; @@ -41,7 +47,8 @@ export type OutgoingMessage = | { kind: 'pkCommand'; plainText: string } | { kind: 'command'; command: Command; plainText: string; customHtml: string } | { kind: 'gifSearch'; query: string } - | { kind: 'empty' }; + | { kind: 'empty' } + | { kind: 'failed'; reason: string }; export interface BuildOutgoingMessageDeps { mx: MatrixClient; @@ -62,6 +69,10 @@ export interface BuildOutgoingMessageDeps { latchedPersona: PerMessageProfileMsc4461 | undefined; isPKCommand: (plainText: string) => boolean; imagePacksUsed: SerializableMap; + embedLinks: TEmbeddItem[]; + embedsEnabled: boolean; + generateBundles: boolean; + store: Store; } const resolveNickname = ( @@ -248,9 +259,32 @@ export async function buildOutgoingMessage( const content: MessageContent = { msgtype: msgType, body: plainText }; content['m.mentions'] = getMentionContent(Array.from(mentionData.users), mentionData.room); content[prefix.MATRIX_UNSTABLE_IMAGE_SOURCE_PACK_PROPERTY_NAME] = imagePacksUsed.toJSON(); - content[prefix.MATRIX_UNSTABLE_EMBEDDED_LINK_PREVIEW_PROPERTY_NAME] = ( - getLinks(serializedChildren) ?? [] - ).map((matched_url) => ({ matched_url })); + content[prefix.MATRIX_UNSTABLE_EMBEDDED_LINK_PREVIEW_PROPERTY_NAME] = []; + //check setting again here, just in case it was disabled while the message got composed + if (deps.embedsEnabled && deps.generateBundles) { + const embedUrls = deps.embedLinks.map((e) => e.url); + const embedObserver = createEmbedFamilyObserverAtom(roomEmbedAtomFamily, embedUrls); + const embeds = deps.store.get(embedObserver); + for (const embed of embeds) { + if (embed.status == EmbedStatus.Loading || embed.status == EmbedStatus.Idle) + return { kind: 'failed', reason: 'bundled embeds not ready' }; + if (embed.status == EmbedStatus.Success) { + embed.data['matched_url'] = embed.url; + content[prefix.MATRIX_UNSTABLE_EMBEDDED_LINK_PREVIEW_PROPERTY_NAME].push(embed.data); + } else { + content[prefix.MATRIX_UNSTABLE_EMBEDDED_LINK_PREVIEW_PROPERTY_NAME].push({ + matched_url: embed.url, + }); + } + } + } else { + const links = getLinks(serializedChildren); + links?.forEach((link) => { + content[prefix.MATRIX_UNSTABLE_EMBEDDED_LINK_PREVIEW_PROPERTY_NAME].push({ + matched_url: link, + }); + }); + } if (replyDraft || !customHtmlEqualsPlainText(customHtml, plainText)) { content.format = 'org.matrix.custom.html'; diff --git a/src/app/features/settings/general/General.tsx b/src/app/features/settings/general/General.tsx index e2e4f7dc47..639d7b89f3 100644 --- a/src/app/features/settings/general/General.tsx +++ b/src/app/features/settings/general/General.tsx @@ -1025,6 +1025,15 @@ function Embeds() { const [urlPreview, setUrlPreview] = useSetting(settingsAtom, 'urlPreview'); const [encUrlPreview, setEncUrlPreview] = useSetting(settingsAtom, 'encUrlPreview'); const [clientUrlPreview, setClientUrlPreview] = useSetting(settingsAtom, 'clientUrlPreview'); + const [generateBundles, setGenerateBundles] = useSetting(settingsAtom, 'generateBundles'); + const [bundleUseHomeserver, setBundleUseHomeserver] = useSetting( + settingsAtom, + 'useHomeserverForBundles' + ); + const [encryptBundledMedia, setEncryptBundledMedia] = useSetting( + settingsAtom, + 'encryptBundledMedia' + ); const [showInteractiveMap, setShowInteractiveMap] = useSetting( settingsAtom, 'showInteractiveMap' @@ -1059,6 +1068,31 @@ function Embeds() { value={bundledPreview} onChange={setBundledPreview} /> + + {generateBundles && ( + <> + + + + )} >; + 'og:image'?: string; + MATRIX_BUNDLED_EMBEDS_ENCRYPTED_PROPERTY_NAME?: Record< + string, + string | Record + >; +} & Omit>; + +export enum EmbedPreviewType { + TitleDescription, + MediaOnly, + TitleDescriptionMedia, +} +export type EmbedPreview = { + title: string; + type: EmbedPreviewType; + description?: string; + media?: Blob; + mediaType?: string; +}; + +export enum EmbedStatus { + Idle = 'idle', + Loading = 'loading', + Success = 'success', + Error = 'error', +} + +export enum LoadingStatus { + Html = 'fetching html', + MediaDown = 'downloading media ', + MediaUp = 'uploading media', + Encrypt = 'encrypting media', +} + +export type EmbedIdle = { + url: string; + status: EmbedStatus.Idle; +}; + +export type EmbedLoading = { + url: string; + status: EmbedStatus.Loading; + progress: LoadingStatus; + preview?: EmbedPreview; + promise: Promise; +}; + +export type EmbedSuccess = { + url: string; + status: EmbedStatus.Success; + preview: EmbedPreview; + data: FixedPreviewUrlResponse; +}; + +export type EmbedError = { + url: string; + status: EmbedStatus.Error; + reason: EmbedErrorReason; +}; + +export type Embed = EmbedIdle | EmbedLoading | EmbedSuccess | EmbedError; + +type EmbedAtomAction = + | { + promise: Promise; + } + | { + progress: LoadingStatus; + preview: EmbedPreview | undefined; + } + | { + data: FixedPreviewUrlResponse; + preview: EmbedPreview; + } + | { + error: EmbedErrorReason; + }; + +export enum EmbedErrorReason { + RequestFailed, + NoOgData, + UploadFailed, + EncryptFailed, +} + +export type fetchEmbedOptions = { + onPromise?: (promise: Promise) => void; + onProgress?: (preview: EmbedPreview, progress: LoadingStatus) => void; + onSuccess: (embed: FixedPreviewUrlResponse, progress: EmbedPreview) => void; + onError: (reason: EmbedErrorReason) => void; +}; + +const hasProperty = (dom: Element, value: string) => { + return value === dom.attribs?.property; +}; + +const parseOg = (html: string) => { + const dom = parseDocument(html); + let head = DomUtils.getElementsByTagName('head', dom); + let metaTags = DomUtils.getElementsByTagName('meta', head); + + let alternatives: Record = { + 'og:title': [['name', 'title']], + 'og:description': [['name', 'description']], + }; + + let requiredTags = ['og:title', 'og:type', 'og:url']; + let optionalStringTags = ['og:image', 'og:image:type', 'og:description', 'og:image:alt']; + let optionalNumberTags = ['og:image:height', 'og:image:width']; + let result = new Map(); + for (const ogTag of requiredTags) { + let tag = DomUtils.findOne((e) => hasProperty(e, ogTag), metaTags); + if (!tag || !tag.attribs?.content) { + let altList = alternatives[ogTag]; + if (!altList) return null; + while (!tag || !tag.attribs?.content) { + let next = altList.pop(); + if (!next) return null; + tag = DomUtils.findOne((e) => e.attribs?.[next[0]] == next[1], metaTags); + } + } + result.set(ogTag, tag.attribs?.content); + } + for (const ogTag of optionalStringTags) { + let tag = DomUtils.findOne((e) => hasProperty(e, ogTag), metaTags); + result.set(ogTag, tag?.attribs?.content); + } + for (const ogTag of optionalNumberTags) { + let tag = DomUtils.findOne((e) => hasProperty(e, ogTag), metaTags); + if (!tag?.attribs?.content) continue; + result.set(ogTag, Number.parseInt(tag?.attribs?.content)); + } + + return result; +}; + +function fetchWrapper(url: string) { + return isTauri() ? taurifetch(url) : fetch(url); +} + +const FetchEmbed = async ( + url: string, + mx: MatrixClient, + encrypted: boolean, + bundleUseHomeserver: boolean, + successCallback: (result: FixedPreviewUrlResponse) => void, + useAuthentication: boolean, + fetchEmbedOptions: fetchEmbedOptions +) => { + if (!fetchEmbedOptions.onProgress) fetchEmbedOptions.onProgress = () => {}; + if (!fetchEmbedOptions.onPromise) fetchEmbedOptions.onPromise = () => {}; + const pushImage = async ( + imgData: Blob, + preview: EmbedPreview, + embedRecord: FixedPreviewUrlResponse + ) => { + preview.media = imgData; + let encInfo: EncryptedAttachmentInfo; + if (encrypted) { + fetchEmbedOptions.onProgress?.(preview, LoadingStatus.Encrypt); + try { + let encryptResult = await encryptFile(imgData); + imgData = encryptResult.file; + encInfo = encryptResult.encInfo; + } catch { + fetchEmbedOptions.onError(EmbedErrorReason.EncryptFailed); + return; + } + } + fetchEmbedOptions.onProgress?.(preview, LoadingStatus.MediaUp); + await uploadContent(mx, new File([imgData], 'mediaPreview'), { + onError: () => { + if (preview.type == EmbedPreviewType.MediaOnly) { + fetchEmbedOptions.onError(EmbedErrorReason.UploadFailed); + return; + } + preview.type = EmbedPreviewType.TitleDescription; + for (const e in embedRecord) { + if (e.startsWith('og:image')) { + embedRecord[e] = undefined; + } + } + fetchEmbedOptions.onSuccess(embedRecord, preview); + successCallback(embedRecord); + }, + onSuccess: (mxc) => { + if (encrypted) { + embedRecord[MATRIX_BUNDLED_EMBEDS_ENCRYPTED_PROPERTY_NAME] = { + ...encInfo, + url: mxc, + }; + embedRecord['matrix:image:size'] = imgData.size; + embedRecord['og:image'] = undefined; + } else { + embedRecord['og:image'] = mxc; + embedRecord['matrix:image:size'] = imgData.size; + } + fetchEmbedOptions.onSuccess(embedRecord, preview); + successCallback(embedRecord); + }, + }); + }; + //TODO filter local ips. How to even do this on web? + if (bundleUseHomeserver) { + var response: IPreviewUrlResponse; + try { + const promise = mx.getUrlPreview(url, Date.now()); + fetchEmbedOptions.onPromise(promise); + response = await promise; + } catch { + fetchEmbedOptions.onError(EmbedErrorReason.RequestFailed); + return; + } + const imageUrl = mxcUrlToHttp( + mx, + response['og:image'] || '', + useAuthentication, + 256, + 256, + 'scale', + false + ); + let preview: EmbedPreview = { + description: response['og:description'], + title: response['og:title'], + type: EmbedPreviewType.TitleDescription, + }; + if (!imageUrl) { + fetchEmbedOptions.onSuccess(response, preview); + successCallback(response); + return; + } + + let fetchPromise = fetch(imageUrl); + fetchEmbedOptions.onProgress(preview, LoadingStatus.MediaDown); + let fetchResponse = await fetchPromise; + let fetchBlob = await fetchResponse.blob().catch(() => null); + if (!fetchBlob) { + fetchEmbedOptions.onSuccess(response, preview); + successCallback(response); + return; + } + preview.media = fetchBlob; + if (fetchBlob.type.length > 0) { + preview.mediaType = fetchBlob.type; + } else if (fetchResponse.headers.get('content-type')?.split(';')[0]) { + preview.mediaType = fetchResponse.headers.get('content-type')?.split(';')[0]; + } + preview.type = EmbedPreviewType.TitleDescriptionMedia; + if (encrypted) { + await pushImage(fetchBlob, preview, response); + } else { + fetchEmbedOptions.onSuccess(response, preview); + successCallback(response); + } + return; + } + let fetchResponse; + try { + let fetchPromise = fetchWrapper(url); + fetchEmbedOptions.onPromise(fetchPromise); + fetchResponse = await fetchPromise; + } catch (e) { + console.error(e); + fetchEmbedOptions.onError(EmbedErrorReason.RequestFailed); + return; + } + if (fetchResponse.ok) { + if (fetchResponse.headers.get('content-type')?.split(';')[0] == 'text/html') { + let prelimEmbed = parseOg(await fetchResponse.text()); + if (!prelimEmbed) { + fetchEmbedOptions.onError(EmbedErrorReason.NoOgData); + return; + } + + let supportedMediaTags = new Set(['og:image']); + const hasSupportedMedia = prelimEmbed.keys().some((item) => supportedMediaTags.has(item)); + + let preview: EmbedPreview = { + description: prelimEmbed.get('og:description'), + title: prelimEmbed.get('og:title'), + type: hasSupportedMedia + ? EmbedPreviewType.TitleDescriptionMedia + : EmbedPreviewType.TitleDescription, + }; + fetchEmbedOptions.onProgress(preview, LoadingStatus.MediaDown); + + let embedRecord: FixedPreviewUrlResponse = ( + Object.fromEntries(prelimEmbed) + ); + if (!hasSupportedMedia) { + fetchEmbedOptions.onSuccess(embedRecord, preview); + successCallback(embedRecord); + } else { + if (embedRecord['og:image']) { + const imgResponse = await fetchWrapper(embedRecord['og:image']); + const imgData = await imgResponse.blob(); + preview.media = imgData; + if (imgData.type.length > 0) { + preview.mediaType = imgData.type; + } else if (imgResponse.headers.get('content-type')?.split(';')[0]) { + preview.mediaType = imgResponse.headers.get('content-type')?.split(';')[0]; + } + await pushImage(imgData, preview, embedRecord); + } else { + //other media types + preview.type = EmbedPreviewType.TitleDescription; + fetchEmbedOptions.onSuccess(embedRecord, preview); + successCallback(embedRecord); + } + } + } else if (isImageMimeType(fetchResponse.headers.get('content-type')?.split(';')[0] || '')) { + //TODO: decide if generating bundled previews for image links (and other media potentially) is something we even want + fetchEmbedOptions.onError(EmbedErrorReason.NoOgData); + } else { + fetchEmbedOptions.onError(EmbedErrorReason.NoOgData); + } + } else { + fetchEmbedOptions.onError(EmbedErrorReason.RequestFailed); + } +}; + +export const createEmbedAtom = (url: string) => { + const baseEmbedAtom = atom({ + url, + status: EmbedStatus.Idle, + }); + return atom( + (get) => get(baseEmbedAtom), + (get, set, update) => { + const embedState = get(baseEmbedAtom); + if ('promise' in update) { + set(baseEmbedAtom, { + status: EmbedStatus.Loading, + url, + progress: LoadingStatus.Html, + promise: update.promise, + }); + return; + } + if ('progress' in update && embedState.status === EmbedStatus.Loading) { + set(baseEmbedAtom, { + ...embedState, + progress: update.progress, + preview: update.preview, + }); + return; + } + if ('data' in update) { + set(baseEmbedAtom, { + status: EmbedStatus.Success, + url, + data: update.data, + preview: update.preview, + }); + return; + } + if ('error' in update) { + set(baseEmbedAtom, { + status: EmbedStatus.Error, + url, + reason: update.error, + }); + } + } + ); +}; +export type TEmbedAtom = ReturnType; + +export const useBindEmbedAtom = ( + mx: MatrixClient, + embedAtom: TEmbedAtom, + encrypt: boolean, + useHomeserver: boolean +) => { + const [embed, setEmbed] = useAtom(embedAtom); + const { url } = embed; + const useAuth = useMediaAuthentication(); + + const startEmbed = useCallback( + (successCallback: (result: FixedPreviewUrlResponse) => void) => + FetchEmbed(url, mx, encrypt, useHomeserver, successCallback, useAuth, { + onSuccess: (data, progress) => setEmbed({ data, preview: progress }), + onError: (error) => setEmbed({ error }), + onProgress: (preview, progress) => setEmbed({ preview, progress }), + onPromise: (promise) => setEmbed({ promise }), + }), + [url, mx, encrypt, useHomeserver, setEmbed, useAuth] + ); + + return { + embed, + startEmbed, + }; +}; + +export const createEmbedAtomFamily = () => atomFamily(createEmbedAtom); +export type TEmbedAtomFamily = ReturnType; + +export const createEmbedFamilyObserverAtom = (uploadFamily: TEmbedAtomFamily, uploads: string[]) => + atom((get) => uploads.map((upload) => get(uploadFamily(upload)))); +export type TEmbedFamilyObserverAtom = ReturnType; diff --git a/src/app/state/list.ts b/src/app/state/list.ts index 2749f70bdb..37d935e277 100644 --- a/src/app/state/list.ts +++ b/src/app/state/list.ts @@ -13,6 +13,9 @@ type ListAction = | { type: 'DELETE'; item: T | T[]; + } + | { + type: 'CLEAR'; }; export const createListAtom = () => { @@ -20,6 +23,10 @@ export const createListAtom = () => { return atom], undefined>( (get) => get(baseListAtom), (get, set, action) => { + if (action.type === 'CLEAR') { + set(baseListAtom, []); + return; + } const items = get(baseListAtom); const newItems = Array.isArray(action.item) ? action.item : [action.item]; if (action.type === 'DELETE') { diff --git a/src/app/state/room/roomInputDrafts.ts b/src/app/state/room/roomInputDrafts.ts index 10525f64a5..946e576102 100644 --- a/src/app/state/room/roomInputDrafts.ts +++ b/src/app/state/room/roomInputDrafts.ts @@ -6,6 +6,7 @@ import type { IEventRelation } from '$types/matrix-sdk'; import type { TUploadContent } from '$utils/matrix'; import { createUploadAtomFamily } from '$state/upload'; import { createListAtom } from '$state/list'; +import { createEmbedAtomFamily } from '$state/bundle.ts'; export type TUploadMetadata = { markedAsSpoiler: boolean; @@ -30,6 +31,15 @@ export const roomIdToUploadItemsAtomFamily = atomFamily export const roomUploadAtomFamily = createUploadAtomFamily(); +export type TEmbeddItem = { + url: string; +}; + +type TEmbeddListAtom = ReturnType>; + +export const roomIdToEmbeddItemsAtomFamily = atomFamily(createListAtom); +export const roomEmbedAtomFamily = createEmbedAtomFamily(); + const createMsgDraftAtom = () => atom([]); type TMsgDraftAtom = ReturnType; export const roomIdToMsgDraftAtomFamily = atomFamily(() => diff --git a/src/app/state/settings.ts b/src/app/state/settings.ts index 546a2a788a..71084ddd24 100644 --- a/src/app/state/settings.ts +++ b/src/app/state/settings.ts @@ -159,6 +159,10 @@ export interface Settings { showInteractiveMap: boolean; showEncInteractiveMap: boolean; + generateBundles: boolean; + useHomeserverForBundles: boolean; + encryptBundledMedia: boolean; + usePushNotifications: boolean; useUnifiedPush: boolean; useInAppNotifications: boolean; @@ -334,6 +338,9 @@ export const defaultSettings: Settings = { enableGifPicker: true, showInteractiveMap: true, showEncInteractiveMap: false, + generateBundles: true, + useHomeserverForBundles: false, + encryptBundledMedia: true, showHiddenEvents: false, showTombstoneEvents: true, hiddenEventEdits: true, diff --git a/src/unstable/prefixes/misc.ts b/src/unstable/prefixes/misc.ts index 09fc536d83..db6d15d412 100644 --- a/src/unstable/prefixes/misc.ts +++ b/src/unstable/prefixes/misc.ts @@ -4,6 +4,9 @@ export const MATRIX_UNSTABLE_PROFILE_BANNER_PROPERTY_NAME = 'chat.commet.profile_banner'; export const MATRIX_COMMET_UNSTABLE_PROFILE_COLOR_SCHEME_PROPERTY_NAME = 'chat.commet.profile_color_scheme'; +///opengraph field for encrypted embeds +export const MATRIX_BUNDLED_EMBEDS_ENCRYPTED_PROPERTY_NAME = 'beeper:image:encryption'; + /** * @deprecated */