From 5de8a8484d99b92b5aeec46ccd8a7f14ea646292 Mon Sep 17 00:00:00 2001 From: AlexZ005 Date: Wed, 23 Sep 2026 14:03:23 +0300 Subject: [PATCH 1/8] [feat] 30b P0: game sounds and game music (C5) - procedural sfx set, seven music presets, two local volumes - gameSfx.js (leaf): twenty procedural WebAudio effects with no asset files - click, pop, whoosh, success, fail, hit, kick, shoot, laser, explosion, coin, levelup, goal, whistle, cheer, step, ring, sparkle, hurt, portal - on the app's one AudioContext, through ONE "Game sounds" gain into the engine's sfx bus; a position spatialises it; a 24-voice cap so a sweep or a barrage cannot stall the audio thread. renderGameSound measures a sound offline (the suite's proof that every name makes one). - gameMusicPresets.js (pure, vitest) + gameMusic.js: arcade / ambient / dungeon / stadium / space / puzzle / studio as four-bar loops of data, scheduled with a 25 ms lookahead timer (not rAF - an immersive session stops window rAF) and numbered off the SESSION clock, so two peers on one preset hear the same bar. Quiet under the effects (MIX 0.55). Refused in Edit (a module starting music from register() must not score the editor) and stopped on the falling edge of Interact/Play. - gameFeel.js (leaf): the ONE "is the player playing" predicate (isLocked === true or editorMode === 'interact') every piece of game feel reads. - moduleSDK: api.playSound routes a game name to the new set, keeps the ping chimes (ding/chime/pluck/bell), and an UNKNOWN name is now a quiet no-op returning false (it used to fall through to the ding) - the C5 contract; api.music.play/stop/current/presets. - Settings > Interface > Sound: "Game sounds" and "Music" sliders, LOCAL, persisted via safeStorage (game:soundVolume / game:musicVolume). - gameKit.js: the new leaves under ONE debug-hook name (App.svelte tails 215/215/215). Suite game-audio (new): 25/25. vitest gameMusicPresets 6/6 (all vitest 233 green). Counterfactuals (all four broken in one run, each red on its own checks): - the stop-on-leave subscription removed -> 7.1, 7.2, 8.2 red (music kept playing, peak 0.0597) - the Edit refusal removed -> 5.1, 5.2 red (music in the editor, peak 0.0751) - the music volume no longer reaching the live gain -> 9.2 red - the coin built as silence -> 1.2 red (quiet: ["coin"]) Held: hud-actions 65/0, game-editor-flow 41/0, sdk-game-seams 78/0 (module-toolbox and editor-modes waited on the machine lock; run at P1). svelte-check 335/47, identical list to base. Build green. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/App.svelte | 8 +- src/components/menu/Settings.svelte | 14 ++ src/lib/gameFeel.js | 38 +++ src/lib/gameKit.js | 7 + src/lib/gameMusic.js | 236 +++++++++++++++++ src/lib/gameMusicPresets.js | 202 +++++++++++++++ src/lib/gameSfx.js | 378 ++++++++++++++++++++++++++++ src/lib/moduleSDK.js | 39 ++- tests/e2e/game-audio.test.cjs | 190 ++++++++++++++ tests/unit/gameMusicPresets.test.js | 62 +++++ 10 files changed, 1168 insertions(+), 6 deletions(-) create mode 100644 src/lib/gameFeel.js create mode 100644 src/lib/gameKit.js create mode 100644 src/lib/gameMusic.js create mode 100644 src/lib/gameMusicPresets.js create mode 100644 src/lib/gameSfx.js create mode 100644 tests/e2e/game-audio.test.cjs create mode 100644 tests/unit/gameMusicPresets.test.js diff --git a/src/App.svelte b/src/App.svelte index 40cf9592..01ac8446 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -478,9 +478,11 @@ import { startMusicToolbox } from './lib/musicToolbox' import('./lib/importGate'), import('./lib/decimate'), import('./lib/playCursor'), - import('./lib/gameStorage') - ]).then(([sceneStore, appStore, flowStore, meshEdit, vrControls, autosave, voiceChat, audioEngineLib, musicClockLib, audioDevicesLib, audioPatchLib, vrPatchLib, musicToolboxLib, micCaptureLib, annotationsHandler, flowRuntime, history, materialsHandler, objectActions, commandsHandler, moduleSDK, drawModeLib, pathCapture, lockControl, prefabsLib, physics, jointsLib, possessLib, handModelsLib, terrainSculptLib, userModulesLib, environmentLib, sceneMusicLib, animatedImports, fileHandler, fileWindowsLib, sceneBounds, cameraClip, ping, sessionsLib, geometryEdit, lightParams, shadowDefaultsLib, paletteLib, viewModeLib, inputRuntimeLib, shortcutsLib, themesLib, vrRadialMenu, vrPaletteLib, vrWindowPosesLib, vrKeyboardLib, faceEditLib, meshToolParamsLib, avatarModelLib, explorerLib, bottomDock, explorerDrop, assetShare, soundRuntime, dungeonPlay, sceneAssetsLib, THREE, GLTFExporterModule, snappingLib, flowSocketsLib, networkQualityLib, packsLib, customNodesLib, nodesHandlerLib, nodeCatalogLib, objectMenuLib, animationPreviewLib, aiProvidersLib, aiToolsLib, aiAssistantLib, meshProvidersLib, meshJobsLib, flowGraphsLib, objectFlowLib, peerServerLib, cloudHooksLib, cloudPluginLib, connectionStateLib, peerApprovalLib, particleRuntimeLib, particleActionsLib, particlePresetsLib, versionLib, whatsNewLib, confirmDialogLib, scenePhysicsLib, playInteractLib, moveSmoothingLib, knockLib, playSettingsLib, colliderSpecLib, colliderHelpersLib, colliderEditLib, editSessionLib, trackpadNavLib, vrSleeveLib, gridSettingsLib, viewPrefsLib, cameraBookmarksLib, cameraObjectsLib, cameraHelpersLib, onionSkinLib, cameraPreviewLib, addObjectsLib, cameraPipLib, inputDeviceLib, sceneTemplatesLib, bvhPickingLib, multiTransformLib, objectOriginLib, moduleGalleryLib, uvEditorLib, uvUnwrapLib, meshTopologyLib, meshBudgetLib, proportionalLib, proportionalRingLib, scenePickLib, snapEngineLib, meshPivotLib, selectionPrefsLib, editOverlaysLib, objectPermissionsLib, scenePostLib, postEffectsLib, viewportOverridesLib, postprocessingModule, shaderBackendsLib, shaderGraphLib, shaderSyncLib, shaderTexturesLib, shaderCatalogLib, unitsLib, postBackendsLib, workspaceLib, editResumeLib, moduleRequirementsLib, hudDocsLib, hudSyncLib, idbLib, hudKindsLib, hudImagesLib, gameStateLib, gameSyncLib, hudActionsLib, moduleNodeIOLib, moduleToolboxesLib, splineTubeLib, splineToolLib, splineEditLib, terrainCarveLib, flattenActionsLib, hudViewportDragLib, gamepadPrefsLib, charControllerLib, hudRichTextLib, moduleHudKindsLib, hudMinimapLib, hudArrangeLib, gamePresenceLib, levelsLib, peerVarsLib, projectManifestLib, projectFileLib, transientObjectsLib, spawnerLib, triggerSyncLib, saveNameLib, sceneIdentityLib, importDuplicatesLib, peerScenesLib, sharedLibraryLib, transferLedgerLib, explorerViewLib, filePreviewLib, saveAsLib, windowTabsLib, colocationLib, colocationCalibrateLib, colocationPresenceLib, xrAnchorsLib, colocationAnchorsLib, colocationNudgeLib, mountedVolumesLib, storageUsageLib, scenePrivacyLib, touchControlsLib, playModeLib, objectListNavLib, inviteLinksLib, helperLayerLib, explorerClipboardLib, lookPresenceLib, postGraphsLib, postGraphPresetsLib, materialSharingLib, diagnosticsLib, wireValidateLib, wireErrorsLib, safeStorageLib, sceneBudgetLib, overloadGuardLib, qualityGovernorLib, importBudgetLib, importGateLib, decimateLib, playCursorLib, gameStorageLib]) => { - window.__stores = { ...sceneStore, ...appStore, ...flowStore, meshEdit, vrControls, autosave, voiceChat, audioEngine: audioEngineLib, musicClock: musicClockLib, audioDevices: audioDevicesLib, audioPatch: audioPatchLib, vrPatch: vrPatchLib, musicToolbox: musicToolboxLib, micCapture: micCaptureLib, annotationsHandler, flowRuntime, history, materialsHandler, objectActions, commandsHandler, moduleSDK, drawMode: drawModeLib, pathCapture, lockControl, prefabs: prefabsLib, physics, joints: jointsLib, possess: possessLib, handModels: handModelsLib, terrainSculpt: terrainSculptLib, userModules: userModulesLib, environment: environmentLib, sceneMusic: sceneMusicLib, animatedImports, fileHandler, fileWindows: fileWindowsLib, sceneBounds, cameraClip, ping, sessions: sessionsLib, geometryEdit, lightParams, shadowDefaults: shadowDefaultsLib, palette: paletteLib, viewModeCtl: viewModeLib, inputRuntime: inputRuntimeLib, shortcutsRegistry: shortcutsLib, themes: themesLib, vrRadialMenu, vrPalette: vrPaletteLib, vrWindowPoses: vrWindowPosesLib, vrKeyboard: vrKeyboardLib, faceEdit: faceEditLib, meshToolParams: meshToolParamsLib, avatarModel: avatarModelLib, explorer: explorerLib, bottomDock, explorerDrop, assetShare, soundRuntime, dungeonPlay, sceneAssets: sceneAssetsLib, THREE, GLTFExporterModule, snapping: snappingLib, flowSockets: flowSocketsLib, networkQuality: networkQualityLib, packs: packsLib, customNodes: customNodesLib, nodesHandler: nodesHandlerLib, nodeCatalog: nodeCatalogLib, objectMenu: objectMenuLib, animationPreview: animationPreviewLib, aiProviders: aiProvidersLib, aiTools: aiToolsLib, aiAssistant: aiAssistantLib, meshProviders: meshProvidersLib, meshJobs: meshJobsLib, flowGraphsCtl: flowGraphsLib, objectFlow: objectFlowLib, peerServer: peerServerLib, cloudHooks: cloudHooksLib, cloudPlugin: cloudPluginLib, connectionState: connectionStateLib, peerApproval: peerApprovalLib, particleRuntime: particleRuntimeLib, particleActions: particleActionsLib, particlePresets: particlePresetsLib, version: versionLib, whatsNew: whatsNewLib, confirmDialog: confirmDialogLib, scenePhysics: scenePhysicsLib, playInteract: playInteractLib, moveSmoothing: moveSmoothingLib, knock: knockLib, playSettings: playSettingsLib, colliderSpec: colliderSpecLib, colliderHelpers: colliderHelpersLib, colliderEdit: colliderEditLib, editSession: editSessionLib, trackpadNav: trackpadNavLib, vrSleeve: vrSleeveLib, gridSettings: gridSettingsLib, viewPrefs: viewPrefsLib, cameraBookmarks: cameraBookmarksLib, cameraObjects: cameraObjectsLib, cameraHelpers: cameraHelpersLib, onionSkin: onionSkinLib, cameraPreview: cameraPreviewLib, addObjects: addObjectsLib, cameraPip: cameraPipLib, inputDevice: inputDeviceLib, sceneTemplates: sceneTemplatesLib, bvhPicking: bvhPickingLib, multiTransform: multiTransformLib, objectOrigin: objectOriginLib, moduleGallery: moduleGalleryLib, uvEditor: uvEditorLib, uvUnwrap: uvUnwrapLib, meshTopology: meshTopologyLib, meshBudget: meshBudgetLib, proportional: proportionalLib, proportionalRing: proportionalRingLib, scenePick: scenePickLib, snapEngine: snapEngineLib, meshPivot: meshPivotLib, selectionPrefs: selectionPrefsLib, editOverlays: editOverlaysLib, objectPermissions: objectPermissionsLib, scenePost: scenePostLib, postEffects: postEffectsLib, viewportOverrides: viewportOverridesLib, postprocessing: postprocessingModule, shaderBackends: shaderBackendsLib, shaderGraph: shaderGraphLib, shaderSync: shaderSyncLib, shaderTextures: shaderTexturesLib, shaderCatalog: shaderCatalogLib, units: unitsLib, postBackends: postBackendsLib, workspace: workspaceLib, editResume: editResumeLib, moduleRequirements: moduleRequirementsLib, hudDocs: hudDocsLib, hudSync: hudSyncLib, idb: idbLib, hudKinds: hudKindsLib, hudImages: hudImagesLib, gameState: gameStateLib, gameSync: gameSyncLib, hudActions: hudActionsLib, moduleNodeIO: moduleNodeIOLib, moduleToolboxes: moduleToolboxesLib, splineTube: splineTubeLib, splineTool: splineToolLib, splineEdit: splineEditLib, terrainCarve: terrainCarveLib, flattenActions: flattenActionsLib, hudViewportDrag: hudViewportDragLib, gamepadPrefs: gamepadPrefsLib, charController: charControllerLib, hudRichText: hudRichTextLib, moduleHudKinds: moduleHudKindsLib, hudMinimap: hudMinimapLib, hudArrange: hudArrangeLib, gamePresence: gamePresenceLib, levels: levelsLib, peerVars: peerVarsLib, projectManifest: projectManifestLib, projectFile: projectFileLib, transientObjects: transientObjectsLib, spawner: spawnerLib, triggerSync: triggerSyncLib, saveName: saveNameLib, sceneIdentity: sceneIdentityLib, importDuplicates: importDuplicatesLib, peerScenes: peerScenesLib, sharedLibrary: sharedLibraryLib, transferLedger: transferLedgerLib, explorerView: explorerViewLib, filePreview: filePreviewLib, saveAs: saveAsLib, windowTabs: windowTabsLib, colocation: colocationLib, colocationCalibrate: colocationCalibrateLib, colocationPresence: colocationPresenceLib, xrAnchors: xrAnchorsLib, colocationAnchors: colocationAnchorsLib, colocationNudge: colocationNudgeLib, mountedVolumes: mountedVolumesLib, storageUsage: storageUsageLib, scenePrivacy: scenePrivacyLib, touchControls: touchControlsLib, playMode: playModeLib, objectListNav: objectListNavLib, inviteLinks: inviteLinksLib, helperLayer: helperLayerLib, explorerClipboard: explorerClipboardLib, lookPresence: lookPresenceLib, postGraphs: postGraphsLib, postGraphPresets: postGraphPresetsLib, materialSharing: materialSharingLib, diagnostics: diagnosticsLib, wireValidate: wireValidateLib, wireErrors: wireErrorsLib, safeStorage: safeStorageLib, sceneBudget: sceneBudgetLib, overloadGuard: overloadGuardLib, qualityGovernor: qualityGovernorLib, importBudget: importBudgetLib, importGate: importGateLib, decimate: decimateLib, playCursor: playCursorLib, gameStorage: gameStorageLib } + import('./lib/gameStorage'), + // 30b (vr-play): the game-feel leaves under ONE name + import('./lib/gameKit') + ]).then(([sceneStore, appStore, flowStore, meshEdit, vrControls, autosave, voiceChat, audioEngineLib, musicClockLib, audioDevicesLib, audioPatchLib, vrPatchLib, musicToolboxLib, micCaptureLib, annotationsHandler, flowRuntime, history, materialsHandler, objectActions, commandsHandler, moduleSDK, drawModeLib, pathCapture, lockControl, prefabsLib, physics, jointsLib, possessLib, handModelsLib, terrainSculptLib, userModulesLib, environmentLib, sceneMusicLib, animatedImports, fileHandler, fileWindowsLib, sceneBounds, cameraClip, ping, sessionsLib, geometryEdit, lightParams, shadowDefaultsLib, paletteLib, viewModeLib, inputRuntimeLib, shortcutsLib, themesLib, vrRadialMenu, vrPaletteLib, vrWindowPosesLib, vrKeyboardLib, faceEditLib, meshToolParamsLib, avatarModelLib, explorerLib, bottomDock, explorerDrop, assetShare, soundRuntime, dungeonPlay, sceneAssetsLib, THREE, GLTFExporterModule, snappingLib, flowSocketsLib, networkQualityLib, packsLib, customNodesLib, nodesHandlerLib, nodeCatalogLib, objectMenuLib, animationPreviewLib, aiProvidersLib, aiToolsLib, aiAssistantLib, meshProvidersLib, meshJobsLib, flowGraphsLib, objectFlowLib, peerServerLib, cloudHooksLib, cloudPluginLib, connectionStateLib, peerApprovalLib, particleRuntimeLib, particleActionsLib, particlePresetsLib, versionLib, whatsNewLib, confirmDialogLib, scenePhysicsLib, playInteractLib, moveSmoothingLib, knockLib, playSettingsLib, colliderSpecLib, colliderHelpersLib, colliderEditLib, editSessionLib, trackpadNavLib, vrSleeveLib, gridSettingsLib, viewPrefsLib, cameraBookmarksLib, cameraObjectsLib, cameraHelpersLib, onionSkinLib, cameraPreviewLib, addObjectsLib, cameraPipLib, inputDeviceLib, sceneTemplatesLib, bvhPickingLib, multiTransformLib, objectOriginLib, moduleGalleryLib, uvEditorLib, uvUnwrapLib, meshTopologyLib, meshBudgetLib, proportionalLib, proportionalRingLib, scenePickLib, snapEngineLib, meshPivotLib, selectionPrefsLib, editOverlaysLib, objectPermissionsLib, scenePostLib, postEffectsLib, viewportOverridesLib, postprocessingModule, shaderBackendsLib, shaderGraphLib, shaderSyncLib, shaderTexturesLib, shaderCatalogLib, unitsLib, postBackendsLib, workspaceLib, editResumeLib, moduleRequirementsLib, hudDocsLib, hudSyncLib, idbLib, hudKindsLib, hudImagesLib, gameStateLib, gameSyncLib, hudActionsLib, moduleNodeIOLib, moduleToolboxesLib, splineTubeLib, splineToolLib, splineEditLib, terrainCarveLib, flattenActionsLib, hudViewportDragLib, gamepadPrefsLib, charControllerLib, hudRichTextLib, moduleHudKindsLib, hudMinimapLib, hudArrangeLib, gamePresenceLib, levelsLib, peerVarsLib, projectManifestLib, projectFileLib, transientObjectsLib, spawnerLib, triggerSyncLib, saveNameLib, sceneIdentityLib, importDuplicatesLib, peerScenesLib, sharedLibraryLib, transferLedgerLib, explorerViewLib, filePreviewLib, saveAsLib, windowTabsLib, colocationLib, colocationCalibrateLib, colocationPresenceLib, xrAnchorsLib, colocationAnchorsLib, colocationNudgeLib, mountedVolumesLib, storageUsageLib, scenePrivacyLib, touchControlsLib, playModeLib, objectListNavLib, inviteLinksLib, helperLayerLib, explorerClipboardLib, lookPresenceLib, postGraphsLib, postGraphPresetsLib, materialSharingLib, diagnosticsLib, wireValidateLib, wireErrorsLib, safeStorageLib, sceneBudgetLib, overloadGuardLib, qualityGovernorLib, importBudgetLib, importGateLib, decimateLib, playCursorLib, gameStorageLib, gameKitLib]) => { + window.__stores = { ...sceneStore, ...appStore, ...flowStore, meshEdit, vrControls, autosave, voiceChat, audioEngine: audioEngineLib, musicClock: musicClockLib, audioDevices: audioDevicesLib, audioPatch: audioPatchLib, vrPatch: vrPatchLib, musicToolbox: musicToolboxLib, micCapture: micCaptureLib, annotationsHandler, flowRuntime, history, materialsHandler, objectActions, commandsHandler, moduleSDK, drawMode: drawModeLib, pathCapture, lockControl, prefabs: prefabsLib, physics, joints: jointsLib, possess: possessLib, handModels: handModelsLib, terrainSculpt: terrainSculptLib, userModules: userModulesLib, environment: environmentLib, sceneMusic: sceneMusicLib, animatedImports, fileHandler, fileWindows: fileWindowsLib, sceneBounds, cameraClip, ping, sessions: sessionsLib, geometryEdit, lightParams, shadowDefaults: shadowDefaultsLib, palette: paletteLib, viewModeCtl: viewModeLib, inputRuntime: inputRuntimeLib, shortcutsRegistry: shortcutsLib, themes: themesLib, vrRadialMenu, vrPalette: vrPaletteLib, vrWindowPoses: vrWindowPosesLib, vrKeyboard: vrKeyboardLib, faceEdit: faceEditLib, meshToolParams: meshToolParamsLib, avatarModel: avatarModelLib, explorer: explorerLib, bottomDock, explorerDrop, assetShare, soundRuntime, dungeonPlay, sceneAssets: sceneAssetsLib, THREE, GLTFExporterModule, snapping: snappingLib, flowSockets: flowSocketsLib, networkQuality: networkQualityLib, packs: packsLib, customNodes: customNodesLib, nodesHandler: nodesHandlerLib, nodeCatalog: nodeCatalogLib, objectMenu: objectMenuLib, animationPreview: animationPreviewLib, aiProviders: aiProvidersLib, aiTools: aiToolsLib, aiAssistant: aiAssistantLib, meshProviders: meshProvidersLib, meshJobs: meshJobsLib, flowGraphsCtl: flowGraphsLib, objectFlow: objectFlowLib, peerServer: peerServerLib, cloudHooks: cloudHooksLib, cloudPlugin: cloudPluginLib, connectionState: connectionStateLib, peerApproval: peerApprovalLib, particleRuntime: particleRuntimeLib, particleActions: particleActionsLib, particlePresets: particlePresetsLib, version: versionLib, whatsNew: whatsNewLib, confirmDialog: confirmDialogLib, scenePhysics: scenePhysicsLib, playInteract: playInteractLib, moveSmoothing: moveSmoothingLib, knock: knockLib, playSettings: playSettingsLib, colliderSpec: colliderSpecLib, colliderHelpers: colliderHelpersLib, colliderEdit: colliderEditLib, editSession: editSessionLib, trackpadNav: trackpadNavLib, vrSleeve: vrSleeveLib, gridSettings: gridSettingsLib, viewPrefs: viewPrefsLib, cameraBookmarks: cameraBookmarksLib, cameraObjects: cameraObjectsLib, cameraHelpers: cameraHelpersLib, onionSkin: onionSkinLib, cameraPreview: cameraPreviewLib, addObjects: addObjectsLib, cameraPip: cameraPipLib, inputDevice: inputDeviceLib, sceneTemplates: sceneTemplatesLib, bvhPicking: bvhPickingLib, multiTransform: multiTransformLib, objectOrigin: objectOriginLib, moduleGallery: moduleGalleryLib, uvEditor: uvEditorLib, uvUnwrap: uvUnwrapLib, meshTopology: meshTopologyLib, meshBudget: meshBudgetLib, proportional: proportionalLib, proportionalRing: proportionalRingLib, scenePick: scenePickLib, snapEngine: snapEngineLib, meshPivot: meshPivotLib, selectionPrefs: selectionPrefsLib, editOverlays: editOverlaysLib, objectPermissions: objectPermissionsLib, scenePost: scenePostLib, postEffects: postEffectsLib, viewportOverrides: viewportOverridesLib, postprocessing: postprocessingModule, shaderBackends: shaderBackendsLib, shaderGraph: shaderGraphLib, shaderSync: shaderSyncLib, shaderTextures: shaderTexturesLib, shaderCatalog: shaderCatalogLib, units: unitsLib, postBackends: postBackendsLib, workspace: workspaceLib, editResume: editResumeLib, moduleRequirements: moduleRequirementsLib, hudDocs: hudDocsLib, hudSync: hudSyncLib, idb: idbLib, hudKinds: hudKindsLib, hudImages: hudImagesLib, gameState: gameStateLib, gameSync: gameSyncLib, hudActions: hudActionsLib, moduleNodeIO: moduleNodeIOLib, moduleToolboxes: moduleToolboxesLib, splineTube: splineTubeLib, splineTool: splineToolLib, splineEdit: splineEditLib, terrainCarve: terrainCarveLib, flattenActions: flattenActionsLib, hudViewportDrag: hudViewportDragLib, gamepadPrefs: gamepadPrefsLib, charController: charControllerLib, hudRichText: hudRichTextLib, moduleHudKinds: moduleHudKindsLib, hudMinimap: hudMinimapLib, hudArrange: hudArrangeLib, gamePresence: gamePresenceLib, levels: levelsLib, peerVars: peerVarsLib, projectManifest: projectManifestLib, projectFile: projectFileLib, transientObjects: transientObjectsLib, spawner: spawnerLib, triggerSync: triggerSyncLib, saveName: saveNameLib, sceneIdentity: sceneIdentityLib, importDuplicates: importDuplicatesLib, peerScenes: peerScenesLib, sharedLibrary: sharedLibraryLib, transferLedger: transferLedgerLib, explorerView: explorerViewLib, filePreview: filePreviewLib, saveAs: saveAsLib, windowTabs: windowTabsLib, colocation: colocationLib, colocationCalibrate: colocationCalibrateLib, colocationPresence: colocationPresenceLib, xrAnchors: xrAnchorsLib, colocationAnchors: colocationAnchorsLib, colocationNudge: colocationNudgeLib, mountedVolumes: mountedVolumesLib, storageUsage: storageUsageLib, scenePrivacy: scenePrivacyLib, touchControls: touchControlsLib, playMode: playModeLib, objectListNav: objectListNavLib, inviteLinks: inviteLinksLib, helperLayer: helperLayerLib, explorerClipboard: explorerClipboardLib, lookPresence: lookPresenceLib, postGraphs: postGraphsLib, postGraphPresets: postGraphPresetsLib, materialSharing: materialSharingLib, diagnostics: diagnosticsLib, wireValidate: wireValidateLib, wireErrors: wireErrorsLib, safeStorage: safeStorageLib, sceneBudget: sceneBudgetLib, overloadGuard: overloadGuardLib, qualityGovernor: qualityGovernorLib, importBudget: importBudgetLib, importGate: importGateLib, decimate: decimateLib, playCursor: playCursorLib, gameStorage: gameStorageLib, gameKit: gameKitLib } }) } }) diff --git a/src/components/menu/Settings.svelte b/src/components/menu/Settings.svelte index 496ad0be..be5b89b2 100644 --- a/src/components/menu/Settings.svelte +++ b/src/components/menu/Settings.svelte @@ -3,6 +3,9 @@ import { HardDrive, Lock, RotateCcw, X } from '@lucide/svelte'; import ThemedSelect from '../ui/ThemedSelect.svelte'; import SettingRow from './SettingRow.svelte'; + // 30b (vr-play) C5: the two LOCAL game-audio volumes + import { gameSoundVolume } from '$lib/gameSfx'; + import { gameMusicVolume } from '$lib/gameMusic'; import { showGrid, vrOverride, vrMenuHand, vrSnapAngle, vrMirrorSnapTurn, vrTeleportEnabled, vrSleeveEnabled, vrVertexHold, vrFlying, vrPassthrough, vrMenuHold, vrTargetHz, peerHandStyle } from '../../stores/sceneStore.js'; import { applyVRFrameRate } from '$lib/vrControls'; import { settingsOpen, settingsSection, hidePanels, restorePanels, advancedMode, showEnvInList, objectSearchEnabled, showSimControls, showToast, showRoomsButton, toastsInDrawerOnly, mobileUndockAllowed, enableShiftAdd, noteDoubleClickToOpen, duplicateCarriesAnimation, duplicateCarriesFlow, duplicateCarriesShader, touchTools, floatingToolbar, toolbarAlwaysOnTop } from '../../stores/appStore.js'; @@ -767,6 +770,17 @@ {/if} +

Sound

+ + + How loud a game's effects are on this device — coins, goals, hits, the clicks a + game makes. Local to you; {Math.round($gameSoundVolume * 100)}% + + + + The background music a game plays while you are in Interact or Play (it stops when + you go back to editing). Local to you; {Math.round($gameMusicVolume * 100)}% +

Notifications

diff --git a/src/lib/gameFeel.js b/src/lib/gameFeel.js new file mode 100644 index 00000000..400353ae --- /dev/null +++ b/src/lib/gameFeel.js @@ -0,0 +1,38 @@ +// 30b (vr-play): "IS THE PLAYER PLAYING?" — the ONE predicate behind every piece of game +// feel: controller haptics, game music, the VR game panel and the hold-and-sweep. +// +// The user's rule, verbatim from the Quest report: "The vibration should be only +// interactive mode, not in edit mode." A game feels like a game in INTERACT and in PLAY, +// and the editor stays quiet. Both halves are LOCAL state on this device (`editorMode` is +// never sent, `isLocked` is this peer's play press), so nothing here replicates. +// +// `isLocked` is THREE-state (null editor / true playing / false the exit transient), so +// playing is `=== true` — the transient must read "not playing" (the 21-E3 rule). +// VR has no pointer lock: a headset session enters INTERACT (30b-vr-modes' C1), which is +// why `editorMode === 'interact'` alone is also enough. +// +// A LEAF: sceneStore + svelte/store only, so vrControls, moduleSDK, the audio leaves and +// the HUD layer can all read it without reaching the history cycle family. +import { derived, get } from 'svelte/store'; +import { isLocked, editorMode } from '../stores/sceneStore'; + +/** true while this device is in Interact or Play @returns {boolean} */ +export function gameFeelActive() { + return get(isLocked) === true || get(editorMode) === 'interact'; +} + +/** the same answer as a store — a falling edge is "the player left the game" */ +export const gameFeelOn = derived( + [isLocked, editorMode], + ([$locked, $mode]) => $locked === true || $mode === 'interact' +); + +/** + * The click MODE a game-side press dispatches with (moduleSDK's CLICK_MODES): 'play' + * under desktop Play, 'interact' in Interact (VR Play enters Interact), 'edit' otherwise. + * @returns {'edit' | 'interact' | 'play'} + */ +export function gameClickMode() { + if (get(isLocked) === true) return 'play'; + return get(editorMode) === 'interact' ? 'interact' : 'edit'; +} diff --git a/src/lib/gameKit.js b/src/lib/gameKit.js new file mode 100644 index 00000000..a3e6c5ce --- /dev/null +++ b/src/lib/gameKit.js @@ -0,0 +1,7 @@ +// 30b (vr-play): the game-feel leaves under ONE debug-hook name (`__stores.gameKit`), so +// this lane adds one entry to App.svelte's three tails instead of one per leaf. Nothing in +// the app imports this file; it exists for the suites. +export * as gameFeel from './gameFeel'; +export * as gameSfx from './gameSfx'; +export * as gameMusic from './gameMusic'; +export * as gameMusicPresets from './gameMusicPresets'; diff --git a/src/lib/gameMusic.js b/src/lib/gameMusic.js new file mode 100644 index 00000000..40fdbde4 --- /dev/null +++ b/src/lib/gameMusic.js @@ -0,0 +1,236 @@ +// 30b (vr-play) C5 — GAME MUSIC: `api.music.play(preset, {volume})` / `api.music.stop()`. +// Seven procedural looping presets (gameMusicPresets.js is the tune as data), scheduled on +// the app's one AudioContext and LOCAL to this device — a peer's music is theirs. +// +// THREE RULES, each one the user's: +// · it plays only while the player is PLAYING (Interact or Play — gameFeel.js). A call +// from the editor is refused, and leaving the game stops it: "music stops on leaving +// Play/Interact". Refusing in Edit is stricter than "stop on leave" and is on purpose — +// a module that started its music from register() would otherwise score the editor. +// · it sits UNDER the effects: one "Music" volume in Settings (persisted via safeStorage) +// times a quiet MIX constant, into the engine's `music` bus. +// · it is TEMPO-SYNCED to the SESSION clock: the step that plays is floor(sessionNow / +// step length), never "steps since the button" — two peers who start one preset hear +// the same bar at the same moment, and a restart lands back on the beat. +// +// The scheduler is the classic lookahead: a 25 ms timer schedules every step whose time +// falls inside the next LOOKAHEAD seconds on the audio clock (audioTimeFor maps a session +// stamp through the engine's filtered clock). A TIMER, not a rAF: an immersive session +// stops window rAF on the Quest, and music must not. +import { writable, get } from 'svelte/store'; +import { ensureAudioContext, bus, audioTimeFor } from './audioEngine'; +import { sessionNow } from './sessionClock'; +import { safeStorage } from './safeStorage'; +import { gameFeelOn, gameFeelActive } from './gameFeel'; +import { sfxTone, sfxNoise } from './gameSfx'; +import { musicPreset, stepSeconds, stepEvents, MUSIC_PRESET_IDS } from './gameMusicPresets'; + +export { MUSIC_PRESET_IDS }; + +const VOLUME_KEY = 'game:musicVolume'; +/** music under effects: the slider's full scale is this loud */ +const MIX = 0.55; +const LOOKAHEAD = 0.18; +const TICK_MS = 25; + +/** @param {string | null} raw @param {number} fallback */ +function readVolume(raw, fallback) { + const n = raw === null ? NaN : Number(raw); + return Number.isFinite(n) ? Math.min(1, Math.max(0, n)) : fallback; +} + +/** "Music" volume, 0..1, LOCAL per device (Settings ▸ Interface ▸ Sound). */ +export const gameMusicVolume = writable(readVolume(safeStorage.getItem(VOLUME_KEY), 0.6)); + +/** what is playing now: `{preset, volume}` or null @type {import('svelte/store').Writable<{preset: string, volume: number} | null>} */ +export const gameMusicState = writable(null); + +/** + * The running session. `gain` is PER PLAY: stopping fades and disconnects it, so any note + * already scheduled into it goes silent with it — no voice bookkeeping needed. + * @type {{preset: any, volume: number, gain: GainNode, timer: any, lastStep: number, scheduled: number} | null} + */ +let current = null; +const debug = { steps: 0, notes: 0, refused: 0, stops: 0 }; + +/** @param {number} volume the per-call volume */ +function levelFor(volume) { + return MIX * get(gameMusicVolume) * volume; +} + +// Declared ABOVE the subscribes (they run synchronously at module evaluation). +gameMusicVolume.subscribe((v) => { + safeStorage.setItem(VOLUME_KEY, String(v)); + if (current) current.gain.gain.value = levelFor(current.volume); +}); +// "music stops on leaving Play/Interact" — the falling edge of the one predicate +gameFeelOn.subscribe((on) => { + if (!on && current) stopGameMusic(); +}); + +/** @param {number} midi */ +const hz = (midi) => 440 * Math.pow(2, (midi - 69) / 12); + +/** + * Build one step's notes into `dest` at audio time `t`. Shared by the live scheduler and + * the offline render, so what the suite measures is what a player hears. + * @param {BaseAudioContext} ctx @param {AudioNode} dest @param {any} preset + * @param {number} step @param {number} t @returns {number} notes built + */ +export function buildMusicStep(ctx, dest, preset, step, t) { + const beat = 60 / preset.bpm; + let n = 0; + for (const event of stepEvents(preset, step)) { + const g = preset.drumGain; + switch (event.kind) { + case 'kick': + sfxTone(ctx, dest, { freq: 140, to: 44, t0: t, dur: 0.24, peak: 0.55 * g, attack: 0.002 }); + break; + case 'snare': + sfxNoise(ctx, dest, { t0: t, dur: 0.14, peak: 0.22 * g, filter: 'bandpass', freq: 1800, q: 0.7, attack: 0.002 }); + sfxTone(ctx, dest, { freq: 190, to: 140, t0: t, dur: 0.08, peak: 0.12 * g, attack: 0.002 }); + break; + case 'hat': + sfxNoise(ctx, dest, { t0: t, dur: 0.04, peak: 0.08 * g, filter: 'highpass', freq: 7000, attack: 0.001 }); + break; + case 'bass': { + const dur = Math.max(0.12, (event.beats ?? 0.5) * beat * 0.95); + const v = preset.bassVoice; + const lp = ctx.createBiquadFilter(); + lp.type = 'lowpass'; + lp.frequency.value = v.cutoff; + lp.connect(dest); + sfxTone(ctx, lp, { freq: hz(/** @type {number} */ (event.midi)), type: v.type, t0: t, dur, peak: v.gain, attack: 0.01 }); + break; + } + case 'pad': { + const dur = (event.beats ?? 4) * beat; + for (const m of /** @type {number[]} */ (event.midi)) + sfxTone(ctx, dest, { freq: hz(m), type: preset.pad.type, t0: t, dur, peak: preset.pad.gain, attack: Math.min(0.6, dur * 0.3) }); + break; + } + case 'arp': + sfxTone(ctx, dest, { freq: hz(/** @type {number} */ (event.midi)), type: preset.arp.type, t0: t, dur: Math.max(0.1, beat * 0.45), peak: preset.arp.gain, attack: 0.004 }); + break; + } + n++; + } + return n; +} + +function tick() { + if (!current) return; + const ctx = ensureAudioContext(); + const stepMs = stepSeconds(current.preset) * 1000; + const now = sessionNow(); + const horizon = now + LOOKAHEAD * 1000; + // the first tick starts at the NEXT step boundary — a step already begun would land + // part-way through its own notes + let step = current.lastStep < 0 ? Math.floor(now / stepMs) + 1 : current.lastStep + 1; + // a stalled timer (a background tab) never replays the backlog: skip to now + if (step * stepMs < now - 200) step = Math.floor(now / stepMs) + 1; + for (; step * stepMs <= horizon; step++) { + const at = audioTimeFor(step * stepMs); + if (at < ctx.currentTime) continue; + debug.notes += buildMusicStep(ctx, current.gain, current.preset, step, at); + debug.steps++; + current.lastStep = step; + } +} + +/** + * Start (or switch to) a preset. Refused — false — outside Interact/Play, and for an + * unknown preset name. The same preset already playing only takes the new volume. + * @param {string} presetId @param {{volume?: number}} [options] + * @returns {boolean} + */ +export function playGameMusic(presetId, options = {}) { + const preset = musicPreset(presetId); + if (!preset || !gameFeelActive()) { + debug.refused++; + return false; + } + const raw = Number(options?.volume); + const volume = Number.isFinite(raw) ? Math.min(1, Math.max(0, raw)) : 1; + if (current && current.preset.id === preset.id) { + current.volume = volume; + current.gain.gain.value = levelFor(volume); + gameMusicState.set({ preset: preset.id, volume }); + return true; + } + if (current) stopGameMusic(); + /** @type {AudioContext} */ + let ctx; + try { + ctx = ensureAudioContext(); + } catch { + return false; + } + if (ctx.state === 'suspended') ctx.resume().catch(() => {}); + const gain = ctx.createGain(); + gain.gain.value = levelFor(volume); + gain.connect(bus('music')); + current = { preset, volume, gain, timer: setInterval(tick, TICK_MS), lastStep: -1, scheduled: 0 }; + tick(); + gameMusicState.set({ preset: preset.id, volume }); + return true; +} + +/** Stop whatever is playing (a short fade — a hard cut clicks). Safe to call anytime. */ +export function stopGameMusic() { + if (!current) return; + const { gain, timer } = current; + current = null; + clearInterval(timer); + debug.stops++; + try { + const ctx = ensureAudioContext(); + gain.gain.cancelScheduledValues(ctx.currentTime); + gain.gain.setValueAtTime(gain.gain.value, ctx.currentTime); + gain.gain.linearRampToValueAtTime(0, ctx.currentTime + 0.3); + } catch { + /* a closed context has nothing left to fade */ + } + setTimeout(() => { + try { + gain.disconnect(); + } catch {} + }, 450); + gameMusicState.set(null); +} + +/** + * Render `bars` bars of a preset OFFLINE and measure them (the suite's proof that every + * preset plays a real loop). + * @param {string} presetId @param {number} [bars] @param {number} [sampleRate] + * @returns {Promise<{preset: string, seconds: number, rms: number, peak: number, notes: number} | null>} + */ +export async function renderGameMusic(presetId, bars = 1, sampleRate = 22050) { + const preset = musicPreset(presetId); + const Offline = typeof window !== 'undefined' ? /** @type {any} */ (window).OfflineAudioContext : null; + if (!preset || !Offline) return null; + const step = stepSeconds(preset); + const seconds = bars * 16 * step + 1; + const ctx = new Offline(1, Math.ceil(seconds * sampleRate), sampleRate); + let notes = 0; + for (let i = 0; i < bars * 16; i++) notes += buildMusicStep(ctx, ctx.destination, preset, i, i * step); + const buffer = await ctx.startRendering(); + const data = buffer.getChannelData(0); + let sum = 0; + let peak = 0; + for (let i = 0; i < data.length; i++) { + sum += data[i] * data[i]; + peak = Math.max(peak, Math.abs(data[i])); + } + return { preset: preset.id, seconds, rms: Math.sqrt(sum / data.length), peak, notes }; +} + +/** the live gain's value, or null when nothing plays */ +export function gameMusicGainValue() { + return current ? current.gain.gain.value : null; +} + +/** counts for the suites */ +export function gameMusicDebug() { + return { ...debug, playing: current?.preset.id ?? null, lastStep: current?.lastStep ?? -1 }; +} diff --git a/src/lib/gameMusicPresets.js b/src/lib/gameMusicPresets.js new file mode 100644 index 00000000..35082b13 --- /dev/null +++ b/src/lib/gameMusicPresets.js @@ -0,0 +1,202 @@ +// 30b (vr-play) C5 — the game MUSIC presets as DATA, plus the pure arithmetic that turns +// a preset and a step number into the notes that step plays. Imports NOTHING: the +// scheduler (gameMusic.js) is the only thing that touches WebAudio, so everything a +// listener would call "the tune" is testable with no browser. +// +// A preset is a four-bar loop in sixteenth notes: a chord PROGRESSION (scale degrees, one +// per bar), drum lanes as 16-char strings ('x' hit, '.' rest), a bass lane ('r' root, +// 'o' octave, 'f' fifth, 't' third), an optional pad (the bar's triad, held) and an +// optional ARP lane of chord-tone indices ('0'-'5' = triad tones up two octaves). +// +// Everything is keyed to a GLOBAL step count (the session clock divided by the step +// length), never to "when play was pressed" — so two peers who start the same preset hear +// the same bar at the same moment, and "tempo-synced" means synced to the session. + +/** @typedef {{kind: 'kick'|'snare'|'hat'|'bass'|'pad'|'arp', midi?: number[] | number, beats?: number}} MusicEvent */ + +/** + * @typedef {object} MusicPreset + * @property {string} id @property {string} name + * @property {number} bpm + * @property {number} root midi note of the key's tonic (octave 4-ish) + * @property {number[]} scale semitone offsets of the mode + * @property {number[]} progression scale degrees (0-based), one chord per bar + * @property {{kick?: string, snare?: string, hat?: string}} drums 16 chars each + * @property {string} bass 16 chars: r o f t . + * @property {{type: OscillatorType, gain: number} | null} pad + * @property {{lane: string, type: OscillatorType, gain: number, octave: number} | null} arp + * @property {{type: OscillatorType, gain: number, cutoff: number}} bassVoice + * @property {number} drumGain + */ + +const MAJOR = [0, 2, 4, 5, 7, 9, 11]; +const MINOR = [0, 2, 3, 5, 7, 8, 10]; +const DORIAN = [0, 2, 3, 5, 7, 9, 10]; +const PHRYGIAN = [0, 1, 3, 5, 7, 8, 10]; +const LYDIAN = [0, 2, 4, 6, 7, 9, 11]; + +/** @type {MusicPreset[]} */ +export const MUSIC_PRESETS = [ + { + id: 'arcade', + name: 'Arcade', + bpm: 138, + root: 60, + scale: MAJOR, + progression: [0, 5, 3, 4], + drums: { kick: 'x...x...x...x...', snare: '....x.......x...', hat: '..x...x...x...x.' }, + bass: 'r.r.o.r.r.r.o.f.', + pad: null, + arp: { lane: '0.1.2.3.2.1.0.1.', type: 'square', gain: 0.045, octave: 1 }, + bassVoice: { type: 'square', gain: 0.07, cutoff: 900 }, + drumGain: 0.8 + }, + { + id: 'ambient', + name: 'Ambient', + bpm: 72, + root: 62, + scale: LYDIAN, + progression: [0, 3, 5, 4], + drums: {}, + bass: 'r...............', + pad: { type: 'triangle', gain: 0.05 }, + arp: { lane: '0.......2.......', type: 'sine', gain: 0.04, octave: 2 }, + bassVoice: { type: 'sine', gain: 0.09, cutoff: 500 }, + drumGain: 0 + }, + { + id: 'dungeon', + name: 'Dungeon', + bpm: 84, + root: 57, + scale: PHRYGIAN, + progression: [0, 0, 1, 6], + drums: { kick: 'x.......x.x.....', snare: '............x...', hat: '' }, + bass: 'r.......r...f...', + pad: { type: 'sawtooth', gain: 0.022 }, + arp: { lane: '....2.......1...', type: 'triangle', gain: 0.035, octave: 1 }, + bassVoice: { type: 'sawtooth', gain: 0.07, cutoff: 350 }, + drumGain: 0.7 + }, + { + id: 'stadium', + name: 'Stadium', + bpm: 120, + root: 55, + scale: MAJOR, + progression: [0, 4, 5, 3], + drums: { kick: 'x...x...x...x...', snare: '....x.......x.x.', hat: 'x.x.x.x.x.x.x.x.' }, + bass: 'r.r.r.r.f.f.o.o.', + pad: { type: 'sawtooth', gain: 0.025 }, + arp: null, + bassVoice: { type: 'sawtooth', gain: 0.07, cutoff: 700 }, + drumGain: 0.9 + }, + { + id: 'space', + name: 'Space', + bpm: 100, + root: 64, + scale: DORIAN, + progression: [0, 6, 5, 4], + drums: { kick: 'x.........x.....', snare: '....x.......x...', hat: '..x...x...x...xx' }, + bass: 'r.....r.....f...', + pad: { type: 'triangle', gain: 0.035 }, + arp: { lane: '0.2.4.2.1.3.5.3.', type: 'triangle', gain: 0.04, octave: 1 }, + bassVoice: { type: 'triangle', gain: 0.09, cutoff: 600 }, + drumGain: 0.6 + }, + { + id: 'puzzle', + name: 'Puzzle', + bpm: 96, + root: 65, + scale: MAJOR, + progression: [0, 3, 1, 4], + drums: { kick: 'x.......x.......', snare: '', hat: '....x.......x...' }, + bass: 'r...f...r...t...', + pad: null, + arp: { lane: '0..2..1..3..2...', type: 'triangle', gain: 0.05, octave: 1 }, + bassVoice: { type: 'triangle', gain: 0.08, cutoff: 800 }, + drumGain: 0.5 + }, + { + id: 'studio', + name: 'Studio', + bpm: 88, + root: 55, + scale: MINOR, + progression: [0, 3, 6, 4], + drums: { kick: 'x......x..x.....', snare: '....x.......x...', hat: 'x.xxx.xxx.xxx.xx' }, + bass: 'r.....r.f.....o.', + pad: { type: 'sine', gain: 0.045 }, + arp: null, + bassVoice: { type: 'sine', gain: 0.1, cutoff: 500 }, + drumGain: 0.6 + } +]; + +/** @param {string} id @returns {MusicPreset | null} */ +export function musicPreset(id) { + return MUSIC_PRESETS.find((p) => p.id === String(id)) ?? null; +} + +/** the names a module can ask for */ +export const MUSIC_PRESET_IDS = MUSIC_PRESETS.map((p) => p.id); + +/** seconds per sixteenth note @param {MusicPreset} preset */ +export function stepSeconds(preset) { + return 60 / preset.bpm / 4; +} + +/** + * The triad (three midi notes) on scale degree `degree`, stacked in thirds inside the + * mode — so a minor key's iv is minor and a major key's V is major, with no chord table. + * @param {MusicPreset} preset @param {number} degree @returns {number[]} + */ +export function triadOn(preset, degree) { + const s = preset.scale; + /** @param {number} i */ + const note = (i) => preset.root + s[((i % 7) + 7) % 7] + 12 * Math.floor(i / 7); + return [note(degree), note(degree + 2), note(degree + 4)]; +} + +/** @param {string | undefined} lane @param {number} i */ +const hitAt = (lane, i) => !!lane && lane[i] !== undefined && lane[i] !== '.'; + +/** + * What global step `step` plays. Pure: same preset + same step = same events, on every + * device. The chord changes per BAR (16 steps), cycling the progression. + * @param {MusicPreset} preset @param {number} step @returns {MusicEvent[]} + */ +export function stepEvents(preset, step) { + const i = ((step % 16) + 16) % 16; + const bar = Math.floor(step / 16); + const degree = preset.progression[((bar % preset.progression.length) + preset.progression.length) % preset.progression.length]; + const triad = triadOn(preset, degree); + /** @type {MusicEvent[]} */ + const out = []; + if (hitAt(preset.drums.kick, i)) out.push({ kind: 'kick' }); + if (hitAt(preset.drums.snare, i)) out.push({ kind: 'snare' }); + if (hitAt(preset.drums.hat, i)) out.push({ kind: 'hat' }); + const b = preset.bass[i]; + if (b && b !== '.') { + const base = triad[0] - 24; + const midi = b === 'o' ? base + 12 : b === 'f' ? base + 7 : b === 't' ? triad[1] - 24 : base; + // a bass note lasts until the next one in its lane + let len = 1; + while (len < 16 && (preset.bass[(i + len) % 16] ?? '.') === '.' && i + len < 16) len++; + out.push({ kind: 'bass', midi, beats: len / 4 }); + } + if (preset.pad && i === 0) out.push({ kind: 'pad', midi: triad, beats: 4 }); + if (preset.arp) { + const c = preset.arp.lane[i]; + if (c && c !== '.') { + const k = Number(c); + const tone = triad[k % 3] + 12 * (Math.floor(k / 3) + preset.arp.octave); + out.push({ kind: 'arp', midi: tone, beats: 0.25 }); + } + } + return out; +} diff --git a/src/lib/gameSfx.js b/src/lib/gameSfx.js new file mode 100644 index 00000000..702caca5 --- /dev/null +++ b/src/lib/gameSfx.js @@ -0,0 +1,378 @@ +// 30b (vr-play) C5 — THE GAME SOUND SET. Twenty procedural WebAudio effects a game can +// name (`api.playSound('coin')`), with NO asset files: every sound is oscillators, a +// shared noise buffer, filters and envelopes, built fresh per play on the app's one +// AudioContext (audioEngine — a second context would have its own listener and clock). +// +// Why procedural and not samples: a module is a self-contained zip, the sound set must be +// the same on every peer with nothing to fetch, and "the user asked for sounds in every +// game" is not worth a megabyte of wav per build. The ping chimes (pingAudio.js) set the +// precedent; this is the same idea grown to a game's vocabulary. +// +// ROUTING: every game sound goes through ONE gain (the "Game sounds" volume in Settings, +// persisted LOCALLY through safeStorage) into the engine's `sfx` bus, so the mixer and the +// limiter see it like any other effect. A `position` spatialises it through a PannerNode +// (the engine keeps the listener on the camera/headset whatever spatialVoice says). +// +// CAPPED: a sweep across a keyboard or a round of explosions must not stack hundreds of +// live voices — past MAX_LIVE the oldest-ending ones are simply not started (a dropped +// sound is inaudible under that many; a stalled audio thread is not). +// +// A LEAF: svelte/store + audioEngine + safeStorage. `renderGameSound` renders one into an +// OfflineAudioContext and MEASURES it — the suite's proof that every name makes a sound. +import { writable, get } from 'svelte/store'; +import { ensureAudioContext, bus } from './audioEngine'; +import { safeStorage } from './safeStorage'; + +/** The names a game can play. Anything else is a quiet no-op (never an error). */ +export const GAME_SOUNDS = [ + 'click', + 'pop', + 'whoosh', + 'success', + 'fail', + 'hit', + 'kick', + 'shoot', + 'laser', + 'explosion', + 'coin', + 'levelup', + 'goal', + 'whistle', + 'cheer', + 'step', + 'ring', + 'sparkle', + 'hurt', + 'portal' +]; +const SOUND_SET = new Set(GAME_SOUNDS); + +/** @param {string} name */ +export function isGameSound(name) { + return SOUND_SET.has(String(name)); +} + +const VOLUME_KEY = 'game:soundVolume'; +const MAX_LIVE = 24; + +/** @param {string | null} raw @param {number} fallback */ +function readVolume(raw, fallback) { + const n = raw === null ? NaN : Number(raw); + // Number.isFinite, never `|| fallback`: a slider dragged to ZERO is a real volume + return Number.isFinite(n) ? Math.min(1, Math.max(0, n)) : fallback; +} + +/** "Game sounds" volume, 0..1, LOCAL per device (Settings ▸ Interface ▸ Sound). */ +export const gameSoundVolume = writable(readVolume(safeStorage.getItem(VOLUME_KEY), 0.8)); + +/** @type {GainNode | null} */ +let gameGain = null; + +// Declared ABOVE the subscribe (the module-level subscribe runs synchronously — TDZ rule). +gameSoundVolume.subscribe((v) => { + safeStorage.setItem(VOLUME_KEY, String(v)); + if (gameGain) gameGain.gain.value = v; +}); + +/** the game-sound gain on the sfx bus, built once @returns {GainNode} */ +function gameOut() { + const ctx = ensureAudioContext(); + if (!gameGain) { + gameGain = ctx.createGain(); + gameGain.gain.value = get(gameSoundVolume); + gameGain.connect(bus('sfx')); + } + return gameGain; +} + +/* ------------------------------------------------------------------ the synth ---- */ + +/** @type {WeakMap} one second of white noise per context */ +const noiseBuffers = new WeakMap(); + +/** @param {BaseAudioContext} ctx @returns {AudioBuffer} */ +function noiseBuffer(ctx) { + let buffer = noiseBuffers.get(ctx); + if (buffer) return buffer; + buffer = ctx.createBuffer(1, ctx.sampleRate, ctx.sampleRate); + const data = buffer.getChannelData(0); + // a fixed LCG, so the same sound renders the same bytes on every device + let seed = 0x2f6b1d; + for (let i = 0; i < data.length; i++) { + seed = (seed * 1664525 + 1013904223) >>> 0; + data[i] = (seed / 0xffffffff) * 2 - 1; + } + noiseBuffers.set(ctx, buffer); + return buffer; +} + +/** exponential ramps cannot reach zero */ +const FLOOR = 0.0001; + +/** + * One enveloped oscillator, optionally gliding. + * @param {BaseAudioContext} ctx @param {AudioNode} dest + * @param {{freq: number, to?: number, type?: OscillatorType, t0: number, dur: number, + * peak?: number, attack?: number, vibrato?: {rate: number, depth: number}}} o + */ +function tone(ctx, dest, o) { + const { freq, to, type = 'sine', t0, dur, peak = 0.25, attack = 0.008, vibrato } = o; + const osc = ctx.createOscillator(); + const gain = ctx.createGain(); + osc.type = type; + osc.frequency.setValueAtTime(freq, t0); + if (to) osc.frequency.exponentialRampToValueAtTime(Math.max(20, to), t0 + dur); + if (vibrato) { + const lfo = ctx.createOscillator(); + const depth = ctx.createGain(); + lfo.frequency.value = vibrato.rate; + depth.gain.value = vibrato.depth; + lfo.connect(depth).connect(osc.frequency); + lfo.start(t0); + lfo.stop(t0 + dur + 0.05); + } + gain.gain.setValueAtTime(FLOOR, t0); + gain.gain.exponentialRampToValueAtTime(peak, t0 + attack); + gain.gain.exponentialRampToValueAtTime(FLOOR, t0 + dur); + osc.connect(gain).connect(dest); + osc.start(t0); + osc.stop(t0 + dur + 0.05); +} + +/** + * A filtered noise burst, optionally sweeping its filter. + * @param {BaseAudioContext} ctx @param {AudioNode} dest + * @param {{t0: number, dur: number, peak?: number, attack?: number, + * filter?: BiquadFilterType, freq?: number, to?: number, q?: number, flutter?: number}} o + */ +function noise(ctx, dest, o) { + const { t0, dur, peak = 0.3, attack = 0.005, filter = 'lowpass', freq = 2000, to, q = 0.8, flutter } = o; + const src = ctx.createBufferSource(); + src.buffer = noiseBuffer(ctx); + src.loop = true; + const biquad = ctx.createBiquadFilter(); + biquad.type = filter; + biquad.Q.value = q; + biquad.frequency.setValueAtTime(freq, t0); + if (to) biquad.frequency.exponentialRampToValueAtTime(Math.max(20, to), t0 + dur); + const gain = ctx.createGain(); + gain.gain.setValueAtTime(FLOOR, t0); + gain.gain.exponentialRampToValueAtTime(peak, t0 + attack); + gain.gain.exponentialRampToValueAtTime(FLOOR, t0 + dur); + src.connect(biquad).connect(gain); + if (flutter) { + // a crowd is not one steady hiss: an amplitude wobble makes it read as voices + const wobble = ctx.createGain(); + wobble.gain.value = 0.6; + const lfo = ctx.createOscillator(); + const depth = ctx.createGain(); + lfo.frequency.value = flutter; + depth.gain.value = 0.4; + lfo.connect(depth).connect(wobble.gain); + lfo.start(t0); + lfo.stop(t0 + dur + 0.05); + gain.connect(wobble).connect(dest); + } else gain.connect(dest); + src.start(t0, (t0 * 7.13) % 0.5); + src.stop(t0 + dur + 0.05); +} + +/** equal-tempered frequency of a midi note @param {number} midi */ +const hz = (midi) => 440 * Math.pow(2, (midi - 69) / 12); + +/** + * Build one sound into `dest` starting at `t0`. Returns its length in seconds, or -1 for + * a name that is not in the set. Pure WebAudio graph building — the live player and the + * offline measurement share it, so what the suite measures is what a player hears. + * @param {BaseAudioContext} ctx @param {AudioNode} dest @param {string} name @param {number} t0 + * @returns {number} + */ +export function buildGameSound(ctx, dest, name, t0) { + switch (name) { + case 'click': + tone(ctx, dest, { freq: 1900, type: 'square', t0, dur: 0.035, peak: 0.12, attack: 0.002 }); + noise(ctx, dest, { t0, dur: 0.03, peak: 0.12, filter: 'highpass', freq: 3000 }); + return 0.05; + case 'pop': + tone(ctx, dest, { freq: 380, to: 1100, t0, dur: 0.09, peak: 0.3, attack: 0.004 }); + return 0.1; + case 'whoosh': + noise(ctx, dest, { t0, dur: 0.45, peak: 0.35, attack: 0.12, filter: 'bandpass', freq: 300, to: 2600, q: 1.4 }); + return 0.5; + case 'success': + [72, 76, 79, 84].forEach((m, i) => + tone(ctx, dest, { freq: hz(m), type: 'triangle', t0: t0 + i * 0.08, dur: i === 3 ? 0.5 : 0.18, peak: 0.28 }) + ); + return 0.8; + case 'fail': + tone(ctx, dest, { freq: 330, type: 'square', t0, dur: 0.22, peak: 0.12 }); + tone(ctx, dest, { freq: 247, type: 'square', t0: t0 + 0.2, dur: 0.22, peak: 0.12 }); + tone(ctx, dest, { freq: 196, to: 150, type: 'sawtooth', t0: t0 + 0.4, dur: 0.45, peak: 0.12 }); + return 0.9; + case 'hit': + tone(ctx, dest, { freq: 160, to: 55, t0, dur: 0.18, peak: 0.5, attack: 0.002 }); + noise(ctx, dest, { t0, dur: 0.12, peak: 0.4, freq: 1400, to: 300, attack: 0.002 }); + return 0.22; + case 'kick': + tone(ctx, dest, { freq: 150, to: 42, t0, dur: 0.3, peak: 0.7, attack: 0.002 }); + noise(ctx, dest, { t0, dur: 0.02, peak: 0.15, filter: 'highpass', freq: 2000, attack: 0.001 }); + return 0.32; + case 'shoot': + tone(ctx, dest, { freq: 900, to: 140, type: 'square', t0, dur: 0.16, peak: 0.16, attack: 0.002 }); + noise(ctx, dest, { t0, dur: 0.1, peak: 0.3, freq: 3000, to: 500, attack: 0.001 }); + return 0.2; + case 'laser': + tone(ctx, dest, { freq: 1500, to: 280, type: 'sawtooth', t0, dur: 0.26, peak: 0.14, attack: 0.003 }); + tone(ctx, dest, { freq: 3000, to: 560, type: 'sine', t0, dur: 0.2, peak: 0.08, attack: 0.003 }); + return 0.3; + case 'explosion': + noise(ctx, dest, { t0, dur: 1.3, peak: 0.7, freq: 2200, to: 120, attack: 0.004, q: 0.5 }); + tone(ctx, dest, { freq: 90, to: 28, t0, dur: 0.9, peak: 0.6, attack: 0.004 }); + return 1.35; + case 'coin': + tone(ctx, dest, { freq: hz(83), type: 'square', t0, dur: 0.08, peak: 0.13, attack: 0.002 }); + tone(ctx, dest, { freq: hz(88), type: 'square', t0: t0 + 0.07, dur: 0.35, peak: 0.13, attack: 0.002 }); + return 0.45; + case 'levelup': + [60, 64, 67, 72, 76, 79, 84, 88].forEach((m, i) => + tone(ctx, dest, { freq: hz(m), type: 'triangle', t0: t0 + i * 0.055, dur: i === 7 ? 0.6 : 0.12, peak: 0.24 }) + ); + return 1.05; + case 'goal': + // a bright major stab that swells, with a rising run on top — a stadium moment + [60, 64, 67, 72].forEach((m) => + tone(ctx, dest, { freq: hz(m), type: 'sawtooth', t0, dur: 1.1, peak: 0.07, attack: 0.05 }) + ); + [79, 84, 88, 91].forEach((m, i) => + tone(ctx, dest, { freq: hz(m), type: 'triangle', t0: t0 + 0.05 + i * 0.07, dur: 0.4, peak: 0.16 }) + ); + noise(ctx, dest, { t0: t0 + 0.1, dur: 1.4, peak: 0.12, attack: 0.3, filter: 'bandpass', freq: 1600, q: 0.7, flutter: 9 }); + return 1.5; + case 'whistle': + // a referee's pea whistle: a high tone with a fast trill + tone(ctx, dest, { freq: 2900, type: 'sine', t0, dur: 0.55, peak: 0.2, attack: 0.02, vibrato: { rate: 32, depth: 160 } }); + return 0.6; + case 'cheer': + noise(ctx, dest, { t0, dur: 1.8, peak: 0.3, attack: 0.35, filter: 'bandpass', freq: 1200, to: 1700, q: 0.6, flutter: 7 }); + noise(ctx, dest, { t0: t0 + 0.1, dur: 1.6, peak: 0.14, attack: 0.4, filter: 'bandpass', freq: 2600, q: 1.2, flutter: 11 }); + return 1.9; + case 'step': + noise(ctx, dest, { t0, dur: 0.08, peak: 0.3, freq: 500, to: 180, attack: 0.002 }); + return 0.1; + case 'ring': + tone(ctx, dest, { freq: 1320, t0, dur: 1.0, peak: 0.2, attack: 0.003 }); + tone(ctx, dest, { freq: 1320 * 2.76, t0, dur: 0.5, peak: 0.07, attack: 0.003 }); + tone(ctx, dest, { freq: 1320 * 5.4, t0, dur: 0.25, peak: 0.04, attack: 0.003 }); + return 1.05; + case 'sparkle': { + // seven high pings on a fixed pattern (deterministic — no Math.random) + const notes = [96, 100, 103, 98, 105, 101, 108]; + notes.forEach((m, i) => tone(ctx, dest, { freq: hz(m), t0: t0 + i * 0.045, dur: 0.22, peak: 0.1, attack: 0.002 })); + return 0.55; + } + case 'hurt': + tone(ctx, dest, { freq: 320, to: 110, type: 'sawtooth', t0, dur: 0.32, peak: 0.2, attack: 0.004 }); + noise(ctx, dest, { t0, dur: 0.15, peak: 0.2, freq: 900, attack: 0.002 }); + return 0.36; + case 'portal': + tone(ctx, dest, { freq: 180, to: 1300, t0, dur: 0.9, peak: 0.18, attack: 0.3, vibrato: { rate: 9, depth: 40 } }); + tone(ctx, dest, { freq: 270, to: 1950, type: 'triangle', t0: t0 + 0.05, dur: 0.85, peak: 0.08, attack: 0.3 }); + noise(ctx, dest, { t0, dur: 0.9, peak: 0.08, attack: 0.4, filter: 'bandpass', freq: 800, to: 4000, q: 2 }); + return 0.95; + default: + return -1; + } +} + +/* ------------------------------------------------------------------ the player --- */ + +/** end times (context seconds) of the sounds still ringing @type {number[]} */ +const live = []; +const debug = { played: 0, dropped: 0, unknown: 0, last: '' }; + +/** + * Play a game sound, LOCAL to this device. Returns true when a sound started; false for + * an unknown name, a missing audio context, or the voice cap. + * @param {string} name @param {number[] | null} [position] world-space, spatialised + * @returns {boolean} + */ +export function playGameSound(name, position = null) { + if (!isGameSound(name)) { + debug.unknown++; + return false; + } + /** @type {AudioContext} */ + let ctx; + try { + ctx = ensureAudioContext(); + } catch { + return false; + } + if (ctx.state === 'suspended') ctx.resume().catch(() => {}); + const now = ctx.currentTime; + while (live.length && live[0] <= now) live.shift(); + if (live.length >= MAX_LIVE) { + debug.dropped++; + return false; + } + /** @type {AudioNode} */ + let dest = gameOut(); + if (Array.isArray(position) && position.length >= 3 && position.every((n) => Number.isFinite(n))) { + const panner = ctx.createPanner(); + panner.panningModel = 'HRTF'; + panner.distanceModel = 'inverse'; + panner.refDistance = 2; + if (panner.positionX) { + panner.positionX.value = position[0]; + panner.positionY.value = position[1]; + panner.positionZ.value = position[2]; + } else panner.setPosition(position[0], position[1], position[2]); + panner.connect(dest); + dest = panner; + } + const length = buildGameSound(ctx, dest, name, now + 0.005); + live.push(now + length); + live.sort((a, b) => a - b); + debug.played++; + debug.last = name; + return true; +} + +/** + * Render one sound OFFLINE and measure it (the suite's "does every name make a sound"). + * @param {string} name @param {number} [sampleRate] + * @returns {Promise<{name: string, seconds: number, rms: number, peak: number} | null>} + */ +export async function renderGameSound(name, sampleRate = 22050) { + const Offline = typeof window !== 'undefined' ? /** @type {any} */ (window).OfflineAudioContext : null; + if (!Offline) return null; + const probe = new Offline(1, 64, sampleRate); + const seconds = buildGameSound(probe, probe.destination, name, 0); + if (seconds < 0) return { name, seconds: -1, rms: 0, peak: 0 }; + const ctx = new Offline(1, Math.ceil((seconds + 0.1) * sampleRate), sampleRate); + buildGameSound(ctx, ctx.destination, name, 0); + const buffer = await ctx.startRendering(); + const data = buffer.getChannelData(0); + let sum = 0; + let peak = 0; + for (let i = 0; i < data.length; i++) { + sum += data[i] * data[i]; + peak = Math.max(peak, Math.abs(data[i])); + } + return { name, seconds, rms: Math.sqrt(sum / data.length), peak }; +} + +/** the live gain's value (null before the first sound) — the volume's measurable end */ +export function gameSoundGainValue() { + return gameGain ? gameGain.gain.value : null; +} + +/** counts for the suites @returns {{played: number, dropped: number, unknown: number, last: string, live: number}} */ +export function gameSfxDebug() { + return { ...debug, live: live.length }; +} + +// the two building blocks, shared with the music scheduler (gameMusic.js) so a game's +// music and its effects are made of the same parts +export { tone as sfxTone, noise as sfxNoise }; diff --git a/src/lib/moduleSDK.js b/src/lib/moduleSDK.js index dff5a4d6..b2c81d6d 100644 --- a/src/lib/moduleSDK.js +++ b/src/lib/moduleSDK.js @@ -51,6 +51,12 @@ export { moduleContentDebug } from './moduleContent'; import { safeStorage } from './safeStorage'; // 30 P4: api.storage — a LEAF (safeStorage only), shared with the Store Value flow node import { makeModuleStorage } from './gameStorage'; +// 30b (vr-play) C5: the game sound set and game music — LEAVES (svelte/store, audioEngine, +// safeStorage, sessionClock, sceneStore), so static edges close no cycle +import { isGameSound, playGameSound } from './gameSfx'; +import { playGameMusic, stopGameMusic, gameMusicState, MUSIC_PRESET_IDS } from './gameMusic'; +/** the ping chimes `api.playSound` still reaches (pingAudio's PING_SOUNDS ids) */ +const PING_NAMES = new Set(['ding', 'chime', 'pluck', 'bell']); /** modules already told they hit the storage cap this session (ONE toast each, never * one per write — a game saving every frame would otherwise bury the screen) */ @@ -1020,10 +1026,37 @@ function makeApi(moduleId, moduleName = moduleId) { flyTo(position, lookAt) { objectActionsRef?.flyTo(position, lookAt ?? position); }, - /** A spatial UI chime (the ping sounds). LOCAL — broadcast your own op if - * peers should hear it too. @param {string=} sound @param {number[]=} position */ + /** + * A sound, LOCAL to this device — broadcast your own op if peers should hear it. + * 30b: the GAME set (procedural, no assets, the "Game sounds" volume): 'click', + * 'pop', 'whoosh', 'success', 'fail', 'hit', 'kick', 'shoot', 'laser', + * 'explosion', 'coin', 'levelup', 'goal', 'whistle', 'cheer', 'step', 'ring', + * 'sparkle', 'hurt', 'portal'; plus the ping chimes 'ding' (the default), + * 'chime', 'pluck', 'bell'. An unknown name is a quiet no-op. `position` + * spatialises it. Returns whether a sound started. + * @param {string=} sound @param {number[]=} position @returns {boolean} + */ playSound(sound = 'ding', position = undefined) { - pingAudioRef?.playPing(sound, position ?? null); + const name = String(sound ?? 'ding'); + if (isGameSound(name)) return playGameSound(name, position ?? null); + if (!PING_NAMES.has(name)) return false; + pingAudioRef?.playPing(name, position ?? null); + return !!pingAudioRef; + }, + /** + * 30b: game MUSIC — procedural loops, LOCAL to this device, tempo-synced to the + * session clock (two peers on one preset hear the same bar), under the effects + * and on the "Music" volume. Plays only in Interact/Play (a call from Edit returns + * false) and stops by itself when the player leaves the game. + * Presets: 'arcade', 'ambient', 'dungeon', 'stadium', 'space', 'puzzle', 'studio'. + */ + music: { + /** @param {string} preset @param {{volume?: number}=} options 0..1 @returns {boolean} */ + play: (preset, options = {}) => playGameMusic(preset, options ?? {}), + stop: () => stopGameMusic(), + /** the preset playing now, or null @returns {string | null} */ + current: () => get(gameMusicState)?.preset ?? null, + presets: () => [...MUSIC_PRESET_IDS] }, /** Park the editor camera behind an object and follow it (the car's chase * cam) — LOCAL, no selection, no undo. @param {string} uuid */ diff --git a/tests/e2e/game-audio.test.cjs b/tests/e2e/game-audio.test.cjs new file mode 100644 index 00000000..67d29852 --- /dev/null +++ b/tests/e2e/game-audio.test.cjs @@ -0,0 +1,190 @@ +// 30b (vr-play) P0 — C5 SOUND AND MUSIC. The user, from a Quest: "There is also no score, +// no sounds, fix this" / "Make for each game improvements, add sounds, music". +// +// 1 every name in the game sound set RENDERS a real sound (offline, measured) · 2 the SDK +// routes playSound: a game name plays, a ping name still plays, an unknown name is a quiet +// no-op · 3 a game sound reaches the speakers (the destination tap, measured live) · 4 every +// music preset renders a real bar · 5 music is REFUSED in Edit · 6 music plays in Interact, +// on the SESSION step grid, audibly · 7 leaving Interact stops it and the output goes +// silent · 8 the same in desktop Play · 9 the two volumes are LOCAL, persisted, and reach +// the gains · 10 the Settings rows drive the stores. +const h = require('./helpers.cjs'); + +h.run(async () => { + const browser = await h.launch({ args: h.AUDIO_ARGS }); + const A = await h.setupPage(browser, 'A', { audio: true }); + const page = A.page; + + await page.evaluate(async () => { + const s = window.__stores; + await s.moduleSDK.initModules([ + { id: 'audio30b', name: 'Audio test', version: '1.0.0', description: 'the 30b sound seams', register(api) { window.__api = api; } } + ]); + }); + + console.log('\n=== 1. every game sound renders a real sound ==='); + const renders = await page.evaluate(async () => { + const k = window.__stores.gameKit.gameSfx; + const out = []; + for (const name of k.GAME_SOUNDS) out.push(await k.renderGameSound(name)); + return out; + }); + h.check(renders.length === 20, '1.1 the set holds the 20 contract names (' + renders.length + ')'); + const quiet = renders.filter((r) => !r || r.rms < 0.004 || r.peak < 0.02); + h.check(quiet.length === 0, '1.2 every name renders energy (quiet: ' + JSON.stringify(quiet.map((r) => r && r.name)) + ')'); + const distinct = new Set(renders.map((r) => r && Math.round(r.rms * 1e5))); + h.check(distinct.size >= 18, '1.3 the sounds are distinct from each other (' + distinct.size + ' distinct levels)'); + const unknownRender = await page.evaluate(() => window.__stores.gameKit.gameSfx.renderGameSound('nope')); + h.check(unknownRender && unknownRender.seconds === -1 && unknownRender.rms === 0, '1.4 an unknown name builds nothing'); + + console.log('\n=== 2. the SDK routes playSound ==='); + const routed = await page.evaluate(async () => { + const k = window.__stores.gameKit.gameSfx; + const before = k.gameSfxDebug(); + const coin = window.__api.playSound('coin', [0, 1, -2]); + const mid = k.gameSfxDebug(); + const unknown = window.__api.playSound('definitely-not-a-sound'); + const after = k.gameSfxDebug(); + // the ping chimes still play through their own module (primed at boot) + await new Promise((r) => setTimeout(r, 300)); + const ding = window.__api.playSound('ding'); + return { coin, unknown, ding, played: mid.played - before.played, playedAfter: after.played - mid.played, last: mid.last }; + }); + h.check(routed.coin === true && routed.played === 1 && routed.last === 'coin', '2.1 playSound("coin") plays the game coin (' + JSON.stringify(routed) + ')'); + h.check(routed.unknown === false && routed.playedAfter === 0, '2.2 an unknown name is a quiet no-op (it used to fall through to the ding)'); + h.check(routed.ding === true, '2.3 a ping name still plays'); + + console.log('\n=== 3. a game sound reaches the speakers ==='); + await page.waitForTimeout(1200); // let 2's sounds ring out + await page.evaluate(() => window.__api.playSound('explosion')); + const live = await h.audioMetrics(A, 500); + h.check(!live.silent && live.peak > 0.01, '3.1 the explosion is audible at the destination (peak ' + live.peak.toFixed(4) + ')'); + + console.log('\n=== 4. every music preset renders a real bar ==='); + const bars = await page.evaluate(async () => { + const m = window.__stores.gameKit.gameMusic; + const out = []; + for (const id of m.MUSIC_PRESET_IDS) out.push(await m.renderGameMusic(id, 1)); + return out; + }); + h.check(bars.length === 7, '4.1 seven presets (' + bars.map((b) => b && b.preset).join(',') + ')'); + const flat = bars.filter((b) => !b || b.rms < 0.003 || b.notes < 4); + h.check(flat.length === 0, '4.2 every preset plays notes with energy (' + JSON.stringify(bars.map((b) => b && [b.preset, +b.rms.toFixed(4), b.notes])) + ')'); + + await page.waitForTimeout(1500); // the explosion's tail + console.log('\n=== 5. music is refused in Edit ==='); + const refused = await page.evaluate(() => { + const s = window.__stores; + s.objectActions.setEditorMode('edit'); + return { ok: window.__api.music.play('arcade'), current: window.__api.music.current() }; + }); + h.check(refused.ok === false && refused.current === null, '5.1 music.play from the editor returns false and plays nothing'); + const editQuiet = await h.audioMetrics(A, 400); + h.check(editQuiet.silent, '5.2 the output is silent (peak ' + editQuiet.peak.toFixed(4) + ')'); + + console.log('\n=== 6. music plays in Interact, on the session step grid ==='); + const started = await page.evaluate(() => { + const s = window.__stores; + s.objectActions.setEditorMode('interact'); + return { ok: window.__api.music.play('arcade', { volume: 0.8 }), current: window.__api.music.current(), unknown: window.__api.music.play('polka') }; + }); + h.check(started.ok === true && started.current === 'arcade', '6.1 music.play("arcade") starts in Interact'); + h.check(started.unknown === false && (await page.evaluate(() => window.__api.music.current())) === 'arcade', '6.2 an unknown preset is refused and the current one keeps playing'); + await page.waitForTimeout(900); + const grid = await page.evaluate(() => { + const k = window.__stores.gameKit; + const d = k.gameMusic.gameMusicDebug(); + const stepMs = k.gameMusicPresets.stepSeconds(k.gameMusicPresets.musicPreset('arcade')) * 1000; + const now = window.__stores.connectionState?.sessionNow?.() ?? null; + return { d, stepMs, now }; + }); + h.check(grid.d.steps >= 5 && grid.d.notes > 0, '6.3 the scheduler scheduled steps and notes (' + JSON.stringify(grid.d) + ')'); + const audible = await h.audioMetrics(A, 700); + h.check(!audible.silent && audible.peak > 0.005, '6.4 the music is audible (peak ' + audible.peak.toFixed(4) + ')'); + const synced = await page.evaluate(async () => { + const k = window.__stores.gameKit; + const { sessionNow } = await import('/src/lib/sessionClock.js'); + const d = k.gameMusic.gameMusicDebug(); + const stepMs = k.gameMusicPresets.stepSeconds(k.gameMusicPresets.musicPreset('arcade')) * 1000; + // the last scheduled step sits inside the lookahead window of the SESSION clock + const lead = d.lastStep * stepMs - sessionNow(); + return { lead, stepMs }; + }); + h.check(synced.lead > -synced.stepMs && synced.lead < 250, '6.5 steps are numbered off the session clock (last step leads now by ' + synced.lead.toFixed(1) + ' ms)'); + + console.log('\n=== 7. leaving Interact stops the music ==='); + const stopped = await page.evaluate(() => { + window.__stores.objectActions.setEditorMode('edit'); + return { current: window.__api.music.current(), gain: window.__stores.gameKit.gameMusic.gameMusicGainValue() }; + }); + h.check(stopped.current === null && stopped.gain === null, '7.1 back to Edit: nothing is playing'); + await page.waitForTimeout(700); + const afterStop = await h.audioMetrics(A, 500); + h.check(afterStop.silent, '7.2 the output went silent (peak ' + afterStop.peak.toFixed(4) + ')'); + + console.log('\n=== 8. the same in desktop Play ==='); + const playPath = await page.evaluate(async () => { + const s = window.__stores; + s.isLocked.set(true); + const ok = window.__api.music.play('dungeon'); + const during = window.__api.music.current(); + s.isLocked.set(false); + const after = window.__api.music.current(); + await new Promise((r) => setTimeout(r, 50)); + return { ok, during, after }; + }); + h.check(playPath.ok && playPath.during === 'dungeon', '8.1 music plays in Play'); + h.check(playPath.after === null, '8.2 leaving Play stops it'); + + console.log('\n=== 9. the volumes are local, persisted, and reach the gains ==='); + const vols = await page.evaluate(async () => { + const s = window.__stores; + const k = s.gameKit; + k.gameSfx.gameSoundVolume.set(0.25); + window.__api.playSound('click'); + const sfxGain = k.gameSfx.gameSoundGainValue(); + s.objectActions.setEditorMode('interact'); + window.__api.music.play('puzzle', { volume: 0.5 }); + k.gameMusic.gameMusicVolume.set(0.4); + const musicGain = k.gameMusic.gameMusicGainValue(); + window.__api.music.stop(); + s.objectActions.setEditorMode('edit'); + return { + sfxGain, + musicGain, + sfxKey: localStorage.getItem('game:soundVolume'), + musicKey: localStorage.getItem('game:musicVolume') + }; + }); + h.check(Math.abs(vols.sfxGain - 0.25) < 1e-6 && vols.sfxKey === '0.25', '9.1 the Game sounds volume sets the gain and persists (' + JSON.stringify(vols) + ')'); + h.check(Math.abs(vols.musicGain - 0.55 * 0.4 * 0.5) < 1e-6 && vols.musicKey === '0.4', '9.2 the Music volume (x the call volume, under the effects) sets the gain and persists'); + + console.log('\n=== 10. the Settings rows drive the stores ==='); + await page.evaluate(() => window.__stores.settingsOpen.set(true)); + await page.waitForTimeout(600); + // the Interface section holds the rows; open it if it is collapsed + const hasRow = await page.evaluate(() => !!document.querySelector('#setting-game-sound-volume')); + if (!hasRow) await page.getByText('Interface', { exact: true }).first().click().catch(() => {}); + await page.waitForSelector('#setting-game-sound-volume', { timeout: 5000 }).catch(() => {}); + const rows = await page.evaluate(() => { + const s = window.__stores.gameKit; + const set = (id, v) => { + const el = document.querySelector(id); + if (!el) return false; + el.value = String(v); + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + return true; + }; + const a = set('#setting-game-sound-volume', 0.6); + const b = set('#setting-game-music-volume', 0.35); + let sv, mv; + s.gameSfx.gameSoundVolume.subscribe((v) => (sv = v))(); + s.gameMusic.gameMusicVolume.subscribe((v) => (mv = v))(); + return { a, b, sv, mv }; + }); + h.check(rows.a && rows.b, '10.1 both rows render in Settings'); + h.check(Math.abs(rows.sv - 0.6) < 1e-6 && Math.abs(rows.mv - 0.35) < 1e-6, '10.2 dragging them sets the stores (' + JSON.stringify(rows) + ')'); + + await h.finish(browser); +}); diff --git a/tests/unit/gameMusicPresets.test.js b/tests/unit/gameMusicPresets.test.js new file mode 100644 index 00000000..4ba951bd --- /dev/null +++ b/tests/unit/gameMusicPresets.test.js @@ -0,0 +1,62 @@ +// 30b (vr-play) C5: the game-music presets as data, with no browser — every preset the +// contract names exists, every lane is a real 16-step bar, the triads are built inside the +// mode, and a step is a pure function of (preset, step), which is what makes two peers on +// one preset hear the same bar. +import { describe, it, expect } from 'vitest'; +import { MUSIC_PRESETS, MUSIC_PRESET_IDS, musicPreset, stepEvents, triadOn, stepSeconds } from '../../src/lib/gameMusicPresets.js'; + +describe('game music presets', () => { + it('ships exactly the contract presets', () => { + expect([...MUSIC_PRESET_IDS].sort()).toEqual(['ambient', 'arcade', 'dungeon', 'puzzle', 'space', 'stadium', 'studio']); + expect(musicPreset('nope')).toBeNull(); + }); + + it('every lane is a 16-step bar and the tempo is sane', () => { + for (const p of MUSIC_PRESETS) { + expect(p.bass.length).toBe(16); + for (const lane of Object.values(p.drums)) if (lane) expect(lane.length).toBe(16); + if (p.arp) expect(p.arp.lane.length).toBe(16); + expect(p.bpm).toBeGreaterThanOrEqual(60); + expect(p.bpm).toBeLessThanOrEqual(160); + expect(stepSeconds(p)).toBeCloseTo(60 / p.bpm / 4, 9); + } + }); + + it('triads stack thirds inside the mode (minor i is minor, major I is major)', () => { + const arcade = /** @type {any} */ (musicPreset('arcade')); // C major + expect(triadOn(arcade, 0)).toEqual([60, 64, 67]); + expect(triadOn(arcade, 4)).toEqual([67, 71, 74]); // G major + const studio = /** @type {any} */ (musicPreset('studio')); // G minor + const [r, t, f] = triadOn(studio, 0); + expect(t - r).toBe(3); + expect(f - r).toBe(7); + }); + + it('every preset plays notes in every bar of its loop', () => { + for (const p of MUSIC_PRESETS) { + for (let bar = 0; bar < p.progression.length; bar++) { + let notes = 0; + for (let i = 0; i < 16; i++) notes += stepEvents(p, bar * 16 + i).length; + expect(notes, p.id + ' bar ' + bar).toBeGreaterThan(0); + } + } + }); + + it('a step is a pure function of (preset, step) and loops with the progression', () => { + const space = /** @type {any} */ (musicPreset('space')); + const loop = space.progression.length * 16; + for (const step of [0, 5, 17, 40, 63]) { + expect(stepEvents(space, step)).toEqual(stepEvents(space, step)); + expect(stepEvents(space, step)).toEqual(stepEvents(space, step + loop)); + } + // the chord changes with the bar: the bass root of bar 0 differs from bar 1 + const root = (/** @type {number} */ step) => stepEvents(space, step).find((e) => e.kind === 'bass')?.midi; + expect(root(0)).not.toEqual(root(16)); + }); + + it('the pad holds the bar triad on its downbeat only', () => { + const ambient = /** @type {any} */ (musicPreset('ambient')); + expect(stepEvents(ambient, 0).some((e) => e.kind === 'pad')).toBe(true); + expect(stepEvents(ambient, 1).some((e) => e.kind === 'pad')).toBe(false); + }); +}); From 552d09020ec88830c89dcf3282635cd3b8f622d3 Mon Sep 17 00:00:00 2001 From: AlexZ005 Date: Wed, 23 Sep 2026 14:39:58 +0300 Subject: [PATCH 2/8] [feat] 30b P1: haptics (C4) - silent in Edit, api.hapticPattern, tap/bump/hit/knock defaults in Interact/Play - vrControls.hapticPulse is now a NO-OP in Edit (gameFeelActive), the one funnel every core pulse and api.haptic go through - "the vibration should be only interactive mode, not in edit mode" (the user, Quest 3). A 64-pulse debug ring + a suppressed count (hapticDebug) is the suites' view, since no headset is attached headless. - hapticPatterns.js (pure, vitest): tap, bump, hit, success, fail, rumble, heartbeat as timed, never-overlapping pulses (a new actuator pulse REPLACES the running one), a scale, and knockHapticScale. vrControls.hapticPattern plays one through setTimeout, each pulse re-gated, so leaving the game mid-rumble stops it. - api.hapticPattern(name, hand?) -> boolean (false in Edit / unknown name). - defaults: a VR grip GRAB lands with 'hit'; a KNOCK plays 'hit' sized by its closing speed (hapticKnock, the knock seam Scene hands knock.js); vrGameInput.js (new, plugs in through vrControls' frame + trigger hook registries, started from Scene) taps a hand when its laser ENTERS something clickable and bumps it when the trigger lands there. "Clickable" is one resolver for tap, bump and the P4 sweep: module interactive-group content, a part of a module device (userData.device - the piano key, not the piano), an On Click target, or userData.clickable === true. - flowRuntime: objectHasOnClick(uuid) (fireObjectClick's own two rules), and fireObjectClick returns how many nodes it reached (additive). Suite game-haptics (new, a stubbed XR session with recording actuators): 18/18. vitest hapticPatterns 5/5 (all vitest 238 green). Counterfactuals (four guards broken in one run): - the Edit gate removed -> 1.1, 1.2, 3.4 red (3 pulses in Edit; rumble played 4 of 4 after leaving) - the hover edge removed (tap every frame) -> 4.2 red (2 extra taps on one target) - every object treated as clickable -> 4.3, 5.2 red - the knock scale ignored -> 6.2 red (0.850 -> 0.850) Held: hud-actions 65/0, game-editor-flow 41/0, sdk-game-seams 78/0, module-toolbox 46/0, editor-modes 45/0. svelte-check 334/47 (base 335: the JSDoc on hapticPulse's forEach removed a pre-existing implicit-any; no new message). Build green. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/components/Scene.svelte | 9 +- src/lib/flowRuntime.js | 15 ++- src/lib/gameKit.js | 2 + src/lib/hapticPatterns.js | 78 ++++++++++++ src/lib/moduleSDK.js | 12 ++ src/lib/vrControls.js | 55 ++++++++- src/lib/vrGameInput.js | 195 ++++++++++++++++++++++++++++++ tests/e2e/game-haptics.test.cjs | 173 ++++++++++++++++++++++++++ tests/unit/hapticPatterns.test.js | 45 +++++++ 9 files changed, 579 insertions(+), 5 deletions(-) create mode 100644 src/lib/hapticPatterns.js create mode 100644 src/lib/vrGameInput.js create mode 100644 tests/e2e/game-haptics.test.cjs create mode 100644 tests/unit/hapticPatterns.test.js diff --git a/src/components/Scene.svelte b/src/components/Scene.svelte index dde54080..10a1cfa4 100644 --- a/src/components/Scene.svelte +++ b/src/components/Scene.svelte @@ -54,7 +54,9 @@ // the annotation is TS syntax — a JSDoc @type cast is ignored here (the documented trap). let knifeFrom: number[] | null = null; import { peerScenes } from '$lib/peerScenes'; - import { initVRControls, updateVRControls, raycastMenu, raycastPanel, raycastPalette, raycastProps, raycastPrefabs, raycastKeyboard, raycastChat, raycastEdit, raycastSnap, raycastSettings, raycastApprove, placePrefabGhost, vrFaceTrigger, vrVertexTrigger, vrVertexGrabStart, vrVertexGrabEnd, beginStretchSliderDrag, endStretchSliderDrag, executeVRMenuAction, resetWorldRig, onInputSourcesChange, worldToContentPose, boxSelectStart, boxSelectEnd, boxSelectActive, applyVRFrameRate, shouldSendHands, onHandPinchStart, onHandPinchEnd, pinchMenuToggledAt, firePingIfArmed, vrModuleTriggerStart, vrModuleTriggerEnd, vrModuleSelectSwallowed, handSnapshot, vrGrabbedUuid, hapticPulse } from '$lib/vrControls'; + import { initVRControls, updateVRControls, raycastMenu, raycastPanel, raycastPalette, raycastProps, raycastPrefabs, raycastKeyboard, raycastChat, raycastEdit, raycastSnap, raycastSettings, raycastApprove, placePrefabGhost, vrFaceTrigger, vrVertexTrigger, vrVertexGrabStart, vrVertexGrabEnd, beginStretchSliderDrag, endStretchSliderDrag, executeVRMenuAction, resetWorldRig, onInputSourcesChange, worldToContentPose, boxSelectStart, boxSelectEnd, boxSelectActive, applyVRFrameRate, shouldSendHands, onHandPinchStart, onHandPinchEnd, pinchMenuToggledAt, firePingIfArmed, vrModuleTriggerStart, vrModuleTriggerEnd, vrModuleSelectSwallowed, handSnapshot, vrGrabbedUuid, hapticKnock } from '$lib/vrControls'; + // 30b (vr-play): the game in your hands — hover/press haptics (P1), the sweep (P4) + import { startVrGameInput, stopVrGameInput } from '$lib/vrGameInput'; import { vrKeyboardTarget } from '$lib/vrKeyboard'; import { measureMode, measureClick } from '$lib/measure'; import { pinsGroup, openAnnotation, showNotePins } from '$lib/annotationsHandler'; @@ -1419,7 +1421,9 @@ // this seam rather than an import (knock.js stays off vrControls' 3500 lines), // and the two "what am I holding" reads keep a probe off its own carried object. // A2: the hand that hit gets a buzz — LOCAL, the same seam shape as the hand poses - startKnock({ hands: handSnapshot, heldUuids: () => [carriedUuid(), vrGrabbedUuid()], haptic: hapticPulse }); + startKnock({ hands: handSnapshot, heldUuids: () => [carriedUuid(), vrGrabbedUuid()], haptic: hapticKnock }); + // 30b: game feel in VR (a frame hook + a trigger hook through vrControls' registries) + startVrGameInput(); xrControllers.forEach((controller) => { controller.addEventListener('select', onXRSelect); @@ -1434,6 +1438,7 @@ offKeyProbe(); // 30 P1 stopPlayInteract(); // 21-B B3 (releases any carried body with zero velocity) stopKnock(); // 24-A A1 + stopVrGameInput(); // 30b element.removeEventListener('pointerdown', onPointerDown); element.removeEventListener('contextmenu', onContextMenu); element.removeEventListener('webglcontextlost', onContextLost); diff --git a/src/lib/flowRuntime.js b/src/lib/flowRuntime.js index 8f5db677..b2719782 100644 --- a/src/lib/flowRuntime.js +++ b/src/lib/flowRuntime.js @@ -2752,14 +2752,27 @@ export function fireModuleTrigger(type, match, opts) { } export function fireObjectClick(uuid) { + let fired = 0; // 30b: how many On Click nodes this reached (additive return) nodes.forEach((node) => { if (node.type !== 'onclick') return; // H1: an unwired OnClick inside the clicked object's own graph also fires - if (reachesObjectSelector(node.id, uuid) || implicitOwnerOf(node) === uuid) + if (reachesObjectSelector(node.id, uuid) || implicitOwnerOf(node) === uuid) { // 21-G4: a perPlayer On Click keeps its pulse LOCAL — that one bit is the // whole per-player collectible (see replicatesPulse) applyNodeTrigger(node.id, syncedNow(), replicatesPulse(node)); + fired++; + } }); + return fired; +} + +/** 30b (vr-play): would a click on `uuid` reach an On Click node? The same two rules + * fireObjectClick fires by — the VR hover tap and the sweep ask it before touching + * anything. @param {string} uuid @returns {boolean} */ +export function objectHasOnClick(uuid) { + return nodes.some( + (node) => node.type === 'onclick' && (reachesObjectSelector(node.id, uuid) || implicitOwnerOf(node) === uuid) + ); } /** diff --git a/src/lib/gameKit.js b/src/lib/gameKit.js index a3e6c5ce..b2524374 100644 --- a/src/lib/gameKit.js +++ b/src/lib/gameKit.js @@ -5,3 +5,5 @@ export * as gameFeel from './gameFeel'; export * as gameSfx from './gameSfx'; export * as gameMusic from './gameMusic'; export * as gameMusicPresets from './gameMusicPresets'; +export * as hapticPatterns from './hapticPatterns'; +export * as vrGameInput from './vrGameInput'; diff --git a/src/lib/hapticPatterns.js b/src/lib/hapticPatterns.js new file mode 100644 index 00000000..f39b99e8 --- /dev/null +++ b/src/lib/hapticPatterns.js @@ -0,0 +1,78 @@ +// 30b (vr-play) C4 — HAPTIC PATTERNS as data. `api.hapticPattern(name, hand)` plays one +// of these on the Quest's controller actuators; core fires the defaults in Interact/Play +// (hover-enter `tap`, a press `bump`, a grab `hit`, a knock scaled by its impulse). +// +// A pattern is a list of PULSES, each `{at, intensity, ms}` with `at` in milliseconds +// from the start. A GamepadHapticActuator plays one pulse at a time and a new pulse +// REPLACES the one running, so a pattern is timed pulses, never overlapping ones — the +// schedule below is what vrControls plays through setTimeout. +// +// Imports NOTHING: the shapes are tested with no browser and no headset. How they FEEL is +// the on-device check (owed) — the numbers are a first pass at "make it cool". + +/** @typedef {{at: number, intensity: number, ms: number}} HapticPulse */ + +/** @type {Record} */ +export const HAPTIC_PATTERNS = { + // a fingertip on a button: the lightest thing that is still felt + tap: [{ at: 0, intensity: 0.18, ms: 12 }], + // a press landing: short and firm + bump: [{ at: 0, intensity: 0.45, ms: 28 }], + // a grab or a solid contact: a hard edge, then a short settle + hit: [ + { at: 0, intensity: 0.85, ms: 35 }, + { at: 55, intensity: 0.3, ms: 25 } + ], + // a win: three rising pulses + success: [ + { at: 0, intensity: 0.3, ms: 40 }, + { at: 90, intensity: 0.5, ms: 40 }, + { at: 180, intensity: 0.8, ms: 70 } + ], + // a miss: two heavy, flat pulses + fail: [ + { at: 0, intensity: 0.7, ms: 90 }, + { at: 160, intensity: 0.7, ms: 140 } + ], + // an engine / an explosion: a long grinding buzz, fading + rumble: [ + { at: 0, intensity: 1, ms: 120 }, + { at: 120, intensity: 0.75, ms: 120 }, + { at: 240, intensity: 0.5, ms: 120 }, + { at: 360, intensity: 0.28, ms: 140 } + ], + // lub-dub + heartbeat: [ + { at: 0, intensity: 0.65, ms: 45 }, + { at: 150, intensity: 0.4, ms: 55 } + ] +}; + +/** the names `api.hapticPattern` accepts */ +export const HAPTIC_PATTERN_NAMES = Object.keys(HAPTIC_PATTERNS); + +/** + * A pattern's pulses, clamped to what an actuator takes (0..1, 1..1000 ms). Unknown -> []. + * `scale` multiplies every intensity — how a knock's impulse sizes a `hit`. + * @param {string} name @param {number} [scale] @returns {HapticPulse[]} + */ +export function hapticSchedule(name, scale = 1) { + const pattern = HAPTIC_PATTERNS[String(name)]; + if (!pattern) return []; + const k = Number.isFinite(scale) ? Math.max(0, scale) : 1; + return pattern.map((p) => ({ + at: Math.max(0, p.at), + intensity: Math.min(1, Math.max(0, p.intensity * k)), + ms: Math.min(1000, Math.max(1, p.ms)) + })); +} + +/** + * The strength a KNOCK buzzes with: a light brush is a tap, a real swing a full hit. + * `speed` is the closing speed in m/s (knockMath's `approach`). + * @param {number} speed @returns {number} 0.2..1 + */ +export function knockHapticScale(speed) { + const s = Number.isFinite(speed) ? Math.max(0, speed) : 0; + return Math.min(1, 0.2 + s / 6); +} diff --git a/src/lib/moduleSDK.js b/src/lib/moduleSDK.js index b2c81d6d..b6c120b3 100644 --- a/src/lib/moduleSDK.js +++ b/src/lib/moduleSDK.js @@ -902,8 +902,20 @@ function makeApi(moduleId, moduleName = moduleId) { * @param {'left'|'right'=} hand */ haptic(intensity = 0.5, durationMs = 50, hand = undefined) { + // 30b: silent in EDIT mode (core's own gate) — vibration is for playing vrControlsRef?.hapticPulse?.(intensity, durationMs, hand); }, + /** + * 30b: a named haptic PATTERN — 'tap' (hover), 'bump' (a press), 'hit' (a grab, + * a contact), 'success', 'fail', 'rumble' (an engine, an explosion), 'heartbeat'. + * On one hand ('left'|'right') or both. LOCAL, Interact/Play only (false in Edit, + * on desktop nothing buzzes). Core already plays tap / bump / hit / knocks for + * you; use this for the game's own moments. + * @param {string} name @param {'left'|'right'=} hand @returns {boolean} + */ + hapticPattern(name, hand = undefined) { + return vrControlsRef?.hapticPattern?.(String(name), hand) ?? false; + }, /** * 24-A A2: every KNOCK this peer sees — its own hand's, and every peer's as the * `hit` message is applied — as `{uuid, by, at, speed, point, linvel, angvel, diff --git a/src/lib/vrControls.js b/src/lib/vrControls.js index eef68025..2e0a095e 100644 --- a/src/lib/vrControls.js +++ b/src/lib/vrControls.js @@ -42,6 +42,9 @@ import { peerHandStyle, pokeScene } from '../stores/sceneStore'; import { activeRing, findMenuEntry, ringEntries, sectorFromStick, pushRing, popRing, resetRings, hubEntry } from './vrRadialMenu'; import { paletteColorAt, barValueAt } from './vrPalette'; +// 30b (vr-play) C4: the ONE game-feel predicate (a leaf) + the pattern shapes (pure) +import { gameFeelActive } from './gameFeel'; +import { hapticSchedule, knockHapticScale } from './hapticPatterns'; import { recordMaterialChange, setMaterialParam } from './materialsHandler'; import { prefabs, instantiatePrefab } from './prefabs'; import { @@ -926,24 +929,71 @@ export function initVRControls(r) { renderer = r; } +/** 30b: what the actuators were asked for (a ring, newest last) and how many pulses the + * Edit-mode gate swallowed — the suites' view, since no headset is attached headless */ +const hapticRing = /** @type {{intensity: number, ms: number, hand: string | null, at: number}[]} */ ([]); +let hapticSuppressed = 0; + /** * Buzz the VR controllers if the session's gamepads support it (no-op on * desktop). Used by modules for press feedback. Optional `hand` targets one * controller — matched by each inputSource's OWN handedness (never a raw slot * index, which diverges from the controller order after a hands<->controllers * swap — 194/210; axesForSlot resolves the same way). + * + * 30b (C4): a NO-OP IN EDIT MODE — "The vibration should be only interactive mode, not + * in edit mode" (the user, from a Quest). Every core pulse and a module's api.haptic + * funnel through here, so this one gate covers all of them. * @param {number} intensity 0..1 @param {number} durationMs * @param {'left'|'right'=} hand omit to pulse both */ export function hapticPulse(intensity = 0.5, durationMs = 50, hand = undefined) { + if (!gameFeelActive()) { + hapticSuppressed++; + return; + } const session = renderer?.xr?.getSession?.(); - session?.inputSources?.forEach((source) => { + if (!session) return; + hapticRing.push({ intensity, ms: durationMs, hand: hand ?? null, at: performance.now() }); + if (hapticRing.length > 64) hapticRing.shift(); + session.inputSources?.forEach((/** @type {any} */ source) => { if (hand && source.handedness !== hand) return; const actuator = source.gamepad?.hapticActuators?.[0]; actuator?.pulse?.(intensity, durationMs); }); } +/** + * 30b (C4): play a named PATTERN ('tap' 'bump' 'hit' 'success' 'fail' 'rumble' + * 'heartbeat' — hapticPatterns.js) as timed pulses. `scale` sizes every pulse (a knock's + * impulse). Returns false for an unknown name or outside Interact/Play; each pulse is + * gated again as it fires, so leaving the game mid-pattern stops it. + * @param {string} name @param {'left'|'right'=} hand @param {number=} scale + * @returns {boolean} + */ +export function hapticPattern(name, hand = undefined, scale = 1) { + const pulses = hapticSchedule(name, scale); + if (!pulses.length || !gameFeelActive()) return false; + for (const pulse of pulses) { + if (pulse.at <= 0) hapticPulse(pulse.intensity, pulse.ms, hand); + else setTimeout(() => hapticPulse(pulse.intensity, pulse.ms, hand), pulse.at); + } + return true; +} + +/** 30b: the knock's haptic seam (Scene hands this to knock.js) — a `hit` sized by the + * knock's own strength, on the hand that hit. `intensity` arrives as knock.js computes + * it (0.2 + speed/10); read back as the speed it encodes. + * @param {number} intensity @param {number} _ms @param {'left'|'right'} hand */ +export function hapticKnock(intensity, _ms, hand) { + hapticPattern('hit', hand, knockHapticScale(Math.max(0, (Number(intensity) - 0.2) * 10))); +} + +/** the suites' view of the actuators @returns {{pulses: any[], suppressed: number}} */ +export function hapticDebug() { + return { pulses: hapticRing.map((p) => ({ ...p })), suppressed: hapticSuppressed }; +} + /** 194: resolve a controller slot by HANDEDNESS. three's getController(i) is a * persistent object; Scene stamps controller.userData.handedness from each * 'connected' event, so this survives a hands<->controllers reorder (the raw @@ -2291,7 +2341,8 @@ function onSqueezeStart(index) { before: transformStateOf(object) }; vrGrabbedHand.set(renderer.xr.getController(index)?.userData?.handedness ?? null); - hapticPulse(0.25, 30); + // 30b (C4): a grab lands with a `hit` (a gated no-op in Edit, like every pulse) + hapticPattern('hit', renderer.xr.getController(index)?.userData?.handedness ?? undefined); selectObject(object.uuid); // locks it for peers, updates selection state } diff --git a/src/lib/vrGameInput.js b/src/lib/vrGameInput.js new file mode 100644 index 00000000..0c858e3e --- /dev/null +++ b/src/lib/vrGameInput.js @@ -0,0 +1,195 @@ +// 30b (vr-play) — THE GAME IN YOUR HANDS, the VR input half: what a controller is pointing +// at or touching in a GAME, and what that feels like. P1 (C4) is the haptic defaults — +// a `tap` when the laser enters something clickable, a `bump` when the trigger lands on +// it; P4 (C3) grows the hold-and-sweep on the same resolver. +// +// "Clickable" is decided HERE, once, for the tap, the bump and the sweep alike: +// · anything under a module's scene-root INTERACTIVE group (a board, a dungeon door); +// · a part of a module DEVICE (`userData.device` on an ancestor — the music lab's piano +// keys, drum pads, transport buttons: every one is its own control); +// · an object an On Click node targets (flowRuntime's own rule), or one a module marked +// `userData.clickable = true`. +// The ENTRY KEY is what "once per entry" counts: the exact mesh for module content and +// device parts (a key is not its piano), the top-level object otherwise (an object with an +// On Click is one thing to press however many meshes it is built from). +// +// Everything is LOCAL and gated on gameFeelActive(): the editor never buzzes (the user's +// rule), and nothing here sends anything — a click that lands replicates through the paths +// it always did (module handlers, the On Click trigger). +// +// Plugs into vrControls through its generic registries (registerVRFrameHook, +// registerVRTriggerHooks) — the vrSleeve precedent — so vrControls never imports it. +// Started from Scene.svelte (`startVrGameInput`), which already imports everything here. +import * as THREE from 'three'; +import { get } from 'svelte/store'; +import { objectsGroup, globalScene, globalRenderer } from '../stores/sceneStore'; +import { gameFeelActive } from './gameFeel'; +import { sceneHits } from './scenePick'; +import { pickStack, primaryIndex } from './selectThrough'; +import { topLevelObjectOf } from './objectActions'; +import { objectHasOnClick } from './flowRuntime'; +import { moduleInteractiveGroups } from './moduleSDK'; +import { registerVRFrameHook, registerVRTriggerHooks, hapticPattern } from './vrControls'; + +const _mat = new THREE.Matrix4(); + +/** + * @typedef {{kind: 'object', key: string, mesh: any, top: any | null, point: any, distance: number, group: string | null}} ClickTarget + */ + +/** the scene-root interactive group a mesh sits under, or null @param {any} mesh */ +function interactiveGroupOf(mesh) { + const scene = get(globalScene); + for (let node = mesh; node && node !== scene; node = node.parent) { + if (node.parent === scene && moduleInteractiveGroups.includes(node.name)) return node.name; + } + return null; +} + +/** is there a module device on the path from the mesh up to `top`? @param {any} mesh @param {any} top */ +function deviceBetween(mesh, top) { + for (let node = mesh; node; node = node.parent) { + if (node.userData?.device) return true; + if (node === top) break; + } + return false; +} + +/** + * Resolve a raycast/overlap HIT to the thing a press would reach — or null when nothing + * clickable is there. Pure over the scene; exported for the suites. + * @param {any} hit a THREE intersection ({object, point, distance}) + * @returns {ClickTarget | null} + */ +export function clickTargetOf(hit) { + const mesh = hit?.object; + if (!mesh) return null; + const group = interactiveGroupOf(mesh); + if (group) return { kind: 'object', key: mesh.uuid, mesh, top: null, point: hit.point, distance: hit.distance ?? 0, group }; + const top = topLevelObjectOf(mesh); + if (!top) return null; + if (deviceBetween(mesh, top)) return { kind: 'object', key: mesh.uuid, mesh, top, point: hit.point, distance: hit.distance ?? 0, group: null }; + if (top.userData?.clickable === true || objectHasOnClick(top.uuid)) + return { kind: 'object', key: top.uuid, mesh, top, point: hit.point, distance: hit.distance ?? 0, group: null }; + return null; +} + +/** + * The nearest clickable along a ray: module interactive groups first (the viewport click's + * order), then the scene with the see-through rule (a faint wall does not take the press). + * @param {any} ray a THREE.Raycaster @returns {ClickTarget | null} + */ +export function clickTargetAlong(ray) { + /** @type {ClickTarget | null} */ + let best = null; + const scene = get(globalScene); + for (const name of moduleInteractiveGroups) { + const root = scene?.getObjectByName(name); + if (!root) continue; + const hits = ray.intersectObject(root, true).filter((/** @type {any} */ h) => h.object.visible !== false); + const target = hits.length ? clickTargetOf(hits[0]) : null; + if (target && (!best || target.distance < best.distance)) best = target; + } + if (get(objectsGroup)) { + const stack = pickStack(sceneHits(ray), topLevelObjectOf); + const entry = stack.length ? stack[primaryIndex(stack)] : null; + const target = entry?.hit ? clickTargetOf(entry.hit) : null; + if (target && (!best || target.distance < best.distance)) best = target; + } + return best; +} + +/* -------------------------------------------------------------- controller rays ---- */ + +/** the world ray of three.js controller SLOT `index` @param {number} index */ +export function controllerRayOf(index) { + const renderer = /** @type {any} */ (get(globalRenderer)); + const controller = renderer?.xr?.getController?.(index); + if (!controller) return null; + const ray = new THREE.Raycaster(); + _mat.identity().extractRotation(controller.matrixWorld); + ray.ray.origin.setFromMatrixPosition(controller.matrixWorld); + ray.ray.direction.set(0, 0, -1).applyMatrix4(_mat); + return ray; +} + +/** 'left' | 'right' | undefined for a controller slot @param {number} index */ +export function handOf(index) { + const renderer = /** @type {any} */ (get(globalRenderer)); + const h = renderer?.xr?.getController?.(index)?.userData?.handedness; + return h === 'left' || h === 'right' ? h : undefined; +} + +/* ----------------------------------------------------------------- the defaults ---- */ + +/** what each slot's laser pointed at last frame (entry key) — the hover-enter edge */ +const hoverKey = /** @type {(string | null)[]} */ ([null, null]); +const stats = { taps: 0, bumps: 0 }; + +/** + * One frame of the hover edge for slot `index`: a NEW clickable under the laser taps that + * hand. Exported so the suites drive it with a posed controller (no XR session headless). + * @param {number} index @param {any} [ray] defaults to the controller's own ray + * @returns {ClickTarget | null} + */ +export function hoverFrame(index, ray = controllerRayOf(index)) { + if (!ray || !gameFeelActive()) { + hoverKey[index] = null; + return null; + } + const target = clickTargetAlong(ray); + const key = target?.key ?? null; + if (key && key !== hoverKey[index]) { + hapticPattern('tap', handOf(index)); + stats.taps++; + } + hoverKey[index] = key; + return target; +} + +/** + * A trigger PRESS on slot `index`: a bump when it lands on something clickable. Never + * consumes the press — the click itself still travels the normal select path. + * @param {number} index @param {any} [ray] @returns {boolean} whether it bumped + */ +export function pressFeedback(index, ray = controllerRayOf(index)) { + if (!ray || !gameFeelActive()) return false; + if (!clickTargetAlong(ray)) return false; + hapticPattern('bump', handOf(index)); + stats.bumps++; + return true; +} + +/** @returns {{taps: number, bumps: number, hover: (string | null)[]}} */ +export function vrGameInputDebug() { + return { ...stats, hover: [...hoverKey] }; +} + +/** @type {(() => void)[]} */ +let offs = []; + +/** Wire the defaults into the VR loop. Returns the stop. Idempotent. */ +export function startVrGameInput() { + stopVrGameInput(); + offs = [ + registerVRFrameHook(() => { + const renderer = /** @type {any} */ (get(globalRenderer)); + if (!renderer?.xr?.isPresenting) return; + hoverFrame(0); + hoverFrame(1); + }), + registerVRTriggerHooks({ + start: (/** @type {number} */ index) => { + pressFeedback(index); + return false; // feedback only — the press is not ours to take + } + }) + ]; + return stopVrGameInput; +} + +export function stopVrGameInput() { + for (const off of offs) off(); + offs = []; + hoverKey[0] = hoverKey[1] = null; +} diff --git a/tests/e2e/game-haptics.test.cjs b/tests/e2e/game-haptics.test.cjs new file mode 100644 index 00000000..28286ab1 --- /dev/null +++ b/tests/e2e/game-haptics.test.cjs @@ -0,0 +1,173 @@ +// 30b (vr-play) P1 — C4 HAPTICS. The user, from a Quest: "For games, also enable controller +// vibration, make it cool like haptic, so it would be good for users to interact with the +// world. The vibration should be only interactive mode, not in edit mode." +// +// No headset runs headless, so the XR session is a STUB whose gamepads carry recording +// actuators — every assertion reads what the actuator was actually asked to do. +// 1 Edit is silent (api.haptic, a core pulse, a pattern) · 2 Interact buzzes, on the named +// hand · 3 patterns play as timed pulses and stop when the player leaves mid-pattern · +// 4 the hover-enter TAP: once per entry, only on clickable things (On Click / device part / +// marked), never in Edit · 5 the press BUMP · 6 the knock is scaled by its strength. +const h = require('./helpers.cjs'); + +h.run(async () => { + const browser = await h.launch(); + const A = await h.setupPage(browser, 'A'); + const page = A.page; + + await page.evaluate(async () => { + const s = window.__stores; + await s.moduleSDK.initModules([ + { id: 'haptic30b', name: 'Haptic test', version: '1.0.0', description: 'the 30b haptic seams', register(api) { window.__api = api; } } + ]); + let renderer; + s.globalRenderer.subscribe((r) => (renderer = r))(); + window.__buzz = []; + const source = (hand) => ({ + handedness: hand, + gamepad: { buttons: [], axes: [], hapticActuators: [{ pulse: (i, ms) => window.__buzz.push({ hand, i, ms, t: performance.now() }) }] } + }); + const session = { inputSources: [source('left'), source('right')] }; + renderer.xr.getSession = () => session; + renderer.xr.getController(0).userData.handedness = 'left'; + renderer.xr.getController(1).userData.handedness = 'right'; + }); + const buzz = () => page.evaluate(() => window.__buzz.splice(0)); + const mode = (m) => page.evaluate((m) => window.__stores.objectActions.setEditorMode(m), m); + + console.log('\n=== 1. Edit mode is silent ==='); + await mode('edit'); + const edit = await page.evaluate(() => { + const s = window.__stores; + const before = s.vrControls.hapticDebug().suppressed; + window.__api.haptic(0.6, 50, 'left'); + s.vrControls.hapticPulse(0.4, 40); // a core pulse (menus, teleport, the sleeve) + const pattern = window.__api.hapticPattern('success', 'right'); + return { pattern, suppressed: s.vrControls.hapticDebug().suppressed - before }; + }); + await page.waitForTimeout(300); + const editBuzz = await buzz(); + h.check(editBuzz.length === 0, '1.1 no actuator pulsed in Edit (' + editBuzz.length + ')'); + h.check(edit.suppressed === 2 && edit.pattern === false, '1.2 the pulses were swallowed by the gate and the pattern refused (' + JSON.stringify(edit) + ')'); + + console.log('\n=== 2. Interact buzzes, on the named hand ==='); + await mode('interact'); + await page.evaluate(() => window.__api.haptic(0.6, 50, 'left')); + const one = await buzz(); + h.check(one.length === 1 && one[0].hand === 'left' && one[0].i === 0.6 && one[0].ms === 50, '2.1 api.haptic pulses the LEFT actuator in Interact (' + JSON.stringify(one) + ')'); + await page.evaluate(() => window.__stores.vrControls.hapticPulse(0.3, 20)); + const both = await buzz(); + h.check(both.length === 2 && both.some((b) => b.hand === 'left') && both.some((b) => b.hand === 'right'), '2.2 a pulse with no hand reaches both'); + + console.log('\n=== 3. patterns are timed pulses and stop when the player leaves ==='); + const ok = await page.evaluate(() => ({ s: window.__api.hapticPattern('success', 'right'), u: window.__api.hapticPattern('zzz') })); + await page.waitForTimeout(400); + const success = await buzz(); + h.check(ok.s === true && ok.u === false, '3.1 a known pattern plays, an unknown one is refused'); + h.check( + success.length === 3 && success.every((b) => b.hand === 'right') && success[0].i < success[1].i && success[1].i < success[2].i, + '3.2 success = three rising pulses on the right (' + JSON.stringify(success.map((b) => b.i)) + ')' + ); + h.check(success[2].t - success[0].t > 120, '3.3 they are spaced in time, not fired at once (' + (success[2].t - success[0].t).toFixed(0) + ' ms)'); + await page.evaluate(() => { + window.__api.hapticPattern('rumble', 'left'); + window.__stores.objectActions.setEditorMode('edit'); + }); + await page.waitForTimeout(600); + const cut = await buzz(); + h.check(cut.length === 1, '3.4 leaving Interact mid-rumble stops it after the first pulse (' + cut.length + ' of 4)'); + await mode('interact'); + + console.log('\n=== 4. the hover-enter tap ==='); + const ids = await page.evaluate(async () => { + const s = window.__stores; + const make = async (at) => { + const [uuid] = await window.__api.create('/create box 1 1 1', { at }); + return uuid; + }; + const clickable = await make([0, 1, -4]); + const plain = await make([3, 1, -4]); + const marked = await make([-3, 1, -4]); + await new Promise((r) => setTimeout(r, 400)); + window.__api.flow.addNodes({ graphId: clickable, nodes: [{ type: 'onclick', x: 40, y: 40, data: {} }] }); + let g; + s.objectsGroup.subscribe((v) => (g = v))(); + g.getObjectByProperty('uuid', marked).userData.clickable = true; + return { clickable, plain, marked }; + }); + await page.waitForTimeout(500); + const hover = await page.evaluate((ids) => { + const s = window.__stores; + const THREE = s.THREE; + const k = s.gameKit.vrGameInput; + let g; + s.objectsGroup.subscribe((v) => (g = v))(); + const rayTo = (uuid) => { + const target = g.getObjectByProperty('uuid', uuid).getWorldPosition(new THREE.Vector3()); + const origin = new THREE.Vector3(target.x, 1, 2); + return new THREE.Raycaster(origin, target.clone().sub(origin).normalize()); + }; + const miss = new THREE.Raycaster(new THREE.Vector3(0, 30, 0), new THREE.Vector3(0, 1, 0)); + const out = {}; + out.first = k.hoverFrame(0, rayTo(ids.clickable))?.key ?? null; + out.a = window.__buzz.splice(0); + k.hoverFrame(0, rayTo(ids.clickable)); + k.hoverFrame(0, rayTo(ids.clickable)); + out.b = window.__buzz.splice(0); + out.plain = k.hoverFrame(0, rayTo(ids.plain))?.key ?? null; + out.c = window.__buzz.splice(0); + out.marked = k.hoverFrame(0, rayTo(ids.marked))?.key ?? null; + out.d = window.__buzz.splice(0); + k.hoverFrame(0, miss); + k.hoverFrame(0, rayTo(ids.clickable)); + out.e = window.__buzz.splice(0); + s.objectActions.setEditorMode('edit'); + k.hoverFrame(0, miss); + out.editKey = k.hoverFrame(0, rayTo(ids.clickable))?.key ?? null; + out.f = window.__buzz.splice(0); + s.objectActions.setEditorMode('interact'); + return out; + }, ids); + h.check(hover.first === ids.clickable && hover.a.length === 1 && hover.a[0].hand === 'left' && hover.a[0].i < 0.25, '4.1 the laser entering an On Click object taps that hand lightly (' + JSON.stringify(hover.a) + ')'); + h.check(hover.b.length === 0, '4.2 staying on it taps nothing more (' + hover.b.length + ')'); + h.check(hover.plain === null && hover.c.length === 0, '4.3 a plain object is not clickable: no tap'); + h.check(hover.marked === ids.marked && hover.d.length === 1, '4.4 an object marked userData.clickable taps'); + h.check(hover.e.length === 1, '4.5 leaving and coming back taps again (re-armed)'); + h.check(hover.editKey === null && hover.f.length === 0, '4.6 in Edit the hover resolves nothing and never taps'); + + console.log('\n=== 5. the press bump ==='); + const press = await page.evaluate((ids) => { + const s = window.__stores; + const THREE = s.THREE; + let g; + s.objectsGroup.subscribe((v) => (g = v))(); + const rayTo = (uuid) => { + const target = g.getObjectByProperty('uuid', uuid).getWorldPosition(new THREE.Vector3()); + const origin = new THREE.Vector3(target.x, 1, 2); + return new THREE.Raycaster(origin, target.clone().sub(origin).normalize()); + }; + const k = s.gameKit.vrGameInput; + const hit = k.pressFeedback(1, rayTo(ids.clickable)); + const a = window.__buzz.splice(0); + const none = k.pressFeedback(1, rayTo(ids.plain)); + const b = window.__buzz.splice(0); + return { hit, a, none, b }; + }, ids); + h.check(press.hit && press.a.length === 1 && press.a[0].hand === 'right' && press.a[0].i >= 0.4, '5.1 a trigger landing on a clickable bumps the pressing hand (' + JSON.stringify(press.a) + ')'); + h.check(press.none === false && press.b.length === 0, '5.2 a press on nothing clickable does not'); + + console.log('\n=== 6. the knock is scaled by its strength ==='); + await page.evaluate(() => { + const v = window.__stores.vrControls; + v.hapticKnock(Math.min(1, 0.2 + 0.4 / 10), 30, 'right'); // a brush at 0.4 m/s + }); + await page.waitForTimeout(800); // headless timers run late — let the second pulse land + const soft = await buzz(); + await page.evaluate(() => window.__stores.vrControls.hapticKnock(Math.min(1, 0.2 + 6 / 10), 30, 'right')); // a swing at 6 m/s + await page.waitForTimeout(800); + const hard = await buzz(); + h.check(soft.length === 2 && hard.length === 2, '6.1 a knock plays the two-pulse hit (' + soft.length + ', ' + hard.length + ')'); + h.check(hard[0].i > soft[0].i * 2 && hard[0].i <= 1, '6.2 a harder knock buzzes harder (' + soft[0].i.toFixed(3) + ' -> ' + hard[0].i.toFixed(3) + ')'); + + await h.finish(browser); +}); diff --git a/tests/unit/hapticPatterns.test.js b/tests/unit/hapticPatterns.test.js new file mode 100644 index 00000000..28f4d547 --- /dev/null +++ b/tests/unit/hapticPatterns.test.js @@ -0,0 +1,45 @@ +// 30b (vr-play) C4: the haptic patterns as data — every contract name, pulses an actuator +// can take (never overlapping: a new pulse REPLACES the running one on a real actuator), +// the scale, and the knock strength curve. +import { describe, it, expect } from 'vitest'; +import { HAPTIC_PATTERN_NAMES, hapticSchedule, knockHapticScale } from '../../src/lib/hapticPatterns.js'; + +describe('haptic patterns', () => { + it('ships the contract presets', () => { + expect([...HAPTIC_PATTERN_NAMES].sort()).toEqual(['bump', 'fail', 'heartbeat', 'hit', 'rumble', 'success', 'tap']); + }); + + it('every pulse fits an actuator and pulses never overlap', () => { + for (const name of HAPTIC_PATTERN_NAMES) { + const pulses = hapticSchedule(name); + expect(pulses.length, name).toBeGreaterThan(0); + pulses.forEach((p, i) => { + expect(p.intensity).toBeGreaterThan(0); + expect(p.intensity).toBeLessThanOrEqual(1); + expect(p.ms).toBeGreaterThanOrEqual(1); + if (i > 0) expect(p.at, name + ' pulse ' + i).toBeGreaterThanOrEqual(pulses[i - 1].at + pulses[i - 1].ms); + }); + } + }); + + it('tap is the lightest and rumble the longest', () => { + const total = (/** @type {string} */ n) => hapticSchedule(n).reduce((s, p) => s + p.ms, 0); + const peak = (/** @type {string} */ n) => Math.max(...hapticSchedule(n).map((p) => p.intensity)); + for (const n of HAPTIC_PATTERN_NAMES) if (n !== 'tap') expect(peak('tap')).toBeLessThan(peak(n)); + for (const n of HAPTIC_PATTERN_NAMES) if (n !== 'rumble') expect(total('rumble')).toBeGreaterThan(total(n)); + }); + + it('unknown names are empty, scale clamps', () => { + expect(hapticSchedule('buzzz')).toEqual([]); + expect(hapticSchedule('hit', 10).every((p) => p.intensity <= 1)).toBe(true); + expect(hapticSchedule('hit', 0.5)[0].intensity).toBeCloseTo(0.425, 6); + expect(hapticSchedule('hit', -3).every((p) => p.intensity === 0)).toBe(true); + }); + + it('a knock scales from a brush to a full hit', () => { + expect(knockHapticScale(0)).toBeCloseTo(0.2, 6); + expect(knockHapticScale(2)).toBeGreaterThan(knockHapticScale(0.5)); + expect(knockHapticScale(50)).toBe(1); + expect(knockHapticScale(NaN)).toBeCloseTo(0.2, 6); + }); +}); From f47d9f246c73ea9852d9dee428307cddcf2f94a5 Mon Sep 17 00:00:00 2001 From: AlexZ005 Date: Wed, 23 Sep 2026 15:16:18 +0300 Subject: [PATCH 3/8] [feat] 30b P2: effects and announce (C6) - pooled particle bursts, api.announce banner - effectsBurst.js: api.effects.burst([x,y,z], {kind, color, count}) - sparkle / confetti / smoke / sparks as a POOL of 12 THREE.Points systems (96 particles each) at the scene root (never objectsGroup - golden rule 5), recycled oldest-first, LOCAL (nothing on the wire). Deterministic spread (a golden-angle spiral), per-kind motion (gravity, drag, rise, swell), additive kinds fade by darkening. Ticked from the flow runtime's frame loop (it runs in a headset, where window rAF stops) and leaving it when idle; the AGE is real time, so a burst lives its life whatever the frame rate (a throttled tab ticks ~5/s). - gameAnnounce.js (leaf) + api.announce(text, {sub, ms, color}): a big centred banner, ONE at a time (a new one replaces), gone by itself; HudLayer draws it on the desktop (#game-announce, pointer-events none, stands down in VR - P3 draws it head-locked). - a scene clear (runSceneClearHandlers) takes the live bursts and the banner with it. - vrControls.hapticPulse gains an additive `force` (4th arg) for 30b-vr-modes' Edit/Interact switch tick, the one pulse that must be felt IN Edit (asked for by that lane). Suite game-effects (new): 22/22. Counterfactuals (four broken in one run): - the burst never joins the frame loop -> 3.1, 3.2 red, and 6.1 (the frame changed 60 px, not 3709) - the pool grows without bound -> 4.1, 4.2 red (21 systems for 20 bursts) - the banner timer never clears -> 7.4 red - the scene-clear hooks removed -> 8.2 red Held: hud-actions 65/0, game-editor-flow 41/0, sdk-game-seams 78/0, module-toolbox 46/0, editor-modes 45/0. svelte-check 334/47, no new message. Build green. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/components/hud/HudLayer.svelte | 56 ++++++ src/lib/effectsBurst.js | 291 +++++++++++++++++++++++++++++ src/lib/gameAnnounce.js | 49 +++++ src/lib/gameKit.js | 2 + src/lib/moduleSDK.js | 33 ++++ src/lib/vrControls.js | 7 +- tests/e2e/game-effects.test.cjs | 220 ++++++++++++++++++++++ 7 files changed, 656 insertions(+), 2 deletions(-) create mode 100644 src/lib/effectsBurst.js create mode 100644 src/lib/gameAnnounce.js create mode 100644 tests/e2e/game-effects.test.cjs diff --git a/src/components/hud/HudLayer.svelte b/src/components/hud/HudLayer.svelte index c1659958..f875bee4 100644 --- a/src/components/hud/HudLayer.svelte +++ b/src/components/hud/HudLayer.svelte @@ -26,6 +26,9 @@ import { claimInput, releaseInput, onInput } from '$lib/inputRuntime'; import { gamepadPrefs } from '$lib/gamepadPrefs'; import { fireHudButton } from '$lib/flowRuntime'; + // 30b (vr-play): api.announce's banner — drawn here on the desktop, and by the VR game + // panel's head-locked banner in a headset (DOM is invisible there) + import { gameAnnouncement } from '$lib/gameAnnounce'; // 21-D5: WHICH documents are on screen — the scene HUD, plus the one keyed by the // camera being looked through (attaching a HUD to a camera IS keying it by that @@ -343,6 +346,15 @@ } +{#if $gameAnnouncement && !$isVRMode && layerAllowed} + {#key $gameAnnouncement.id} +
+
{$gameAnnouncement.text}
+ {#if $gameAnnouncement.sub}
{$gameAnnouncement.sub}
{/if} +
+ {/key} +{/if} + {#if anyVisible}
{#each elements as el (el.__key + ':' + el.id)} @@ -380,6 +392,50 @@ .hud-slot { position: absolute; } + /* 30b: api.announce — a big centred banner over the game, never clickable, gone by + itself (the store clears it; the fade only dresses the exit) */ + .game-announce { + position: fixed; + left: 50%; + top: 32%; + transform: translate(-50%, -50%); + z-index: var(--z-hud, 45); + pointer-events: none; + text-align: center; + padding: 18px 40px; + border-radius: 18px; + background: rgb(10 14 22 / 0.72); + box-shadow: 0 10px 40px rgb(0 0 0 / 0.45); + animation: game-announce-in 220ms ease-out, game-announce-out 300ms ease-in calc(var(--announce-ms, 1800ms) - 300ms) forwards; + } + .game-announce-title { + font-size: clamp(32px, 6vw, 72px); + font-weight: 800; + letter-spacing: 0.02em; + line-height: 1.05; + text-shadow: 0 2px 12px rgb(0 0 0 / 0.6); + } + .game-announce-sub { + margin-top: 6px; + font-size: clamp(14px, 2vw, 22px); + color: #e5e7eb; + } + @keyframes game-announce-in { + from { + opacity: 0; + transform: translate(-50%, -50%) scale(0.85); + } + } + @keyframes game-announce-out { + to { + opacity: 0; + } + } + @media (prefers-reduced-motion: reduce) { + .game-announce { + animation: none; + } + } /* 30 P1: a previewed GAME screen is a picture of the menu, not the menu — every click goes through it to the viewport (the elements opt back INTO pointer events one by one, so the override has to reach them) */ diff --git a/src/lib/effectsBurst.js b/src/lib/effectsBurst.js new file mode 100644 index 00000000..4032f4c4 --- /dev/null +++ b/src/lib/effectsBurst.js @@ -0,0 +1,291 @@ +// 30b (vr-play) C6 — `api.effects.burst([x,y,z], {kind, color, count})`: a short-lived +// particle burst for a game's moments — the sparkle when a Towers ring is reached, the +// confetti on a goal, smoke off an explosion, sparks off a hit. +// +// POOLED: a fixed set of THREE.Points systems (POOL_SIZE x MAX_COUNT particles) built on +// first use and recycled oldest-first, so a burst per frame costs no allocation and no GPU +// upload of a new geometry — only the attribute ranges it rewrites. LOCAL by contract: +// nothing is sent (a module that wants peers to see it broadcasts its own op), and the +// systems live at the SCENE ROOT under fixed names (golden rule 5: objectsGroup is +// replicated, the scene root is local), so no burst can leak into a save or a GLTF sync. +// +// Ticked from the flow runtime's frame loop (the module frame-task list — it runs on the +// desktop AND in a headset, where window rAF stops), and parked invisible between bursts. +// `depthWrite: false` is right for soft sprites; a burst is short enough that the AO/outline +// trade-off (the documented depthWrite trap) does not matter. +// +// Kinds (the motion model per kind is the only thing that differs): +// sparkle gold twinkling stars that drift up, additive +// confetti many-coloured flakes thrown up, falling under gravity, normal blending +// smoke grey puffs that rise, swell and thin +// sparks fast orange streaks, gravity, very short +import * as THREE from 'three'; +import { get } from 'svelte/store'; +import { globalScene } from '../stores/sceneStore'; +import { moduleFrameTasks } from './moduleSDK'; + +export const BURST_KINDS = ['sparkle', 'confetti', 'smoke', 'sparks']; +export const POOL_SIZE = 12; +export const MAX_COUNT = 96; + +/** + * The per-kind recipe. Pure data; `life` in seconds, speeds in m/s, `size` in metres. + * @type {Record} + */ +export const BURST_RECIPES = { + sparkle: { count: 40, life: 1.1, speed: 1.2, up: 0.8, gravity: -0.4, drag: 1.6, size: 0.09, grow: -0.5, additive: true, color: '#ffd76a' }, + confetti: { count: 80, life: 2.2, speed: 3.2, up: 3.2, gravity: -5.5, drag: 1.2, size: 0.07, grow: 0, additive: false, color: '#ffffff', palette: ['#ef4444', '#f59e0b', '#22c55e', '#3b82f6', '#a855f7', '#ec4899'] }, + smoke: { count: 28, life: 2.4, speed: 0.5, up: 0.9, gravity: 0.3, drag: 0.8, size: 0.35, grow: 1.4, additive: false, color: '#9ca3af' }, + sparks: { count: 60, life: 0.55, speed: 5.5, up: 1.5, gravity: -9.8, drag: 0.6, size: 0.045, grow: -0.6, additive: true, color: '#ffa53a' } +}; + +/** + * Where each particle starts heading: a deterministic spread over the sphere (a golden- + * angle spiral), so a burst looks the same every time and the maths is testable. Returns + * unit directions with the upward kick of the recipe folded into y later. + * @param {number} n @returns {number[][]} + */ +export function burstDirections(n) { + /** @type {number[][]} */ + const out = []; + const golden = Math.PI * (3 - Math.sqrt(5)); + for (let i = 0; i < n; i++) { + const y = 1 - (2 * (i + 0.5)) / n; + const r = Math.sqrt(Math.max(0, 1 - y * y)); + const a = golden * i; + out.push([Math.cos(a) * r, y, Math.sin(a) * r]); + } + return out; +} + +/** @type {THREE.Texture | null} */ +let sprite = null; +/** a soft round dot, drawn once (a square point reads as a pixel, not a particle) */ +function spriteTexture() { + if (sprite) return sprite; + const canvas = document.createElement('canvas'); + canvas.width = canvas.height = 64; + const g = /** @type {CanvasRenderingContext2D} */ (canvas.getContext('2d')); + const grad = g.createRadialGradient(32, 32, 0, 32, 32, 32); + grad.addColorStop(0, 'rgba(255,255,255,1)'); + grad.addColorStop(0.35, 'rgba(255,255,255,0.85)'); + grad.addColorStop(1, 'rgba(255,255,255,0)'); + g.fillStyle = grad; + g.fillRect(0, 0, 64, 64); + sprite = new THREE.CanvasTexture(canvas); + sprite.colorSpace = THREE.SRGBColorSpace; + return sprite; +} + +/** + * @typedef {object} BurstSystem + * @property {THREE.Points} points + * @property {Float32Array} pos @property {Float32Array} vel @property {Float32Array} col + * @property {Float32Array} base the per-particle start colour (col fades from it) + * @property {number} count @property {number} age @property {number} life + * @property {any} recipe @property {number} startedAt + */ + +/** @type {BurstSystem[]} */ +const pool = []; +let lastTick = 0; +let ticking = false; +const stats = { fired: 0, recycled: 0 }; + +/** @param {number} index @returns {BurstSystem} */ +function makeSystem(index) { + const geometry = new THREE.BufferGeometry(); + const pos = new Float32Array(MAX_COUNT * 3); + const col = new Float32Array(MAX_COUNT * 3); + geometry.setAttribute('position', new THREE.BufferAttribute(pos, 3).setUsage(THREE.DynamicDrawUsage)); + geometry.setAttribute('color', new THREE.BufferAttribute(col, 3).setUsage(THREE.DynamicDrawUsage)); + geometry.setDrawRange(0, 0); + const material = new THREE.PointsMaterial({ + size: 0.1, + map: spriteTexture(), + vertexColors: true, + transparent: true, + depthWrite: false, + sizeAttenuation: true + }); + const points = new THREE.Points(geometry, material); + points.name = 'effects-burst-' + index; + points.frustumCulled = false; // the particles fly out of any bounds computed at t=0 + points.visible = false; + points.renderOrder = 996; + points.userData.localOnly = true; + return { + points, + pos, + vel: new Float32Array(MAX_COUNT * 3), + col, + base: new Float32Array(MAX_COUNT * 3), + count: 0, + age: 0, + life: 0, + recipe: null, + startedAt: 0 + }; +} + +/** the free system, or the oldest running one @returns {BurstSystem | null} */ +function takeSystem() { + const scene = /** @type {any} */ (get(globalScene)); + if (!scene) return null; + while (pool.length < POOL_SIZE) pool.push(makeSystem(pool.length)); + for (const sys of pool) if (sys.points.parent !== scene) scene.add(sys.points); + const idle = pool.find((sys) => !sys.points.visible); + if (idle) return idle; + stats.recycled++; + return pool.reduce((a, b) => (a.startedAt <= b.startedAt ? a : b)); +} + +/** @param {string | undefined} kind */ +function recipeOf(kind) { + return BURST_RECIPES[kind && BURST_RECIPES[kind] ? kind : 'sparkle']; +} + +/** + * Fire a burst at a world position. LOCAL. Returns the system's name, or null when there + * is no scene or the position is not three finite numbers. + * @param {number[]} position @param {{kind?: string, color?: string, count?: number}} [options] + * @returns {string | null} + */ +export function burst(position, options = {}) { + if (!Array.isArray(position) || position.length < 3 || !position.slice(0, 3).every((n) => Number.isFinite(Number(n)))) return null; + const sys = takeSystem(); + if (!sys) return null; + const recipe = recipeOf(options?.kind); + const wanted = Number(options?.count); + const count = Math.max(1, Math.min(MAX_COUNT, Math.round(Number.isFinite(wanted) && wanted > 0 ? wanted : recipe.count))); + const dirs = burstDirections(count); + const tint = new THREE.Color(); + const custom = typeof options?.color === 'string' && options.color ? options.color : null; + for (let i = 0; i < count; i++) { + const d = dirs[i]; + // a small per-particle speed variation from the index (deterministic) + const speed = recipe.speed * (0.55 + 0.45 * (((i * 7919) % 97) / 97)); + sys.pos[i * 3] = Number(position[0]); + sys.pos[i * 3 + 1] = Number(position[1]); + sys.pos[i * 3 + 2] = Number(position[2]); + sys.vel[i * 3] = d[0] * speed; + sys.vel[i * 3 + 1] = d[1] * speed + recipe.up; + sys.vel[i * 3 + 2] = d[2] * speed; + const colour = custom ?? (recipe.palette ? recipe.palette[i % recipe.palette.length] : recipe.color); + tint.set(colour); + sys.base[i * 3] = sys.col[i * 3] = tint.r; + sys.base[i * 3 + 1] = sys.col[i * 3 + 1] = tint.g; + sys.base[i * 3 + 2] = sys.col[i * 3 + 2] = tint.b; + } + sys.count = count; + sys.age = 0; + sys.life = recipe.life; + sys.recipe = recipe; + sys.startedAt = performance.now(); + const material = /** @type {THREE.PointsMaterial} */ (sys.points.material); + material.blending = recipe.additive ? THREE.AdditiveBlending : THREE.NormalBlending; + material.size = recipe.size; + material.opacity = 1; + material.needsUpdate = true; + const geometry = sys.points.geometry; + geometry.setDrawRange(0, count); + geometry.attributes.position.needsUpdate = true; + geometry.attributes.color.needsUpdate = true; + sys.points.visible = true; + stats.fired++; + ensureTicking(); + return sys.points.name; +} + +/** + * Advance every live burst by `dt` seconds. Exported so the suites step it + * deterministically; the frame task calls it with the real frame time. + * @param {number} dt + */ +export function stepBursts(dt) { + // the AGE runs on the real clock (a burst lives its life whatever the frame rate — a + // throttled tab ticks five times a second); only the INTEGRATION step is capped, so a + // long frame cannot fling a particle across the room + const elapsed = Math.min(1, Math.max(0, dt)); + const step = Math.min(0.1, elapsed); + for (const sys of pool) { + if (!sys.points.visible) continue; + sys.age += elapsed; + const r = sys.recipe; + if (sys.age >= sys.life) { + sys.points.visible = false; + sys.points.geometry.setDrawRange(0, 0); + continue; + } + const damp = Math.exp(-r.drag * step); + const fade = 1 - sys.age / sys.life; + for (let i = 0; i < sys.count; i++) { + const k = i * 3; + sys.vel[k] *= damp; + sys.vel[k + 1] = sys.vel[k + 1] * damp + r.gravity * step; + sys.vel[k + 2] *= damp; + sys.pos[k] += sys.vel[k] * step; + sys.pos[k + 1] += sys.vel[k + 1] * step; + sys.pos[k + 2] += sys.vel[k + 2] * step; + // additive particles fade by DARKENING (black adds nothing); a sparkle twinkles + const twinkle = r === BURST_RECIPES.sparkle ? 0.6 + 0.4 * Math.sin(sys.age * 24 + i * 1.7) : 1; + const k2 = r.additive ? fade * twinkle : 1; + sys.col[k] = sys.base[k] * k2; + sys.col[k + 1] = sys.base[k + 1] * k2; + sys.col[k + 2] = sys.base[k + 2] * k2; + } + const material = /** @type {THREE.PointsMaterial} */ (sys.points.material); + if (!r.additive) material.opacity = Math.min(1, fade * 1.6); + material.size = Math.max(0.005, r.size * (1 + r.grow * (sys.age / sys.life))); + sys.points.geometry.attributes.position.needsUpdate = true; + sys.points.geometry.attributes.color.needsUpdate = true; + } +} + +function frameTask() { + const now = performance.now(); + const dt = lastTick ? (now - lastTick) / 1000 : 0; + lastTick = now; + stepBursts(dt); + if (!pool.some((sys) => sys.points.visible)) { + // nothing alive: leave the frame loop until the next burst. AFTER the loop that is + // calling us — a splice inside its forEach would skip the next module's task + ticking = false; + lastTick = 0; + queueMicrotask(() => { + if (ticking) return; // a burst fired in the meantime + const i = moduleFrameTasks.indexOf(frameTask); + if (i >= 0) moduleFrameTasks.splice(i, 1); + }); + } +} + +function ensureTicking() { + if (ticking) return; + ticking = true; + lastTick = 0; + if (!moduleFrameTasks.includes(frameTask)) moduleFrameTasks.push(frameTask); +} + +/** live systems for the suites @returns {{fired: number, recycled: number, live: {name: string, count: number, age: number, kind: string}[]}} */ +export function burstDebug() { + return { + ...stats, + live: pool + .filter((sys) => sys.points.visible) + .map((sys) => ({ + name: sys.points.name, + count: sys.count, + age: sys.age, + kind: Object.keys(BURST_RECIPES).find((k) => BURST_RECIPES[k] === sys.recipe) ?? '' + })) + }; +} + +/** Drop every live burst (a scene clear) */ +export function clearBursts() { + for (const sys of pool) { + sys.points.visible = false; + sys.points.geometry.setDrawRange(0, 0); + } +} diff --git a/src/lib/gameAnnounce.js b/src/lib/gameAnnounce.js new file mode 100644 index 00000000..2aea5559 --- /dev/null +++ b/src/lib/gameAnnounce.js @@ -0,0 +1,49 @@ +// 30b (vr-play) — `api.announce(text, {sub, ms, color})`: a BIG centred banner for a +// game's moments — "GOAL!", "Level 3", "Ring 2 reached". The user, on Towers: "When I +// reach some of the rings and go to the top, it should dynamically tell me". +// +// A LEAF holding one store; two renderers read it — HudLayer on the desktop (DOM) and the +// VR game panel's head-locked banner in a headset (DOM is invisible there). LOCAL: nothing +// is sent (a module announces on the peer that saw the moment, or broadcasts its own op). +// A new announcement REPLACES the one showing — a banner queue would still be showing +// "Ring 1" when the player reached ring 3. +import { writable, get } from 'svelte/store'; + +/** @typedef {{id: number, text: string, sub: string, color: string, ms: number, at: number}} Announcement */ + +/** the banner on screen, or null @type {import('svelte/store').Writable} */ +export const gameAnnouncement = writable(null); + +const DEFAULT_MS = 1800; +const MAX_MS = 15000; +let nextId = 1; +/** @type {any} */ +let timer = null; + +/** + * Show a banner. Returns its id (0 for empty text). `ms` is clamped to 300..15000. + * @param {string} text @param {{sub?: string, ms?: number, color?: string}} [options] + * @returns {number} + */ +export function announce(text, options = {}) { + const line = String(text ?? '').trim().slice(0, 80); + if (!line) return 0; + const raw = Number(options?.ms); + const ms = Number.isFinite(raw) ? Math.min(MAX_MS, Math.max(300, raw)) : DEFAULT_MS; + const color = typeof options?.color === 'string' && /^#?[0-9a-z(),.% -]{3,40}$/i.test(options.color) ? options.color : '#ffd76a'; + const id = nextId++; + gameAnnouncement.set({ id, text: line, sub: String(options?.sub ?? '').slice(0, 120), color, ms, at: Date.now() }); + if (timer) clearTimeout(timer); + timer = setTimeout(() => { + timer = null; + if (get(gameAnnouncement)?.id === id) gameAnnouncement.set(null); + }, ms); + return id; +} + +/** Take the banner down now (a scene clear, leaving the game). */ +export function clearAnnouncement() { + if (timer) clearTimeout(timer); + timer = null; + gameAnnouncement.set(null); +} diff --git a/src/lib/gameKit.js b/src/lib/gameKit.js index b2524374..ecc87b37 100644 --- a/src/lib/gameKit.js +++ b/src/lib/gameKit.js @@ -7,3 +7,5 @@ export * as gameMusic from './gameMusic'; export * as gameMusicPresets from './gameMusicPresets'; export * as hapticPatterns from './hapticPatterns'; export * as vrGameInput from './vrGameInput'; +export * as effectsBurst from './effectsBurst'; +export * as gameAnnounce from './gameAnnounce'; diff --git a/src/lib/moduleSDK.js b/src/lib/moduleSDK.js index b6c120b3..9e0d7a12 100644 --- a/src/lib/moduleSDK.js +++ b/src/lib/moduleSDK.js @@ -55,6 +55,8 @@ import { makeModuleStorage } from './gameStorage'; // safeStorage, sessionClock, sceneStore), so static edges close no cycle import { isGameSound, playGameSound } from './gameSfx'; import { playGameMusic, stopGameMusic, gameMusicState, MUSIC_PRESET_IDS } from './gameMusic'; +// 30b (vr-play): api.announce's banner store — a LEAF (svelte/store only) +import { announce as announceBanner, clearAnnouncement } from './gameAnnounce'; /** the ping chimes `api.playSound` still reaches (pingAudio's PING_SOUNDS ids) */ const PING_NAMES = new Set(['ding', 'chime', 'pluck', 'bell']); @@ -182,6 +184,9 @@ const sceneClearHandlers = []; /** Called by the clear-scene path (local and remote) */ export function runSceneClearHandlers() { + // 30b: a cleared scene takes its game's bursts and banner with it + effectsRef?.clearBursts?.(); + clearAnnouncement(); sceneClearHandlers.forEach((fn) => { try { fn(); @@ -259,6 +264,8 @@ let nodeCatalogRef = null; let knockRef = null; /** @type {Promise} */ let knockReady = Promise.resolve(null); +/** 30b: primed for api.effects (effectsBurst imports moduleFrameTasks from here) @type {any} */ +let effectsRef = null; if (typeof window !== 'undefined') { knockReady = import('./knock').then((m) => (knockRef = m)); import('./inputRuntime').then((m) => (inputRuntimeRef = m)); @@ -279,6 +286,8 @@ if (typeof window !== 'undefined') { import('./flowRuntime').then((m) => (flowRuntimeRef = m)); import('./flowGraphs').then((m) => (flowGraphsRef = m)); import('./nodeCatalog').then((m) => (nodeCatalogRef = m)); + // 30b (C6): the burst pool reads moduleFrameTasks from here, so the edge back is dynamic + import('./effectsBurst').then((m) => (effectsRef = m)); } // --- api.pointerRay (190): where the user is POINTING, as a world ray -------- @@ -1062,6 +1071,30 @@ function makeApi(moduleId, moduleName = moduleId) { * false) and stops by itself when the player leaves the game. * Presets: 'arcade', 'ambient', 'dungeon', 'stadium', 'space', 'puzzle', 'studio'. */ + /** + * 30b (C6): a short, pooled particle BURST at a world position — 'sparkle' (the + * default), 'confetti', 'smoke' or 'sparks', an optional CSS colour and a count + * (1..96). LOCAL: broadcast your own op if peers should see it too. Returns + * whether a burst started. + * @param {number[]} position @param {{kind?: string, color?: string, count?: number}=} options + * @returns {boolean} + */ + effects: { + burst: (/** @type {number[]} */ position, /** @type {{kind?: string, color?: string, count?: number}} */ options = {}) => + !!effectsRef?.burst?.(position, options ?? {}), + kinds: () => ['sparkle', 'confetti', 'smoke', 'sparks'] + }, + /** + * 30b: a BIG centred banner — "GOAL!", "Level 3", "Ring 2 reached" — on the desktop + * HUD and, in a headset, head-locked in front of the player. `sub` is a second, + * smaller line; `ms` how long it stays (300..15000, default 1800); `color` the + * title's colour. A new banner replaces the one showing. LOCAL. Returns its id. + * @param {string} text @param {{sub?: string, ms?: number, color?: string}=} options + * @returns {number} + */ + announce(text, options = {}) { + return announceBanner(text, options ?? {}); + }, music: { /** @param {string} preset @param {{volume?: number}=} options 0..1 @returns {boolean} */ play: (preset, options = {}) => playGameMusic(preset, options ?? {}), diff --git a/src/lib/vrControls.js b/src/lib/vrControls.js index 2e0a095e..4fa27da7 100644 --- a/src/lib/vrControls.js +++ b/src/lib/vrControls.js @@ -946,9 +946,12 @@ let hapticSuppressed = 0; * funnel through here, so this one gate covers all of them. * @param {number} intensity 0..1 @param {number} durationMs * @param {'left'|'right'=} hand omit to pulse both + * @param {boolean=} force pulse even in Edit (the mode-switch tick only) */ -export function hapticPulse(intensity = 0.5, durationMs = 50, hand = undefined) { - if (!gameFeelActive()) { +export function hapticPulse(intensity = 0.5, durationMs = 50, hand = undefined, force = false) { + // `force`: the one pulse that must be felt IN Edit — 30b-vr-modes' Edit/Interact switch + // tick, which confirms the switch INTO Edit + if (!force && !gameFeelActive()) { hapticSuppressed++; return; } diff --git a/tests/e2e/game-effects.test.cjs b/tests/e2e/game-effects.test.cjs new file mode 100644 index 00000000..4f21db1b --- /dev/null +++ b/tests/e2e/game-effects.test.cjs @@ -0,0 +1,220 @@ +// 30b (vr-play) P2 — C6 EFFECTS and api.announce. The user, on Towers: "When I reach some of +// the rings and go to the top, it should dynamically tell me and give effects such as +// sparkles or anything similar." +// +// 1 a burst is a scene-root, pooled, LOCAL particle system (nothing in objectsGroup, +// nothing on the wire) · 2 it moves the way its kind says (confetti falls, smoke rises) · +// 3 it dies after its life and leaves the frame loop · 4 the pool is bounded and recycles · +// 5 inputs are clamped and bad ones refused · 6 it reaches the PIXELS · 7 announce shows a +// big banner, replaces the one showing, and goes by itself · 8 a scene clear takes both. +const h = require('./helpers.cjs'); + +h.run(async () => { + const browser = await h.launch(); + const A = await h.setupPage(browser, 'A', { context: { viewport: { width: 1280, height: 720 } } }); + const page = A.page; + + await page.evaluate(async () => { + const s = window.__stores; + await s.moduleSDK.initModules([ + { id: 'fx30b', name: 'Effects test', version: '1.0.0', description: 'the 30b effect seams', register(api) { window.__api = api; } } + ]); + window.__sent = []; + let peer; + s.peers.subscribe((p) => (peer = p))(); + const orig = peer.send.bind(peer); + peer.send = (data) => { + window.__sent.push(data?.type); + return orig(data); + }; + }); + await page.waitForTimeout(600); // the primed effects import + + console.log('\n=== 1. a burst is local, scene-root, pooled ==='); + const fired = await page.evaluate(() => { + const s = window.__stores; + let scene, group; + s.globalScene.subscribe((v) => (scene = v))(); + s.objectsGroup.subscribe((v) => (group = v))(); + const before = group.children.length; + const ok = window.__api.effects.burst([0, 1.5, -3], { kind: 'confetti' }); + const fx = s.gameKit.effectsBurst.burstDebug(); + const live = fx.live[0]; + const points = live ? scene.getObjectByName(live.name) : null; + return { + ok, + live, + parentIsScene: points?.parent === scene, + inGroup: !!group.getObjectByName(live?.name ?? '-'), + groupDelta: group.children.length - before, + drawCount: points?.geometry?.drawRange?.count ?? -1, + sent: window.__sent.slice() + }; + }); + h.check(fired.ok === true && fired.live?.kind === 'confetti', '1.1 api.effects.burst starts a confetti burst (' + JSON.stringify(fired.live) + ')'); + h.check(fired.parentIsScene && !fired.inGroup && fired.groupDelta === 0, '1.2 it lives at the scene root, never in objectsGroup'); + h.check(fired.drawCount === 80, '1.3 confetti draws its recipe count (80, got ' + fired.drawCount + ')'); + h.check(!fired.sent.length, '1.4 nothing was sent (' + JSON.stringify(fired.sent) + ')'); + + console.log('\n=== 2. each kind moves the way it says ==='); + const motion = await page.evaluate(() => { + const s = window.__stores; + const fx = s.gameKit.effectsBurst; + fx.clearBursts(); + let scene; + s.globalScene.subscribe((v) => (scene = v))(); + const meanY = (name) => { + const p = scene.getObjectByName(name); + const a = p.geometry.attributes.position.array; + const n = p.geometry.drawRange.count; + let y = 0; + for (let i = 0; i < n; i++) y += a[i * 3 + 1]; + return y / n; + }; + const spread = (name) => { + const p = scene.getObjectByName(name); + const a = p.geometry.attributes.position.array; + let m = 0; + for (let i = 0; i < p.geometry.drawRange.count; i++) m = Math.max(m, Math.hypot(a[i * 3], a[i * 3 + 1] - 1, a[i * 3 + 2])); + return m; + }; + const c = fx.burst([0, 1, 0], { kind: 'confetti' }); + const m = fx.burst([0, 1, 0], { kind: 'smoke' }); + for (let i = 0; i < 12; i++) fx.stepBursts(1 / 60); + const early = { confetti: meanY(c), smoke: meanY(m), spread: spread(c) }; + for (let i = 0; i < 60; i++) fx.stepBursts(1 / 60); + const late = { confetti: meanY(c), smoke: meanY(m) }; + return { early, late }; + }); + h.check(motion.early.spread > 0.3, '2.1 the particles fly apart (' + motion.early.spread.toFixed(2) + ' m after 0.2 s)'); + h.check(motion.late.confetti < motion.early.confetti, '2.2 confetti falls under gravity (' + motion.early.confetti.toFixed(2) + ' -> ' + motion.late.confetti.toFixed(2) + ')'); + h.check(motion.late.smoke > motion.early.smoke && motion.late.smoke > 1, '2.3 smoke rises (' + motion.early.smoke.toFixed(2) + ' -> ' + motion.late.smoke.toFixed(2) + ')'); + + console.log('\n=== 3. a burst dies and leaves the frame loop ==='); + const lifetime = await page.evaluate(async () => { + const s = window.__stores; + const fx = s.gameKit.effectsBurst; + fx.clearBursts(); + await new Promise((r) => setTimeout(r, 200)); // let the idle task leave + const tasksBefore = s.moduleSDK.moduleFrameTasks.length; + fx.burst([0, 1, -2], { kind: 'sparks' }); + const during = s.moduleSDK.moduleFrameTasks.length; + await new Promise((r) => setTimeout(r, 1400)); // sparks live 0.55 s, on the REAL frame loop + return { tasksBefore, during, after: s.moduleSDK.moduleFrameTasks.length, live: fx.burstDebug().live.length }; + }); + h.check(lifetime.during === lifetime.tasksBefore + 1, '3.1 a live burst ticks from the frame loop (' + JSON.stringify(lifetime) + ')'); + h.check(lifetime.live === 0, '3.2 the real frame loop aged it out (no live bursts after its life)'); + h.check(lifetime.after === lifetime.tasksBefore, '3.3 and its frame task left the loop with it'); + + console.log('\n=== 4. the pool is bounded and recycles ==='); + const pool = await page.evaluate(() => { + const s = window.__stores; + const fx = s.gameKit.effectsBurst; + let scene; + s.globalScene.subscribe((v) => (scene = v))(); + const before = fx.burstDebug().recycled; + for (let i = 0; i < 20; i++) fx.burst([i * 0.1, 1, -2], { kind: 'sparkle' }); + const systems = scene.children.filter((c) => /^effects-burst-/.test(c.name)).length; + return { systems, live: fx.burstDebug().live.length, recycled: fx.burstDebug().recycled - before, cap: fx.POOL_SIZE }; + }); + h.check(pool.systems === pool.cap && pool.live === pool.cap, '4.1 twenty bursts use at most the pool (' + JSON.stringify(pool) + ')'); + h.check(pool.recycled === 20 - pool.cap, '4.2 the oldest ones were recycled'); + + console.log('\n=== 5. inputs are clamped and bad ones refused ==='); + const inputs = await page.evaluate(() => { + const s = window.__stores; + const fx = s.gameKit.effectsBurst; + fx.clearBursts(); + let scene; + s.globalScene.subscribe((v) => (scene = v))(); + const big = fx.burst([0, 1, 0], { count: 5000, kind: 'nope', color: '#00ff00' }); + const p = scene.getObjectByName(big); + const col = p.geometry.attributes.color.array; + return { + count: p.geometry.drawRange.count, + kind: fx.burstDebug().live[0].kind, + green: col[1] > 0.9 && col[0] < 0.1, + bad: window.__api.effects.burst([0, NaN, 0]), + short: window.__api.effects.burst([1, 2]) + }; + }); + h.check(inputs.count === 96, '5.1 count clamps to 96 (' + inputs.count + ')'); + h.check(inputs.kind === 'sparkle' && inputs.green, '5.2 an unknown kind falls back to sparkle, the colour is honoured'); + h.check(inputs.bad === false && inputs.short === false, '5.3 a bad position is refused'); + + console.log('\n=== 6. it reaches the pixels ==='); + await page.evaluate(() => { + window.__stores.gameKit.effectsBurst.clearBursts(); + window.__stores.objectActions.deselectObject?.(); + }); + await page.waitForTimeout(400); + const quiet1 = await h.grabFrame(A); + await page.waitForTimeout(150); + const quiet2 = await h.grabFrame(A); + await page.evaluate(() => { + const s = window.__stores; + let cam; + s.globalCamera.subscribe((c) => (cam = c))(); + const THREE = s.THREE; + const at = cam.getWorldPosition(new THREE.Vector3()).add(cam.getWorldDirection(new THREE.Vector3()).multiplyScalar(4)); + window.__api.effects.burst(at.toArray(), { kind: 'sparkle', count: 96, color: '#ffffff' }); + }); + await page.waitForTimeout(120); + const lit = await h.grabFrame(A); + const idle = await h.frameDelta(page, quiet1, quiet2); + const burstDelta = await h.frameDelta(page, quiet2, lit); + h.check(burstDelta.changed > idle.changed + 400, '6.1 a sparkle in view changes the frame (' + burstDelta.changed + ' px vs idle ' + idle.changed + ')'); + + console.log('\n=== 7. announce ==='); + const ann = await page.evaluate(async () => { + const api = window.__api; + const empty = api.announce(' '); + const id1 = api.announce('Ring 1 reached', { ms: 3000 }); + const id2 = api.announce('GOAL!', { sub: 'Blue 1 - 0', ms: 900, color: '#22c55e' }); + await new Promise((r) => setTimeout(r, 100)); + const el = document.querySelector('#game-announce'); + const title = el?.querySelector('.game-announce-title'); + const out = { + empty, + id1, + id2, + count: document.querySelectorAll('#game-announce').length, + text: el?.textContent?.replace(/\s+/g, ' ').trim(), + color: title ? getComputedStyle(title).color : null, + pe: el ? getComputedStyle(el).pointerEvents : null + }; + await new Promise((r) => setTimeout(r, 1200)); + out.gone = !document.querySelector('#game-announce'); + return out; + }); + h.check(ann.empty === 0 && ann.id1 > 0 && ann.id2 > ann.id1, '7.1 an empty line is refused, each banner gets an id'); + h.check(ann.count === 1 && ann.text === 'GOAL! Blue 1 - 0', '7.2 ONE banner, the newest replacing the first (' + ann.text + ')'); + h.check(ann.color === 'rgb(34, 197, 94)' && ann.pe === 'none', '7.3 the title takes its colour and the banner never takes a click'); + h.check(ann.gone, '7.4 it goes by itself after its ms'); + const vr = await page.evaluate(async () => { + const s = window.__stores; + s.isVRMode.set(true); + window.__api.announce('Level 3'); + await new Promise((r) => setTimeout(r, 100)); + const shown = !!document.querySelector('#game-announce'); + s.isVRMode.set(false); + return shown; + }); + h.check(vr === false, '7.5 in VR the DOM banner stands down (the head-locked one takes over, P3)'); + + console.log('\n=== 8. a scene clear takes both ==='); + const cleared = await page.evaluate(async () => { + const s = window.__stores; + window.__api.effects.burst([0, 1, -2], { kind: 'smoke' }); + window.__api.announce('Wave 2', { ms: 5000 }); + await new Promise((r) => setTimeout(r, 50)); + const before = { live: s.gameKit.effectsBurst.burstDebug().live.length, banner: !!document.querySelector('#game-announce') }; + s.moduleSDK.runSceneClearHandlers(); + await new Promise((r) => setTimeout(r, 100)); + return { before, live: s.gameKit.effectsBurst.burstDebug().live.length, banner: !!document.querySelector('#game-announce') }; + }); + h.check(cleared.before.live > 0 && cleared.before.banner, '8.1 premise: a burst and a banner are up'); + h.check(cleared.live === 0 && !cleared.banner, '8.2 a scene clear removes both (' + JSON.stringify(cleared) + ')'); + + await h.finish(browser); +}); From 57a623fef50f5172f7e47f3e5a82218c1f697bb1 Mon Sep 17 00:00:00 2001 From: AlexZ005 Date: Wed, 23 Sep 2026 16:57:29 +0300 Subject: [PATCH 4/8] [feat] 30b P3: the game UI in VR (C2) - the HUD menu as a world board, the wrist card, the top strip, the banner - vrGamePanel.js: DOM is invisible in a headset, so the game's HUD documents are drawn into CANVAS TEXTURES on four scene-root meshes (default layer = both eyes, never the helper layer; LOCAL; only while presenting AND in Interact/Play): - the BOARD: a menu-like screen (input 'menu', a control on it, or bound to menu / paused / over) laid out with the desktop's own 9-grid (rectInFrame over the 1280x720 stage), CROPPED to the part of the stage the screen uses and sized from it (1.6 mm per stage px, 1.15 m for the Towers menu) - drawn whole it was a wall of empty backdrop with the menu small in the middle. 1.2 m ahead at chest height, following the head LAZILY with hysteresis: a glance inside 35 degrees leaves it, a real turn brings it all the way back in front (without the latch it parked 28 degrees off to the side). A footer of ours: "Edit mode" (objectActions.setEditorMode('edit')) and "Top strip". - the WRIST card: the other screens (score, timer, level) as short lines on the LEFT wrist, shown when the wrist turns toward the face, with the same two buttons - a way back to editing mid-round. Placed clear of 30b-vr-modes' mode label. - the TOP STRIP: the glanceable readouts in one head-locked row (a sentence-long hint stays on the wrist), LOCAL on/off (vr:gameStrip), default on. - the ANNOUNCE banner, head-locked (api.announce in a headset). A press is the desktop press verbatim (fireHudButton; a toggle writes its value first; sliders/dropdowns/tabs step), with a 'click' sound and a 'bump'. - vrGameInput wires it: the frame hook draws, hovers (the laser on the board wins over the world behind it) and POKES (the tip through a button presses once, re-arms on retreat); the trigger hook presses a board button and CONSUMES the press, and the trailing 'select' is swallowed once; two beam providers end the laser on the board/wrist. Suite vr-game-panel (new): 36/36, including section 13 on the REAL Towers template: its menu on the board, "Start round" pressed by laser starts the round, its HUD on the wrist/strip. Evidence: /home/deck/.code/lanes-30/after-30b/30b-vr-play/. Counterfactuals (five broken in one run): - the Interact/Play gate removed -> 8.2 red (the wrist and strip stayed up in Edit) - the board press not consuming -> 3.2, 8.1 red - the poke never re-arming -> 6.3 red - the crop removed -> 1.6 red (a 2.05 m board) - the follow latch releasing at the band edge -> 9.3 red (parked at 52.6 of 80 degrees) svelte-check 334/47, no new message. Build green. Held suites run once for P3+P4 (machine-wide e2e queue of 11 lanes); see the P4 commit. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/lib/gameKit.js | 1 + src/lib/vrGameInput.js | 90 +++- src/lib/vrGamePanel.js | 898 +++++++++++++++++++++++++++++++ tests/e2e/vr-game-panel.test.cjs | 337 ++++++++++++ 4 files changed, 1319 insertions(+), 7 deletions(-) create mode 100644 src/lib/vrGamePanel.js create mode 100644 tests/e2e/vr-game-panel.test.cjs diff --git a/src/lib/gameKit.js b/src/lib/gameKit.js index ecc87b37..51b0bcfc 100644 --- a/src/lib/gameKit.js +++ b/src/lib/gameKit.js @@ -9,3 +9,4 @@ export * as hapticPatterns from './hapticPatterns'; export * as vrGameInput from './vrGameInput'; export * as effectsBurst from './effectsBurst'; export * as gameAnnounce from './gameAnnounce'; +export * as vrGamePanel from './vrGamePanel'; diff --git a/src/lib/vrGameInput.js b/src/lib/vrGameInput.js index 0c858e3e..a8e245e4 100644 --- a/src/lib/vrGameInput.js +++ b/src/lib/vrGameInput.js @@ -29,7 +29,9 @@ import { pickStack, primaryIndex } from './selectThrough'; import { topLevelObjectOf } from './objectActions'; import { objectHasOnClick } from './flowRuntime'; import { moduleInteractiveGroups } from './moduleSDK'; -import { registerVRFrameHook, registerVRTriggerHooks, hapticPattern } from './vrControls'; +import { registerVRFrameHook, registerVRTriggerHooks, registerPanelGroupProvider, controllerIndexFor, hapticPattern } from './vrControls'; +// P3 (C2): the game UI in VR — the panel, the wrist card, the strip, the banner +import { vrGamePanelFrame, panelTargetAlong, panelHover, pressPanelTarget, pokeFrame, uAcross, vrGameSurface, hideVrGamePanel } from './vrGamePanel'; const _mat = new THREE.Matrix4(); @@ -165,6 +167,55 @@ export function vrGameInputDebug() { return { ...stats, hover: [...hoverKey] }; } +/* --------------------------------------------------------------- P3 the panel ---- */ + +const _pos = new THREE.Vector3(); +const _quat = new THREE.Quaternion(); + +/** one hand's controller world pose, or null @param {'left'|'right'} hand */ +function handPose(hand) { + const renderer = /** @type {any} */ (get(globalRenderer)); + const index = controllerIndexFor(hand); + const controller = index >= 0 ? renderer?.xr?.getController?.(index) : null; + if (!controller) return null; + controller.getWorldPosition(_pos); + controller.getWorldQuaternion(_quat); + return { position: _pos.clone(), quaternion: _quat.clone() }; +} + +/** the controller's TIP for a poke (the target-ray origin is its front end) @param {number} index */ +function tipOf(index) { + const renderer = /** @type {any} */ (get(globalRenderer)); + const controller = renderer?.xr?.getController?.(index); + return controller ? controller.getWorldPosition(new THREE.Vector3()) : null; +} + +/** true once a press of OURS (a panel button) took this trigger: its trailing 'select' + * must not also reach Scene's pick */ +let swallowNext = false; +/** was a headset presenting on the last frame (the session-end edge) */ +let presenting = false; + +/** + * A trigger PRESS on slot `index`: a game-panel button under the laser is pressed and the + * press consumed; anything else only gets its haptic bump and falls through untouched. + * Exported for the suites (the live hook calls it). + * @param {number} index @returns {boolean} consumed + */ +export function triggerStart(index) { + swallowNext = false; + const ray = controllerRayOf(index); + const target = ray && gameFeelActive() ? panelTargetAlong(ray) : null; + if (target) { + pressPanelTarget(target, { source: 'trigger', u: uAcross(target) }); + hapticPattern('bump', handOf(index)); + swallowNext = true; + return true; + } + pressFeedback(index); + return false; +} + /** @type {(() => void)[]} */ let offs = []; @@ -174,15 +225,39 @@ export function startVrGameInput() { offs = [ registerVRFrameHook(() => { const renderer = /** @type {any} */ (get(globalRenderer)); - if (!renderer?.xr?.isPresenting) return; - hoverFrame(0); - hoverFrame(1); + if (!renderer?.xr?.isPresenting) { + // the session ENDED: take the surfaces down once (this hook runs every desktop + // frame too, where there is nothing to do) + if (presenting) hideVrGamePanel(); + presenting = false; + return; + } + presenting = true; + vrGamePanelFrame({ hands: [handPose('left'), null] }); + for (const index of [0, 1]) { + const ray = controllerRayOf(index); + // the laser on the board wins over the world behind it + if (panelHover(index, gameFeelActive() ? ray : null)) hoverKey[index] = null; + else hoverFrame(index, ray); + pokeFrame(index, gameFeelActive() ? tipOf(index) : null); + } }), registerVRTriggerHooks({ - start: (/** @type {number} */ index) => { - pressFeedback(index); - return false; // feedback only — the press is not ours to take + start: (/** @type {number} */ index) => triggerStart(index), + swallow: () => { + if (!swallowNext) return false; + swallowNext = false; + return true; } + }), + // the beam ends ON the board and the wrist card (the reticle sits on the button) + registerPanelGroupProvider(() => { + const s = vrGameSurface('vr-game-panel'); + return s?.mesh.visible ? s.mesh : null; + }), + registerPanelGroupProvider(() => { + const s = vrGameSurface('vr-game-wrist'); + return s?.mesh.visible ? s.mesh : null; }) ]; return stopVrGameInput; @@ -192,4 +267,5 @@ export function stopVrGameInput() { for (const off of offs) off(); offs = []; hoverKey[0] = hoverKey[1] = null; + hideVrGamePanel(); } diff --git a/src/lib/vrGamePanel.js b/src/lib/vrGamePanel.js new file mode 100644 index 00000000..33e856cc --- /dev/null +++ b/src/lib/vrGamePanel.js @@ -0,0 +1,898 @@ +// 30b (vr-play) C2 — THE GAME UI IN VR. The user, on a Quest: "in VR I should be able to +// see the game menu and interact with it", "I also should see the score in the VR". +// +// DOM is invisible in a headset, so the HUD layer (HudLayer.svelte) draws nothing there. +// This module draws the SAME documents into CANVAS TEXTURES on four scene-root meshes: +// +// · the GAME PANEL — a world-space board ~1.2 m in front at chest height that LAZILY +// follows your head yaw (it re-centres only once you have turned past ~35 degrees, so +// it holds still while you read it). It shows a game's MENU-like screens (menu, pause, +// results: a screen with input 'menu', a control on it, or bound to menu/paused/over) +// laid out exactly as the desktop HUD lays them out (the 9-grid over the 1280x720 +// reference stage), plus a footer with two buttons of ours: "Edit mode" (the Edit ⇄ +// Interact switch 30b-vr-modes adds; a plain `editorMode` write is enough to leave) and +// "Top strip" on/off. Its buttons answer the controller LASER + trigger and a +// fingertip/controller-tip POKE. +// · the WRIST card — the game's small overlays (score, timer, level) as short lines on +// the LEFT wrist, shown when you turn the wrist toward your face, with the same two +// buttons, so there is always a way back to editing even mid-round. +// · the TOP STRIP — the same lines in one head-locked row across the top of the view, +// LOCAL on/off (persisted), default on. +// · the ANNOUNCE banner — api.announce(), head-locked, big and centred. +// +// A button press is the desktop press verbatim: `fireHudButton(id)` pulses the element's +// Button nodes through the replicated trigger log, a toggle writes its value first — so a +// press in the headset and a click on the desktop cannot disagree about what happened. +// +// LOCAL and non-replicated throughout; everything sits at the SCENE ROOT under fixed names +// (golden rule 5) on the default layer (both eyes, never the helper layer), and only while +// a headset session is presenting AND the player is in Interact/Play. The frame work is +// driven by vrGameInput's frame hook; the suites drive `vrGamePanelFrame` with a synthetic +// head pose, since no XR session runs headless. +import * as THREE from 'three'; +import { get } from 'svelte/store'; +import { globalScene, globalRenderer } from '../stores/sceneStore'; +import { setEditorMode } from './objectActions'; +import { hudDocs, hudRuntime, hudValueOf, setHudValue, visibleScreen, activeHudKeys, rectInFrame, hudScreenOverride } from './hudDocs'; +import { isInteractiveKind, isRenderableKind } from './hudKinds'; +import { fireHudButton, hudOptionsOf } from './flowRuntime'; +import { cameraPreview } from './cameraPreview'; +import { gameState } from './gameState'; +import { gameFeelActive } from './gameFeel'; +import { gameAnnouncement } from './gameAnnounce'; +import { playGameSound } from './gameSfx'; +import { hudImageFor, resolveHudImage } from './hudImages'; +import { safeStorage } from './safeStorage'; + +/** the HUD's authoring reference — the editor artboard's stage */ +export const STAGE_W = 1280; +export const STAGE_H = 720; +/** the panel's footer (our own buttons), in stage pixels */ +export const FOOTER_H = 96; +const PANEL_DIST = 1.2; +const PANEL_DROP = 0.22; // below the eyes: chest height +const FOLLOW_DEG = 35; +const SETTLE_DEG = 2; +const WRIST_W = 0.15; +const STRIP_KEY = 'vr:gameStrip'; + +/** the board canvas's pixels per stage pixel: ~1000 px across a metre-wide board, what a + * headset's ~20 px per degree needs at 1.2 m */ +const SCALE = 1.5; + +/* ------------------------------------------------------------ the classification --- */ + +/** a screen the PANEL shows (a menu, a pause, a results card) rather than the wrist + * @param {any} screen */ +export function isPanelScreen(screen) { + if (!screen) return false; + if (screen.input === 'menu') return true; + if (['menu', 'paused', 'over'].includes(String(screen.showWhile ?? ''))) return true; + return (screen.elements ?? []).some((/** @type {any} */ el) => isInteractiveKind(el.kind) && el.enabled !== false); +} + +/** kinds with no VR form on the board (a crosshair, a plot, the debug pill, module DOM) */ +const NO_VR_FORM = new Set(['crosshair', 'minimap', 'debug', 'custom', 'damageflash']); +/** the board never gets narrower than its footer needs, in stage pixels */ +const MIN_CROP_W = 720; +const CROP_PAD = 36; + +/** + * The part of the 1280x720 stage a screen actually USES — the union of its elements' + * rects, padded, widened to fit the footer. A desktop menu is usually a card in the middle + * of a transparent screen; drawn whole, the headset saw a wall of empty backdrop with the + * menu small in its middle (measured on the Towers template). Pure; exported. + * @param {any} screen @returns {{x: number, y: number, w: number, h: number}} + */ +export function panelCrop(screen) { + let x0 = Infinity; + let y0 = Infinity; + let x1 = -Infinity; + let y1 = -Infinity; + for (const el of screen?.elements ?? []) { + if (!isRenderableKind(el.kind) || NO_VR_FORM.has(el.kind)) continue; + const r = rectInFrame(el, STAGE_W, STAGE_H); + x0 = Math.min(x0, r.left); + y0 = Math.min(y0, r.top); + x1 = Math.max(x1, r.left + r.w); + y1 = Math.max(y1, r.top + r.h); + } + if (!Number.isFinite(x0)) return { x: 0, y: 0, w: STAGE_W, h: STAGE_H }; + x0 = Math.max(0, x0 - CROP_PAD); + y0 = Math.max(0, y0 - CROP_PAD); + x1 = Math.min(STAGE_W, x1 + CROP_PAD); + y1 = Math.min(STAGE_H, y1 + CROP_PAD); + let w = x1 - x0; + if (w < MIN_CROP_W) { + const cx = (x0 + x1) / 2; + w = MIN_CROP_W; + x0 = Math.min(Math.max(0, cx - w / 2), STAGE_W - w); + } + return { x: x0, y: y0, w, h: Math.max(120, y1 - y0) }; +} + +/** metres per stage pixel on the board: a 14 px label reads ~1 degree tall at 1.2 m */ +const METRES_PER_PX = 0.0016; + +/** + * The screens on show right now, the HudLayer rule (scene doc + the one keyed by the camera + * looked through), split into what the panel shows and what the wrist/strip show. + * @returns {{panel: {key: string, screen: any}[], overlay: {key: string, screen: any}[]}} + */ +export function vrScreens() { + const through = get(cameraPreview)?.uuid ?? null; + /** @type {{key: string, screen: any}[]} */ + const panel = []; + /** @type {{key: string, screen: any}[]} */ + const overlay = []; + for (const key of activeHudKeys(through)) { + const screen = visibleScreen(key); + if (!screen) continue; + (isPanelScreen(screen) ? panel : overlay).push({ key, screen }); + } + return { panel, overlay }; +} + +/** + * The overlay as SHORT LINES — what a score/timer/level card SAYS, readable on a wrist. + * Pure over (elements, runtime, values); exported for the suites. + * @param {any[]} elements @param {Record} runtime @returns {string[]} + */ +export function overlayLines(elements, runtime) { + /** @type {string[]} */ + const out = []; + const sorted = [...elements].sort((a, b) => (a.y ?? 0) - (b.y ?? 0) || (a.x ?? 0) - (b.x ?? 0)); + for (const el of sorted) { + if (!isRenderableKind(el.kind)) continue; + const rt = runtime?.[el.id] ?? null; + const text = rt?.text !== undefined && rt?.text !== null ? String(rt.text) : String(el.label ?? ''); + if (el.kind === 'text' || el.kind === 'timer' || el.kind === 'richtext' || el.kind === 'panel') { + const clean = text.replace(/\[[^\]]*\]/g, '').replace(/\s+/g, ' ').trim(); + if (clean) out.push(clean); + } else if (el.kind === 'bar' || el.kind === 'progressradial') { + const value = Number(rt?.value ?? el.value ?? 0); + const min = Number(rt?.min ?? el.min ?? 0); + const max = Number(rt?.max ?? el.max ?? 1); + const pct = max - min > 1e-9 ? Math.round(Math.min(1, Math.max(0, (value - min) / (max - min))) * 100) : 0; + out.push((text ? text + ' ' : '') + pct + '%'); + } else if (el.kind === 'list') { + const rows = Array.isArray(rt?.rows) && rt.rows.length ? rt.rows : String(el.rowsText ?? '').split('\n').filter(Boolean); + if (text) out.push(text); + for (const row of rows.slice(0, 3)) out.push(String(row)); + } + if (out.length >= 6) break; + } + return out.slice(0, 6); +} + +/* --------------------------------------------------------------------- drawing ----- */ + +/** a style value, token names resolved to literals (a canvas cannot take var()) */ +function paint(/** @type {any} */ value, /** @type {string} */ fallback) { + if (value === undefined || value === null || value === '') return fallback; + const text = String(value); + if (/^[a-z][a-z0-9-]*$/i.test(text) && !/^(transparent|white|black|red|green|blue|gray|grey|yellow|orange|none)$/i.test(text)) { + try { + const v = getComputedStyle(document.documentElement).getPropertyValue('--' + text).trim(); + return v || fallback; + } catch { + return fallback; + } + } + return text; +} + +/** @param {CanvasRenderingContext2D} g @param {number} x @param {number} y @param {number} w @param {number} h @param {number} r */ +function roundRect(g, x, y, w, h, r) { + const rr = Math.max(0, Math.min(r, w / 2, h / 2)); + g.beginPath(); + g.moveTo(x + rr, y); + g.arcTo(x + w, y, x + w, y + h, rr); + g.arcTo(x + w, y + h, x, y + h, rr); + g.arcTo(x, y + h, x, y, rr); + g.arcTo(x, y, x + w, y, rr); + g.closePath(); +} + +/** @type {Map} */ +const images = new Map(); +let imageTick = 0; + +/** @typedef {{id: string, key: string, kind: string, x: number, y: number, w: number, h: number, footer?: string}} HitRect */ + +/** + * Draw one menu screen + our footer into a 2D context sized (STAGE_W x (STAGE_H+FOOTER_H)) + * * SCALE, and return the pressable rects in CANVAS pixels. `hover` rings the element the + * laser is on. Pure over its arguments (the DOM is only touched for token colours and + * image decode); exported for the suites. + * @param {CanvasRenderingContext2D} g @param {{key: string, screen: any} | null} entry + * @param {Record} runtime @param {{hover?: string | null, strip?: boolean, mode?: string, crop?: {x: number, y: number, w: number, h: number}}} [opts] + * @returns {HitRect[]} + */ +export function drawPanel(g, entry, runtime, opts = {}) { + const crop = opts.crop ?? { x: 0, y: 0, w: STAGE_W, h: STAGE_H }; + const W = g.canvas.width; + const H = g.canvas.height; + const k = W / crop.w; + g.clearRect(0, 0, W, H); + // the board: a dark rounded plate, so a screen authored over the game view still reads + roundRect(g, 0, 0, W, H, 28 * k); + g.fillStyle = 'rgba(12, 16, 26, 0.9)'; + g.fill(); + g.lineWidth = 3 * k; + g.strokeStyle = 'rgba(148, 163, 184, 0.55)'; + g.stroke(); + /** @type {HitRect[]} */ + const hits = []; + const elements = (entry?.screen?.elements ?? []).filter((/** @type {any} */ el) => isRenderableKind(el.kind)); + const sorted = [...elements].sort((a, b) => (a.z ?? 0) - (b.z ?? 0)); + g.save(); + g.beginPath(); + g.rect(0, 0, W, crop.h * k); + g.clip(); + for (const el of sorted) { + const r = rectInFrame(el, STAGE_W, STAGE_H); + const x = (r.left - crop.x) * k; + const y = (r.top - crop.y) * k; + const w = r.w * k; + const h = r.h * k; + const style = el.style ?? {}; + const rt = runtime?.[el.id] ?? null; + const text = rt?.text !== undefined && rt?.text !== null ? String(rt.text) : String(el.label ?? ''); + const size = Math.max(10, Number(style.size ?? 14)) * k; + const weight = String(style.weight ?? (el.kind === 'button' ? 600 : 400)); + const colour = paint(style.color, '#f3f4f6'); + const disabled = el.enabled === false; + g.globalAlpha = Number(style.opacity ?? 1) * (disabled ? 0.45 : 1); + const bg = paint(style.bg, el.kind === 'button' || el.kind === 'toggle' || el.kind === 'dropdown' || el.kind === 'tabs' ? '#374151' : 'transparent'); + const radius = Number(style.radius ?? (el.kind === 'button' ? 8 : 0)) * k; + if (bg !== 'transparent') { + roundRect(g, x, y, w, h, radius); + g.fillStyle = bg; + g.fill(); + } + if (style.border) { + roundRect(g, x, y, w, h, radius); + g.lineWidth = 2 * k; + g.strokeStyle = paint(style.border, 'rgba(75,85,99,0.7)'); + g.stroke(); + } + g.fillStyle = colour; + g.font = `${weight} ${size}px system-ui, sans-serif`; + g.textBaseline = 'middle'; + const align = el.kind === 'button' ? 'center' : String(style.align ?? 'left'); + g.textAlign = align === 'center' ? 'center' : align === 'right' ? 'right' : 'left'; + const pad = Number(style.pad ?? 0) * k; + const tx = align === 'center' ? x + w / 2 : align === 'right' ? x + w - pad - 4 * k : x + pad + 4 * k; + if (el.kind === 'bar' || el.kind === 'progressradial' || el.kind === 'slider') { + const value = Number(el.kind === 'slider' ? hudValueOf(el.id, el.value ?? el.min ?? 0) : rt?.value ?? el.value ?? 0); + const min = Number(rt?.min ?? el.min ?? 0); + const max = Number(rt?.max ?? el.max ?? (el.kind === 'slider' ? 100 : 1)); + const pct = max - min > 1e-9 ? Math.min(1, Math.max(0, (value - min) / (max - min))) : 0; + roundRect(g, x, y + h * 0.3, w, h * 0.4, h * 0.2); + g.fillStyle = 'rgba(255,255,255,0.15)'; + g.fill(); + roundRect(g, x, y + h * 0.3, w * pct, h * 0.4, h * 0.2); + g.fillStyle = paint(style.color, '#ef562f'); + g.fill(); + if (text && el.kind !== 'slider') { + g.fillStyle = '#f3f4f6'; + g.textAlign = 'center'; + g.fillText(text, x + w / 2, y + h / 2); + } + } else if (el.kind === 'toggle') { + const on = !!hudValueOf(el.id, el.value); + roundRect(g, x + 6 * k, y + h * 0.2, h * 1.1, h * 0.6, h * 0.3); + g.fillStyle = on ? '#22c55e' : 'rgba(255,255,255,0.2)'; + g.fill(); + g.beginPath(); + g.arc(x + 6 * k + (on ? h * 0.8 : h * 0.3), y + h / 2, h * 0.24, 0, Math.PI * 2); + g.fillStyle = '#fff'; + g.fill(); + g.fillStyle = colour; + g.textAlign = 'left'; + g.fillText(text, x + h * 1.3 + 10 * k, y + h / 2); + } else if (el.kind === 'dropdown' || el.kind === 'tabs') { + const options = hudOptionsOf(el.id, el); + const held = hudValueOf(el.id, el.value ?? (el.kind === 'tabs' ? 0 : options[0])); + const shown = el.kind === 'tabs' ? options[Math.round(Number(held)) || 0] ?? '' : String(held ?? ''); + g.textAlign = 'center'; + g.fillText((text ? text + ': ' : '') + '‹ ' + shown + ' ›', x + w / 2, y + h / 2); + } else if (el.kind === 'list') { + const rows = Array.isArray(rt?.rows) && rt.rows.length ? rt.rows : String(el.rowsText ?? '').split('\n').filter(Boolean); + const rowH = Number(el.rowHeight ?? 18) * k; + let yy = y + rowH / 2; + if (text) { + g.fillText(text, tx, yy); + yy += rowH; + } + for (const row of rows) { + if (yy > y + h) break; + g.fillText(String(row), tx, yy); + yy += rowH; + } + } else if (el.kind === 'image') { + const url = hudImageFor(String(el.src ?? '')); + if (!url && el.src) void resolveHudImage(String(el.src)); + if (url) { + let img = images.get(url); + if (!img) { + img = new Image(); + img.onload = () => (imageTick += 1); + img.src = url; + images.set(url, img); + } + if (img.complete && img.naturalWidth) g.drawImage(img, x, y, w, h); + } + } else if (el.kind === 'crosshair' || el.kind === 'minimap' || el.kind === 'debug' || el.kind === 'custom' || el.kind === 'damageflash') { + // a crosshair, a plot, a debug pill and a module's own DOM have no VR form here + } else if (text) { + // text, timer, button, panel, richtext and anything else that says something + const lines = el.wrap || el.kind === 'richtext' ? wrapText(g, text.replace(/\[[^\]]*\]/g, ''), w - 8 * k) : [text]; + const lh = size * 1.25; + let yy = y + h / 2 - ((lines.length - 1) * lh) / 2; + for (const line of lines) { + g.fillText(line, tx, yy); + yy += lh; + } + } + g.globalAlpha = 1; + const pressable = isInteractiveKind(el.kind) && !disabled; + if (pressable) { + hits.push({ id: el.id, key: entry?.key ?? 'scene', kind: el.kind, x, y, w, h }); + if (opts.hover === el.id) { + roundRect(g, x - 4 * k, y - 4 * k, w + 8 * k, h + 8 * k, radius + 4 * k); + g.lineWidth = 4 * k; + g.strokeStyle = '#5fd0ff'; + g.stroke(); + } + } + } + g.restore(); + // the footer: OUR two buttons, always there + const fy = crop.h * k; + g.fillStyle = 'rgba(255,255,255,0.06)'; + g.fillRect(0, fy, W, 2 * k); + const buttons = [ + { id: 'edit', label: 'Edit mode' }, + { id: 'strip', label: opts.strip === false ? 'Top strip: off' : 'Top strip: on' } + ]; + const bw = 300 * k; + const bh = 60 * k; + buttons.forEach((b, i) => { + const bx = W / 2 + (i === 0 ? -bw - 20 * k : 20 * k); + const by = fy + (FOOTER_H * k - bh) / 2; + roundRect(g, bx, by, bw, bh, 12 * k); + g.fillStyle = b.id === 'edit' ? '#1f2937' : '#111827'; + g.fill(); + g.lineWidth = (opts.hover === 'footer:' + b.id ? 5 : 2) * k; + g.strokeStyle = opts.hover === 'footer:' + b.id ? '#5fd0ff' : 'rgba(148,163,184,0.6)'; + g.stroke(); + g.fillStyle = '#f3f4f6'; + g.font = `600 ${26 * k}px system-ui, sans-serif`; + g.textAlign = 'center'; + g.textBaseline = 'middle'; + g.fillText(b.label, bx + bw / 2, by + bh / 2); + hits.push({ id: 'footer:' + b.id, key: '', kind: 'footer', footer: b.id, x: bx, y: by, w: bw, h: bh }); + }); + return hits; +} + +/** a line cut to fit `max` pixels with an ellipsis @param {CanvasRenderingContext2D} g + * @param {string} text @param {number} max */ +function fitText(g, text, max) { + if (g.measureText(text).width <= max) return text; + let lo = 0; + let hi = text.length; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (g.measureText(text.slice(0, mid) + '…').width <= max) lo = mid; + else hi = mid - 1; + } + return text.slice(0, lo).trimEnd() + '…'; +} + +/** the strip is one glanceable row: a sentence (a hint, an instruction) belongs on the + * wrist, not across the top of the view @param {string} line */ +export function stripWorthy(line) { + return line.length <= 28; +} + +/** @param {CanvasRenderingContext2D} g @param {string} text @param {number} max */ +function wrapText(g, text, max) { + /** @type {string[]} */ + const out = []; + for (const para of text.split('\n')) { + let line = ''; + for (const word of para.split(/\s+/)) { + const next = line ? line + ' ' + word : word; + if (line && g.measureText(next).width > max) { + out.push(line); + line = word; + } else line = next; + } + if (line) out.push(line); + } + return out.slice(0, 12); +} + +/** + * A compact card of lines (+ optional footer buttons) — the wrist, the top strip and the + * banner are all this. Returns the pressable rects. + * @param {CanvasRenderingContext2D} g @param {string[]} lines + * @param {{title?: string, row?: boolean, buttons?: boolean, strip?: boolean, hover?: string | null, big?: boolean, color?: string}} [opts] + * @returns {HitRect[]} + */ +export function drawCard(g, lines, opts = {}) { + const W = g.canvas.width; + const H = g.canvas.height; + g.clearRect(0, 0, W, H); + roundRect(g, 0, 0, W, H, Math.min(24, H / 4)); + g.fillStyle = 'rgba(12, 16, 26, 0.86)'; + g.fill(); + g.lineWidth = 2; + g.strokeStyle = 'rgba(148, 163, 184, 0.5)'; + g.stroke(); + g.fillStyle = opts.color ?? '#f3f4f6'; + g.textBaseline = 'middle'; + /** @type {HitRect[]} */ + const hits = []; + if (opts.big) { + g.textAlign = 'center'; + g.font = `800 ${Math.round(H * 0.42)}px system-ui, sans-serif`; + g.fillText(fitText(g, lines[0] ?? '', W - 40), W / 2, lines[1] ? H * 0.4 : H / 2); + if (lines[1]) { + g.font = `500 ${Math.round(H * 0.16)}px system-ui, sans-serif`; + g.fillStyle = '#e5e7eb'; + g.fillText(lines[1], W / 2, H * 0.78); + } + return hits; + } + if (opts.row) { + g.textAlign = 'center'; + g.font = `600 ${Math.round(H * 0.46)}px system-ui, sans-serif`; + g.fillText(fitText(g, lines.join(' · '), W - 24), W / 2, H / 2); + return hits; + } + const pad = 18; + const footer = opts.buttons ? 64 : 0; + const rows = Math.max(1, lines.length + (opts.title ? 1 : 0)); + const rowH = Math.min(52, (H - footer - pad * 2) / rows); + let y = pad + rowH / 2; + g.textAlign = 'left'; + if (opts.title) { + g.font = `700 ${Math.round(rowH * 0.62)}px system-ui, sans-serif`; + g.fillStyle = '#93c5fd'; + g.fillText(opts.title, pad, y); + y += rowH; + } + g.fillStyle = '#f3f4f6'; + g.font = `600 ${Math.round(rowH * 0.62)}px system-ui, sans-serif`; + for (const line of lines) { + g.fillText(fitText(g, line, W - pad * 2), pad, y); + y += rowH; + } + if (opts.buttons) { + const labels = [ + { id: 'edit', label: 'Edit' }, + { id: 'strip', label: opts.strip === false ? 'Strip off' : 'Strip on' } + ]; + const bw = (W - pad * 3) / 2; + labels.forEach((b, i) => { + const bx = pad + i * (bw + pad); + const by = H - footer + 8; + roundRect(g, bx, by, bw, footer - 22, 10); + g.fillStyle = '#1f2937'; + g.fill(); + g.lineWidth = opts.hover === 'footer:' + b.id ? 4 : 2; + g.strokeStyle = opts.hover === 'footer:' + b.id ? '#5fd0ff' : 'rgba(148,163,184,0.6)'; + g.stroke(); + g.fillStyle = '#f3f4f6'; + g.font = `600 22px system-ui, sans-serif`; + g.textAlign = 'center'; + g.fillText(b.label, bx + bw / 2, by + (footer - 22) / 2); + hits.push({ id: 'footer:' + b.id, key: '', kind: 'footer', footer: b.id, x: bx, y: by, w: bw, h: footer - 22 }); + }); + } + return hits; +} + +/* ------------------------------------------------------------------ the meshes ----- */ + +/** + * @typedef {{name: string, mesh: THREE.Mesh, canvas: HTMLCanvasElement, g: CanvasRenderingContext2D, + * texture: THREE.CanvasTexture, w: number, h: number, hits: HitRect[], sig: string}} Surface + */ + +/** @type {Record} */ +const surfaces = {}; + +/** @param {string} name @param {number} pxW @param {number} pxH @param {number} worldW */ +function surface(name, pxW, pxH, worldW) { + let s = surfaces[name]; + const scene = /** @type {any} */ (get(globalScene)); + if (!s) { + const canvas = document.createElement('canvas'); + canvas.width = pxW; + canvas.height = pxH; + const g = /** @type {CanvasRenderingContext2D} */ (canvas.getContext('2d')); + const texture = new THREE.CanvasTexture(canvas); + texture.colorSpace = THREE.SRGBColorSpace; + texture.anisotropy = 4; + const worldH = (worldW * pxH) / pxW; + const mesh = new THREE.Mesh( + new THREE.PlaneGeometry(worldW, worldH), + new THREE.MeshBasicMaterial({ map: texture, transparent: true, depthTest: false, depthWrite: false, side: THREE.DoubleSide }) + ); + mesh.name = name; + mesh.renderOrder = 1000; + mesh.visible = false; + mesh.frustumCulled = false; + mesh.userData.localOnly = true; + s = { name, mesh, canvas, g, texture, w: worldW, h: worldH, hits: [], sig: '' }; + surfaces[name] = s; + } + if (scene && s.mesh.parent !== scene) scene.add(s.mesh); + return s; +} + +/** give a surface a new canvas size and world size (a new crop) @param {Surface} s + * @param {number} pxW @param {number} pxH @param {number} worldW */ +function resizeSurface(s, pxW, pxH, worldW) { + if (s.canvas.width === pxW && s.canvas.height === pxH && Math.abs(s.w - worldW) < 1e-6) return; + s.canvas.width = pxW; + s.canvas.height = pxH; + // a CanvasTexture keeps its GPU allocation at the OLD size unless it is disposed + s.texture.dispose(); + s.texture.needsUpdate = true; + s.w = worldW; + s.h = (worldW * pxH) / pxW; + s.mesh.geometry.dispose(); + s.mesh.geometry = new THREE.PlaneGeometry(s.w, s.h); +} + +/* -------------------------------------------------------------------- the state ---- */ + +/** LOCAL: the head-locked strip, on by default */ +function stripOn() { + return safeStorage.getItem(STRIP_KEY) !== 'false'; +} +/** @param {boolean} on */ +export function setVrStrip(on) { + safeStorage.setItem(STRIP_KEY, on ? 'true' : 'false'); +} + +const panelYaw = { value: NaN, following: false }; +/** the laser's current panel target per controller slot, for the hover ring */ +const hover = /** @type {(string | null)[]} */ ([null, null]); +const debug = { presses: 0, pokes: 0, lastPress: /** @type {string | null} */ (null), edits: 0 }; + +const _head = new THREE.Vector3(); +const _headQ = new THREE.Quaternion(); +const _fwd = new THREE.Vector3(); +const _up = new THREE.Vector3(); +const _v = new THREE.Vector3(); +const _e = new THREE.Euler(); + +/** the headset's world pose @returns {{position: THREE.Vector3, quaternion: THREE.Quaternion} | null} */ +function headPose() { + const renderer = /** @type {any} */ (get(globalRenderer)); + if (!renderer?.xr?.isPresenting) return null; + const cam = renderer.xr.getCamera(); + cam.updateMatrixWorld?.(true); + cam.getWorldPosition(_head); + cam.getWorldQuaternion(_headQ); + return { position: _head, quaternion: _headQ }; +} + +/** @param {THREE.Quaternion} q yaw of a quaternion (radians) */ +function yawOf(q) { + _e.setFromQuaternion(q, 'YXZ'); + return _e.y; +} + +/** + * Where the panel should yaw: it FOLLOWS lazily, with HYSTERESIS — a glance inside + * FOLLOW_DEG leaves it alone, and once the head has turned past that the panel eases ALL + * the way back in front (stopping at SETTLE_DEG). Without the second half a big turn left + * it parked FOLLOW_DEG off to the side, forever. Pure over (current, head, dt, state); + * `state.following` carries the latch between frames. Exported. + * @param {number} current @param {number} head @param {number} dt seconds + * @param {{following: boolean}} state + * @returns {number} + */ +export function followYaw(current, head, dt, state) { + if (!Number.isFinite(current)) { + state.following = false; + return head; + } + let d = head - current; + d = Math.atan2(Math.sin(d), Math.cos(d)); + const off = Math.abs(d); + if (!state.following && off > (FOLLOW_DEG * Math.PI) / 180) state.following = true; + if (state.following && off < (SETTLE_DEG * Math.PI) / 180) state.following = false; + if (!state.following) return current; + return current + d * Math.min(1, dt * 3); +} + +/** + * One frame of the VR game UI. `opts.head` injects a head pose and `opts.hands` the two + * controller world poses — the suites drive both; the live frame reads the XR camera and + * the controllers. Returns what is on show. + * @param {{head?: {position: any, quaternion: any} | null, hands?: ({position: any, quaternion: any} | null)[], dt?: number, force?: boolean}} [opts] + */ +export function vrGamePanelFrame(opts = {}) { + const head = opts.head === undefined ? headPose() : opts.head; + const live = !!head && (opts.force || gameFeelActive()); + const { panel, overlay } = live ? vrScreens() : { panel: [], overlay: [] }; + const runtime = get(hudRuntime); + const strip = stripOn(); + void get(hudDocs); + void get(hudScreenOverride); + void get(gameState); + + // ---- the panel + const entry = panel[0] ?? null; + const crop = panelCrop(entry?.screen); + const board = surface('vr-game-panel', Math.round(crop.w * SCALE), Math.round((crop.h + FOOTER_H) * SCALE), crop.w * METRES_PER_PX); + board.mesh.visible = !!entry; + if (entry && head) { + const sig = JSON.stringify([entry.key, entry.screen, runtime, hover, strip, imageTick, valuesSig(entry.screen)]); + if (sig !== board.sig) { + board.sig = sig; + resizeSurface(board, Math.round(crop.w * SCALE), Math.round((crop.h + FOOTER_H) * SCALE), crop.w * METRES_PER_PX); + board.hits = drawPanel(board.g, entry, runtime, { hover: hover[0] ?? hover[1], strip, crop }); + board.texture.needsUpdate = true; + } + const headYaw = yawOf(head.quaternion); + panelYaw.value = followYaw(panelYaw.value, headYaw, opts.dt ?? 1 / 72, panelYaw); + _fwd.set(-Math.sin(panelYaw.value), 0, -Math.cos(panelYaw.value)); + board.mesh.position.copy(head.position).addScaledVector(_fwd, PANEL_DIST); + board.mesh.position.y = head.position.y - PANEL_DROP; + board.mesh.rotation.set(-0.12, panelYaw.value, 0, 'YXZ'); + board.mesh.updateMatrixWorld(true); + } else if (!entry) panelYaw.value = NaN; // a new menu appears straight ahead + + // ---- the wrist + the strip (the overlay lines; the wrist is there even with none) + const lines = overlayLines( + overlay.flatMap((o) => o.screen.elements ?? []), + runtime + ); + const wrist = surface('vr-game-wrist', 320, 300, WRIST_W); + const leftHand = opts.hands?.[0] ?? null; + if (live && leftHand) { + const sig = JSON.stringify([lines, strip, hover]); + if (sig !== wrist.sig) { + // a line count change re-sizes the card + wrist.sig = sig; + wrist.hits = drawCard(wrist.g, lines.length ? lines : ['No score yet'], { title: 'Game', buttons: true, strip, hover: hover[0] ?? hover[1] }); + wrist.texture.needsUpdate = true; + } + // up the forearm, facing up off it — clear of 30b-vr-modes' mode label, which sits on + // the same controller at (0, 0.03, 0.07) + _v.set(-0.01, 0.035, 0.21).applyQuaternion(leftHand.quaternion).add(leftHand.position); + wrist.mesh.position.copy(_v); + wrist.mesh.quaternion.copy(leftHand.quaternion).multiply(WRIST_TILT); + wrist.mesh.updateMatrixWorld(true); + // shown when the card's face points at the head ("turn the wrist to read") + _up.set(0, 0, 1).applyQuaternion(wrist.mesh.quaternion); + _v.copy(head ? head.position : _v).sub(wrist.mesh.position).normalize(); + wrist.mesh.visible = _up.dot(_v) > 0.35; + } else wrist.mesh.visible = false; + + const stripLines = lines.filter(stripWorthy); + const bar = surface('vr-game-strip', 1024, 72, 0.9); + bar.mesh.visible = live && strip && stripLines.length > 0; + if (bar.mesh.visible && head) { + const sig = JSON.stringify(stripLines); + if (sig !== bar.sig) { + bar.sig = sig; + drawCard(bar.g, stripLines, { row: true }); + bar.texture.needsUpdate = true; + } + headLocked(bar.mesh, head, 1.3, 0.36); + } + + // ---- the announce banner (head-locked; shown in any mode while presenting) + const note = get(gameAnnouncement); + const banner = surface('vr-game-announce', 1024, 256, 0.95); + banner.mesh.visible = !!head && !!note; + if (banner.mesh.visible && head && note) { + const sig = JSON.stringify([note.id, note.text, note.sub, note.color]); + if (sig !== banner.sig) { + banner.sig = sig; + drawCard(banner.g, [note.text, note.sub].filter(Boolean), { big: true, color: note.color }); + banner.texture.needsUpdate = true; + } + headLocked(banner.mesh, head, 1.5, 0.12); + const left = note.at + note.ms - Date.now(); + /** @type {any} */ (banner.mesh.material).opacity = Math.max(0, Math.min(1, left / 300)); + } + return { + panel: board.mesh.visible ? entry?.screen?.id ?? null : null, + lines, + stripLines, + wrist: wrist.mesh.visible, + strip: bar.mesh.visible, + banner: banner.mesh.visible + }; +} + +const WRIST_TILT = new THREE.Quaternion().setFromEuler(new THREE.Euler(-Math.PI / 2, 0, 0)); + +/** @param {any} screen the values a menu's inputs hold (a toggle flip must redraw) */ +function valuesSig(screen) { + return (screen?.elements ?? []).filter((/** @type {any} */ el) => isInteractiveKind(el.kind)).map((/** @type {any} */ el) => hudValueOf(el.id, el.value)); +} + +/** @param {THREE.Object3D} mesh @param {{position: any, quaternion: any}} head @param {number} dist @param {number} up */ +function headLocked(mesh, head, dist, up) { + _fwd.set(0, 0, -1).applyQuaternion(head.quaternion); + _up.set(0, 1, 0).applyQuaternion(head.quaternion); + mesh.position.copy(head.position).addScaledVector(_fwd, dist).addScaledVector(_up, up); + mesh.quaternion.copy(head.quaternion); + mesh.updateMatrixWorld(true); +} + +/* ---------------------------------------------------------------- interaction ----- */ + +/** @typedef {{surface: string, hit: HitRect, point: THREE.Vector3, distance: number}} PanelTarget */ + +/** @param {Surface} s @param {THREE.Vector2} uv @returns {HitRect | null} */ +function hitAtUv(s, uv) { + const px = uv.x * s.canvas.width; + const py = (1 - uv.y) * s.canvas.height; + return s.hits.find((r) => px >= r.x && px <= r.x + r.w && py >= r.y && py <= r.y + r.h) ?? null; +} + +/** + * What a ray points at on the panel or the wrist — a pressable rect or null. Exported for + * the sweep (P4) and the suites. + * @param {THREE.Raycaster} ray @returns {PanelTarget | null} + */ +export function panelTargetAlong(ray) { + /** @type {PanelTarget | null} */ + let best = null; + for (const name of ['vr-game-panel', 'vr-game-wrist']) { + const s = surfaces[name]; + if (!s?.mesh.visible) continue; + const hit = ray.intersectObject(s.mesh, false)[0]; + if (!hit?.uv) continue; + const rect = hitAtUv(s, hit.uv); + if (rect && (!best || hit.distance < best.distance)) best = { surface: name, hit: rect, point: hit.point.clone(), distance: hit.distance }; + } + return best; +} + +/** + * The laser's hover ring for slot `index` (redraws the board when it changes). + * @param {number} index @param {THREE.Raycaster | null} ray @returns {PanelTarget | null} + */ +export function panelHover(index, ray) { + const target = ray ? panelTargetAlong(ray) : null; + hover[index] = target?.hit.id ?? null; + return target; +} + +/** + * PRESS a panel target — the desktop press verbatim. Returns true when something acted. + * @param {PanelTarget} target @param {{source?: string, u?: number}} [opts] + */ +export function pressPanelTarget(target, opts = {}) { + const hit = target?.hit; + if (!hit) return false; + debug.presses++; + debug.lastPress = hit.id; + playGameSound('click', target.point ? target.point.toArray() : null); + if (hit.kind === 'footer') { + if (hit.footer === 'edit') { + debug.edits++; + // the ONE Edit/Interact switch (30b-vr-modes' Y button calls it too); the store + // write is the whole of leaving Interact, the rest re-seats a desktop gizmo + setEditorMode('edit'); + } else if (hit.footer === 'strip') setVrStrip(!stripOn()); + return true; + } + const el = elementOf(hit); + if (!el) return false; + if (el.kind === 'button') fireHudButton(el.id); + else if (el.kind === 'toggle') { + setHudValue(el.id, !hudValueOf(el.id, el.value), { shared: !!el.shared }); + fireHudButton(el.id); + } else if (el.kind === 'slider') { + const min = Number(el.min ?? 0); + const max = Number(el.max ?? 100); + const step = Number(el.step || 1); + const u = Number.isFinite(opts.u) ? Number(opts.u) : 0.5; + const raw = min + (max - min) * Math.min(1, Math.max(0, u)); + setHudValue(el.id, Math.round(raw / step) * step, { shared: !!el.shared }); + } else if (el.kind === 'dropdown' || el.kind === 'tabs') { + const options = hudOptionsOf(el.id, el); + if (!options.length) return false; + if (el.kind === 'tabs') { + const at = Math.max(0, Math.round(Number(hudValueOf(el.id, el.value ?? 0)))); + setHudValue(el.id, (at + 1) % options.length, { shared: !!el.shared }); + } else { + const held = String(hudValueOf(el.id, el.value ?? options[0])); + setHudValue(el.id, options[(Math.max(0, options.indexOf(held)) + 1) % options.length], { shared: !!el.shared }); + } + } else fireHudButton(el.id); // a module kind with sub-presses: its own id pulses + return true; +} + +/** @param {HitRect} hit */ +function elementOf(hit) { + for (const { screen } of vrScreens().panel) { + const el = (screen.elements ?? []).find((/** @type {any} */ e) => e.id === hit.id); + if (el) return el; + } + return null; +} + +/** the u (0..1 across the rect) of a panel target — a slider's value @param {PanelTarget} target */ +export function uAcross(target) { + const s = surfaces[target.surface]; + if (!s) return 0.5; + const local = s.mesh.worldToLocal(target.point.clone()); + const px = (local.x / s.w + 0.5) * s.canvas.width; + return (px - target.hit.x) / Math.max(1, target.hit.w); +} + +/* ------------------------------------------------------------------------ poke ---- */ + +/** a tip is TOUCHING within this of the plane; it must back off past RELEASE to re-arm */ +const POKE_TOUCH = 0.018; +const POKE_RELEASE = 0.04; +const pokeArmed = [true, true]; + +/** + * One frame of the fingertip / controller-tip POKE for slot `index`: pushing the tip + * through a button presses it once; pulling back re-arms. Returns the pressed target. + * @param {number} index @param {THREE.Vector3 | null} tip world position + * @returns {PanelTarget | null} + */ +export function pokeFrame(index, tip) { + if (!tip) return null; + let pressed = null; + let nearest = Infinity; + for (const name of ['vr-game-panel', 'vr-game-wrist']) { + const s = surfaces[name]; + if (!s?.mesh.visible) continue; + const local = s.mesh.worldToLocal(tip.clone()); + if (Math.abs(local.x) > s.w / 2 || Math.abs(local.y) > s.h / 2) continue; + const depth = Math.abs(local.z); + nearest = Math.min(nearest, depth); + if (depth > POKE_TOUCH || !pokeArmed[index]) continue; + const uv = new THREE.Vector2(local.x / s.w + 0.5, local.y / s.h + 0.5); + const rect = hitAtUv(s, uv); + if (!rect) continue; + const target = { surface: name, hit: rect, point: tip.clone(), distance: 0 }; + pokeArmed[index] = false; + debug.pokes++; + pressPanelTarget(target, { source: 'poke', u: uAcross(target) }); + pressed = target; + } + if (nearest > POKE_RELEASE) pokeArmed[index] = true; + return pressed; +} + +/** @returns {{presses: number, pokes: number, lastPress: string | null, edits: number, hover: (string | null)[], strip: boolean, hits: Record}} */ +export function vrGamePanelDebug() { + /** @type {Record} */ + const hits = {}; + for (const [name, s] of Object.entries(surfaces)) hits[name] = s.hits.map((h) => ({ ...h })); + return { ...debug, hover: [...hover], strip: stripOn(), hits }; +} + +/** the surfaces' meshes (suites read visibility, poses and canvases) @param {string} name */ +export function vrGameSurface(name) { + const s = surfaces[name]; + return s ? { mesh: s.mesh, canvas: s.canvas } : null; +} + +/** Hide everything (the session ended, the player left the game). */ +export function hideVrGamePanel() { + for (const s of Object.values(surfaces)) s.mesh.visible = false; + hover[0] = hover[1] = null; + panelYaw.value = NaN; +} diff --git a/tests/e2e/vr-game-panel.test.cjs b/tests/e2e/vr-game-panel.test.cjs new file mode 100644 index 00000000..2607cb74 --- /dev/null +++ b/tests/e2e/vr-game-panel.test.cjs @@ -0,0 +1,337 @@ +// 30b (vr-play) P3 — C2 THE GAME UI IN VR. The user, on a Quest 3: "in VR I should be able to +// see the game menu and interact with it", "I also should see the score in the VR", "there +// should be an option available to go into the edit mode". +// +// No headset runs headless: the suite drives vrGamePanelFrame with a SYNTHETIC head pose and +// posed controllers (matrixAutoUpdate on — the documented XR-controller trap), and reads +// what a player would see: which surfaces are visible, where they sit, and what their +// canvases hold. +// 1 a menu screen becomes the world panel, 1.2 m ahead at chest height, drawn · 2 its +// buttons are hit rects, ours included · 3 the LASER + trigger presses Start through the +// real trigger hook: the game starts, the press is consumed and its trailing select +// swallowed · 4 in play the score becomes the wrist card + the top strip · 5 the wrist +// shows only when turned toward the face · 6 a POKE presses once and re-arms on retreat · +// 7 the strip toggles · 8 Edit mode leaves Interact and every surface goes · 9 the panel +// follows the head LAZILY · 10 the announce banner in VR · 11 not presenting = nothing · +// 12 a toggle flips its value · 13 the REAL Towers template: its menu in VR, Start pressed. +const h = require('./helpers.cjs'); +const fs = require('fs'); +const path = require('path'); + +const SCENES_REPO = [path.resolve(__dirname, '../../../theprototype.app-scenes'), path.resolve(__dirname, '../../../scenes')].find((p) => + fs.existsSync(p) +); +const TOWERS = process.env.TOWERS_TPSCENE || (SCENES_REPO && path.join(SCENES_REPO, 'games/towers/scene.tpscene')); + +h.run(async () => { + const browser = await h.launch(); + const A = await h.setupPage(browser, 'A'); + const page = A.page; + + await page.evaluate(async () => { + const s = window.__stores; + await s.moduleSDK.initModules([ + { id: 'panel30b', name: 'Panel test', version: '1.0.0', description: 'the 30b VR panel', register(api) { window.__api = api; } } + ]); + localStorage.removeItem('vr:gameStrip'); + // a game: a menu screen bound to 'menu' with Start + a music toggle, and a play HUD + s.hudDocs.setHudDocFor('scene', { + active: '', + screens: [ + { + id: 'menu', + name: 'Menu', + showWhile: 'menu', + input: 'menu', + elements: [ + { id: 'title', kind: 'text', anchor: 'top-center', x: 0, y: 80, w: 600, h: 90, label: 'TOWERS', style: { size: 64, align: 'center', weight: '800' } }, + { id: 'start', kind: 'button', anchor: 'center', x: 0, y: 0, w: 320, h: 80, label: 'Start', style: { bg: '#2563eb', size: 32 } }, + { id: 'music', kind: 'toggle', anchor: 'center', x: 0, y: 120, w: 320, h: 56, label: 'Music', value: false } + ] + }, + { + id: 'hud', + name: 'HUD', + showWhile: 'playing', + elements: [ + { id: 'score', kind: 'text', anchor: 'top-left', x: 24, y: 24, w: 240, h: 40, label: 'Height: 0 m' }, + { id: 'time', kind: 'timer', anchor: 'top-right', x: 24, y: 24, w: 160, h: 40, label: '1:30' } + ] + } + ] + }); + s.gameState.setGameState('menu'); + s.hudActions.addBinding('start', 'start'); + s.objectActions.setEditorMode('interact'); + // posed controllers: three writes their matrices per XR frame, so a test must switch + // that back on before posing them by hand + let r; + s.globalRenderer.subscribe((v) => (r = v))(); + for (const i of [0, 1]) { + const c = r.xr.getController(i); + c.matrixAutoUpdate = true; + c.userData.handedness = i === 0 ? 'left' : 'right'; + } + window.__T = { + head: (yawDeg = 0) => { + const THREE = s.THREE; + return { + position: new THREE.Vector3(0, 1.6, 0), + quaternion: new THREE.Quaternion().setFromEuler(new THREE.Euler(0, (yawDeg * Math.PI) / 180, 0, 'YXZ')) + }; + }, + // the left hand held up, palm-side card facing the eyes (or turned away) + leftHand: (facing = true) => { + const THREE = s.THREE; + const q = new THREE.Quaternion().setFromEuler(new THREE.Euler(facing ? 0.9 : -2.2, 0, 0, 'YXZ')); + return { position: new THREE.Vector3(-0.1, 1.2, -0.35), quaternion: q }; + }, + // the world point of a hit rect's centre on a surface + rectPoint: (surfaceName, id) => { + const k = s.gameKit.vrGamePanel; + const surf = k.vrGameSurface(surfaceName); + const rect = k.vrGamePanelDebug().hits[surfaceName].find((x) => x.id === id); + if (!surf || !rect) return null; + const g = surf.mesh.geometry.parameters; + const cx = (rect.x + rect.w / 2) / surf.canvas.width; + const cy = (rect.y + rect.h / 2) / surf.canvas.height; + return surf.mesh.localToWorld(new s.THREE.Vector3((cx - 0.5) * g.width, (0.5 - cy) * g.height, 0)); + }, + aim: (index, from, to) => { + const c = r.xr.getController(index); + c.position.copy(from); + // Object3D.lookAt turns a plain object's +Z toward the target, and a controller's + // ray is its -Z: look at the point MIRRORED through the controller instead + c.lookAt(from.clone().multiplyScalar(2).sub(to)); + c.updateMatrixWorld(true); + } + }; + }); + const frame = (opts = '{}') => + page.evaluate((o) => { + const T = window.__T; + const k = window.__stores.gameKit.vrGamePanel; + return k.vrGamePanelFrame({ head: T.head(o.yaw ?? 0), hands: [o.hand === false ? null : T.leftHand(o.facing !== false), null], dt: o.dt ?? 1 / 72 }); + }, JSON.parse(opts)); + + console.log('\n=== 1. the menu screen becomes the world panel ==='); + const f1 = await frame(); + const panel = await page.evaluate(() => { + const k = window.__stores.gameKit.vrGamePanel; + const surf = k.vrGameSurface('vr-game-panel'); + const g = surf.canvas.getContext('2d'); + const data = g.getImageData(0, 0, surf.canvas.width, surf.canvas.height).data; + let ink = 0; + for (let i = 0; i < data.length; i += 4) if (data[i + 3] > 200 && data[i] + data[i + 1] + data[i + 2] > 600) ink++; + const start = k.vrGamePanelDebug().hits['vr-game-panel'].find((x) => x.id === 'start'); + const px = g.getImageData(Math.round(start.x + 8), Math.round(start.y + start.h / 2), 1, 1).data; + let scene; + window.__stores.globalScene.subscribe((v) => (scene = v))(); + return { + pos: surf.mesh.position.toArray(), + visible: surf.mesh.visible, + parentIsScene: surf.mesh.parent === scene, + layer0: surf.mesh.layers.test(new window.__stores.THREE.Layers()), + ink, + width: surf.mesh.geometry.parameters.width, + startPx: Array.from(px) + }; + }); + h.check(f1.panel === 'menu' && panel.visible, '1.1 the menu screen is on the panel (' + JSON.stringify(f1) + ')'); + h.check(Math.abs(panel.pos[2] + 1.2) < 0.05 && panel.pos[1] < 1.6 && panel.pos[1] > 1.2, '1.2 ~1.2 m ahead, below the eyes (' + panel.pos.map((n) => n.toFixed(2)) + ')'); + h.check(panel.parentIsScene && panel.layer0, '1.3 a scene-root mesh on the default layer (both eyes)'); + h.check(panel.ink > 1500, '1.4 the canvas holds drawn text (' + panel.ink + ' white px)'); + h.check(panel.width > 0.6 && panel.width < 1.3, '1.6 the board is sized to the menu it shows, not the whole stage (' + panel.width.toFixed(2) + ' m wide)'); + h.check(panel.startPx[2] > 200 && panel.startPx[0] < 80, '1.5 the Start button is drawn in its authored blue (' + panel.startPx + ')'); + + console.log('\n=== 2. the buttons are hit rects, ours included ==='); + const ids = await page.evaluate(() => window.__stores.gameKit.vrGamePanel.vrGamePanelDebug().hits['vr-game-panel'].map((x) => x.id)); + h.check(ids.includes('start') && ids.includes('music') && !ids.includes('title'), '2.1 Start and the toggle press, the title does not (' + ids + ')'); + h.check(ids.includes('footer:edit') && ids.includes('footer:strip'), '2.2 our Edit mode and Top strip buttons are on the footer'); + + console.log('\n=== 3. laser + trigger presses Start through the real trigger hook ==='); + await page.waitForTimeout(700); // a fresh binding must be seen by a tick before its first press (actionSeenAt) + const press = await page.evaluate(async () => { + const s = window.__stores; + const T = window.__T; + const target = T.rectPoint('vr-game-panel', 'start'); + T.aim(1, new s.THREE.Vector3(0.25, 1.3, -0.3), target); + const hover = s.gameKit.vrGamePanel.panelHover(1, s.gameKit.vrGameInput.controllerRayOf(1))?.hit.id ?? null; + const consumed = s.vrControls.vrModuleTriggerStart(1); + await new Promise((r) => setTimeout(r, 500)); // the Set Game State node acts on the next tick + let state; + s.gameState.gameState.subscribe((v) => (state = v.state))(); + const swallow1 = s.vrControls.vrModuleSelectSwallowed(); + const swallow2 = s.vrControls.vrModuleSelectSwallowed(); + s.vrControls.vrModuleTriggerEnd(1); + return { hover, consumed, state, swallow1, swallow2 }; + }); + h.check(press.hover === 'start', '3.1 the laser hovers Start (' + press.hover + ')'); + h.check(press.consumed === true && press.state === 'playing', '3.2 the trigger pressed it: the game is playing (' + JSON.stringify(press) + ')'); + h.check(press.swallow1 === true && press.swallow2 === false, '3.3 the trailing select is swallowed once, not forever'); + + console.log('\n=== 4. in play the score is on the wrist and the strip ==='); + const f4 = await frame(); + h.check(f4.panel === null, '4.1 the panel goes when the menu does'); + h.check(JSON.stringify(f4.lines) === JSON.stringify(['Height: 0 m', '1:30']), '4.2 the overlay reads as lines (' + JSON.stringify(f4.lines) + ')'); + h.check(f4.wrist && f4.strip, '4.3 the wrist card and the top strip are up'); + const stripPose = await page.evaluate(() => window.__stores.gameKit.vrGamePanel.vrGameSurface('vr-game-strip').mesh.position.toArray()); + h.check(stripPose[1] > 1.8 && stripPose[2] < -1, '4.4 the strip rides high in front of the eyes (' + stripPose.map((n) => n.toFixed(2)) + ')'); + + console.log('\n=== 5. the wrist shows only when turned to the face ==='); + const away = await frame('{"facing": false}'); + h.check(away.wrist === false && away.strip === true, '5.1 turned away: no wrist card (the strip stays)'); + const noHand = await frame('{"hand": false}'); + h.check(noHand.wrist === false, '5.2 no tracked left hand: no card'); + + console.log('\n=== 6. a POKE presses once and re-arms on retreat ==='); + await frame(); + const poke = await page.evaluate(() => { + const s = window.__stores; + const T = window.__T; + const k = s.gameKit.vrGamePanel; + const at = T.rectPoint('vr-game-wrist', 'footer:strip'); + const surf = k.vrGameSurface('vr-game-wrist'); + const normal = new s.THREE.Vector3(0, 0, 1).applyQuaternion(surf.mesh.quaternion); + const before = k.vrGamePanelDebug().strip; + const first = k.pokeFrame(0, at.clone().addScaledVector(normal, 0.005))?.hit.id ?? null; + const afterFirst = k.vrGamePanelDebug().strip; + const held = k.pokeFrame(0, at.clone().addScaledVector(normal, -0.004))?.hit.id ?? null; + k.pokeFrame(0, at.clone().addScaledVector(normal, 0.1)); // pull back + const again = k.pokeFrame(0, at.clone().addScaledVector(normal, 0.003))?.hit.id ?? null; + return { before, first, afterFirst, held, again, after: k.vrGamePanelDebug().strip }; + }); + h.check(poke.first === 'footer:strip' && poke.before === true && poke.afterFirst === false, '6.1 pushing the tip into "Strip on" presses it (' + JSON.stringify(poke) + ')'); + h.check(poke.held === null, '6.2 holding the tip in the button does not press again'); + h.check(poke.again === 'footer:strip' && poke.after === true, '6.3 pulling back re-arms it: the next push presses'); + + console.log('\n=== 7. the strip toggles ==='); + await page.evaluate(() => window.__stores.gameKit.vrGamePanel.setVrStrip(false)); + const off = await frame(); + h.check(off.strip === false && off.wrist === true, '7.1 strip off: gone, the wrist stays'); + h.check((await page.evaluate(() => localStorage.getItem('vr:gameStrip'))) === 'false', '7.2 persisted locally'); + await page.evaluate(() => window.__stores.gameKit.vrGamePanel.setVrStrip(true)); + + console.log('\n=== 8. Edit mode leaves Interact, and everything goes ==='); + await frame(); + const edit = await page.evaluate(() => { + const s = window.__stores; + const T = window.__T; + const at = T.rectPoint('vr-game-wrist', 'footer:edit'); + T.aim(1, at.clone().add(new s.THREE.Vector3(0.1, 0.25, 0.1)), at); + const consumed = s.vrControls.vrModuleTriggerStart(1); + s.vrControls.vrModuleSelectSwallowed(); + s.vrControls.vrModuleTriggerEnd(1); + let mode; + s.editorMode.subscribe((v) => (mode = v))(); + return { consumed, mode }; + }); + h.check(edit.consumed && edit.mode === 'edit', '8.1 "Edit" on the wrist switched to Edit (' + JSON.stringify(edit) + ')'); + const f8 = await frame(); + h.check(!f8.panel && !f8.wrist && !f8.strip, '8.2 in Edit no game surface shows (' + JSON.stringify(f8) + ')'); + await page.evaluate(() => { + window.__stores.objectActions.setEditorMode('interact'); + window.__stores.gameState.setGameState('menu'); + }); + + console.log('\n=== 9. the panel follows the head LAZILY ==='); + await page.evaluate(() => window.__stores.gameKit.vrGamePanel.hideVrGamePanel()); + await frame('{"yaw": 0}'); + const yaws = await page.evaluate(() => { + const k = window.__stores.gameKit.vrGamePanel; + const T = window.__T; + const pos = () => k.vrGameSurface('vr-game-panel').mesh.position.clone(); + const at0 = pos(); + k.vrGamePanelFrame({ head: T.head(20), hands: [null, null], dt: 0.5 }); + const at20 = pos(); + for (let i = 0; i < 30; i++) k.vrGamePanelFrame({ head: T.head(80), hands: [null, null], dt: 0.1 }); + const at80 = pos(); + const bearing = (p) => (Math.atan2(-p.x, -p.z) * 180) / Math.PI; + return { b0: bearing(at0), b20: bearing(at20), b80: bearing(at80) }; + }); + h.check(Math.abs(yaws.b20 - yaws.b0) < 0.5, '9.1 a 20 degree glance leaves it where it is (' + yaws.b0.toFixed(1) + ' -> ' + yaws.b20.toFixed(1) + ')'); + h.check(yaws.b80 > 40, '9.2 turning 80 degrees brings it round (' + yaws.b80.toFixed(1) + ' degrees)'); + h.check(yaws.b80 > 75, '9.3 ...all the way back in front, not parked at the edge of the band (' + yaws.b80.toFixed(1) + ')'); + + console.log('\n=== 10. the announce banner, head-locked ==='); + const ann = await page.evaluate(async () => { + const k = window.__stores.gameKit.vrGamePanel; + const T = window.__T; + window.__api.announce('Ring 2 reached', { sub: '+50', ms: 500 }); + const on = k.vrGamePanelFrame({ head: T.head(0), hands: [null, null] }).banner; + const pos = k.vrGameSurface('vr-game-announce').mesh.position.toArray(); + await new Promise((r) => setTimeout(r, 700)); + const later = k.vrGamePanelFrame({ head: T.head(0), hands: [null, null] }).banner; + return { on, pos, later }; + }); + h.check(ann.on && Math.abs(ann.pos[0]) < 0.01 && ann.pos[2] < -1.4, '10.1 api.announce shows a banner straight ahead (' + ann.pos.map((n) => n.toFixed(2)) + ')'); + h.check(ann.later === false, '10.2 and it goes after its ms'); + + console.log('\n=== 11. not presenting: nothing ==='); + const flat = await page.evaluate(() => window.__stores.gameKit.vrGamePanel.vrGamePanelFrame({ hands: [null, null] })); + h.check(!flat.panel && !flat.wrist && !flat.strip && !flat.banner, '11.1 with no headset session nothing is drawn (' + JSON.stringify(flat) + ')'); + + console.log('\n=== 12. a toggle flips its value ==='); + await frame(); + const tog = await page.evaluate(() => { + const s = window.__stores; + const T = window.__T; + const at = T.rectPoint('vr-game-panel', 'music'); + T.aim(1, new s.THREE.Vector3(0.2, 1.3, -0.2), at); + const before = s.hudDocs.hudValueOf('music', false); + s.vrControls.vrModuleTriggerStart(1); + s.vrControls.vrModuleSelectSwallowed(); + s.vrControls.vrModuleTriggerEnd(1); + return { before, after: s.hudDocs.hudValueOf('music', false) }; + }); + h.check(tog.before === false && tog.after === true, '12.1 the Music toggle flipped through the laser (' + JSON.stringify(tog) + ')'); + + console.log('\n=== 13. the REAL Towers template ==='); + if (!TOWERS || !fs.existsSync(TOWERS)) console.log('SKIP 13: no games/towers/scene.tpscene'); + else { + const bytes = Array.from(fs.readFileSync(TOWERS)); + await page.evaluate(async (arr) => { + const s = window.__stores; + const payload = await s.sessions.readSessionZip(new Uint8Array(arr).buffer); + await s.sessions.applySession(payload, { backup: false }); + }, bytes); + await page.waitForTimeout(2500); + const towers = await page.evaluate(() => { + const s = window.__stores; + s.gameState.setGameState('menu'); + s.objectActions.setEditorMode('interact'); + const k = s.gameKit.vrGamePanel; + const T = window.__T; + k.hideVrGamePanel(); + const f = k.vrGamePanelFrame({ head: T.head(0), hands: [T.leftHand(true), null] }); + const hits = k.vrGamePanelDebug().hits['vr-game-panel'].filter((x) => x.kind !== 'footer'); + const labels = []; + for (const { screen } of k.vrScreens().panel) for (const el of screen.elements) if (hits.some((x) => x.id === el.id)) labels.push(el.label); + return { panel: f.panel, labels, hits: hits.map((x) => x.id) }; + }); + h.check(!!towers.panel && towers.hits.length > 0, '13.1 the Towers menu is on the VR panel with pressable controls (' + JSON.stringify(towers) + ')'); + const startId = towers.hits.find((id, i) => /start|play/i.test(towers.labels[i] ?? '')) ?? towers.hits[0]; + const started = await page.evaluate(async (id) => { + const s = window.__stores; + const T = window.__T; + const at = T.rectPoint('vr-game-panel', id); + T.aim(1, new s.THREE.Vector3(0.2, 1.3, -0.2), at); + s.vrControls.vrModuleTriggerStart(1); + s.vrControls.vrModuleSelectSwallowed(); + s.vrControls.vrModuleTriggerEnd(1); + await new Promise((r) => setTimeout(r, 400)); + let state; + s.gameState.gameState.subscribe((v) => (state = v.state))(); + const f = s.gameKit.vrGamePanel.vrGamePanelFrame({ head: T.head(0), hands: [T.leftHand(true), null] }); + return { state, f }; + }, startId); + h.check(started.state === 'playing', '13.2 pressing its Start in VR starts the round (' + JSON.stringify(started) + ')'); + h.check(started.f.lines.length > 0, '13.3 the round\'s HUD reads on the wrist/strip (' + JSON.stringify(started.f.lines) + ')'); + h.check( + started.f.stripLines.length > 0 && started.f.stripLines.every((l) => l.length <= 28) && started.f.stripLines.length < started.f.lines.length, + '13.4 the strip keeps the short readouts and leaves the sentence-long hint to the wrist (' + JSON.stringify(started.f.stripLines) + ')' + ); + } + + await h.finish(browser); +}); From fbf517da37a98bb762c7ecddf9de61fda931e452 Mon Sep 17 00:00:00 2001 From: AlexZ005 Date: Wed, 23 Sep 2026 17:51:26 +0300 Subject: [PATCH 5/8] [feat] 30b P4: hold the trigger and sweep (C3) - tip and laser sweep in Interact/Play, ctx.source, {sweep: false} - vrGameInput: while the trigger is held in Interact/Play, the controller TIP (a 6 cm sphere just ahead of the controller) clicks the clickable it becomes NEAREST to, once, spent while still touched and re-armed on leaving; with the tip touching nothing, the LASER does the same for targets out of reach (> 25 cm) and for the game board's buttons. The first entry is the press itself (source 'trigger', fired on the PRESS so an instrument sounds when hit), later entries are 'sweep'. The sweep's hook does not consume; on its first frame it reads vrControls.triggerClaimed(index) and stands down under a gesture another hook claimed (knob drag, cable, sleeve). A press that clicked anything swallows its trailing 'select'. - nearest-only is a deliberate reading of "every clickable the tip ENTERS": a 6 cm sphere is wider than a 2.2 cm key and the literal reading played clusters (QUESTIONS fork 2). - moduleSDK: every click handler gets ctx = {source, mode} (additive); a handler registered with {sweep: false} still hears the press and never a sweep. - vrControls: triggerClaims per slot + triggerClaimed(index) (additive). - Scene: in Interact/Play the VR trigger's final fallthrough is interactClick (handlers, On Click, the miss) instead of raycastSelect - an Interact press selects nothing (30b-vr-modes handed this path to C3). Suite vr-sweep (new): 24/24, including the REAL Jam Room (a held sweep over three piano keys plays key-64, key-65, key-67 once each; over three drum steps flips each). Counterfactuals: - re-arm removed -> 2.1 red; {sweep:false} ignored -> 3.1 red; the swallow never set -> 4.1, 5.2 red; the claim check removed -> 6.1 red; the Interact fallthrough removed -> 9.1 red (the release selected the object) [one run, five broken] - every touched target fired (no nearest rule) -> 11.1 red (["n1","n0","n2","n3","n4","n5","n6"]) svelte-check 334/47, no new message. Build green. Held suites for P3+P4 NOT yet re-run (budget checkpoint; P2's held run was green) - owed before the PR. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/components/Scene.svelte | 8 + src/lib/moduleSDK.js | 34 ++- src/lib/vrControls.js | 12 +- src/lib/vrGameInput.js | 187 +++++++++++++-- tests/e2e/vr-sweep.test.cjs | 455 ++++++++++++++++++++++++++++++++++++ 5 files changed, 676 insertions(+), 20 deletions(-) create mode 100644 tests/e2e/vr-sweep.test.cjs diff --git a/src/components/Scene.svelte b/src/components/Scene.svelte index 10a1cfa4..36d17b7d 100644 --- a/src/components/Scene.svelte +++ b/src/components/Scene.svelte @@ -57,6 +57,7 @@ import { initVRControls, updateVRControls, raycastMenu, raycastPanel, raycastPalette, raycastProps, raycastPrefabs, raycastKeyboard, raycastChat, raycastEdit, raycastSnap, raycastSettings, raycastApprove, placePrefabGhost, vrFaceTrigger, vrVertexTrigger, vrVertexGrabStart, vrVertexGrabEnd, beginStretchSliderDrag, endStretchSliderDrag, executeVRMenuAction, resetWorldRig, onInputSourcesChange, worldToContentPose, boxSelectStart, boxSelectEnd, boxSelectActive, applyVRFrameRate, shouldSendHands, onHandPinchStart, onHandPinchEnd, pinchMenuToggledAt, firePingIfArmed, vrModuleTriggerStart, vrModuleTriggerEnd, vrModuleSelectSwallowed, handSnapshot, vrGrabbedUuid, hapticKnock } from '$lib/vrControls'; // 30b (vr-play): the game in your hands — hover/press haptics (P1), the sweep (P4) import { startVrGameInput, stopVrGameInput } from '$lib/vrGameInput'; + import { gameFeelActive } from '$lib/gameFeel'; import { vrKeyboardTarget } from '$lib/vrKeyboard'; import { measureMode, measureClick } from '$lib/measure'; import { pinsGroup, openAnnotation, showNotePins } from '$lib/annotationsHandler'; @@ -1388,6 +1389,13 @@ tempMatrix.identity().extractRotation(controller.matrixWorld); selectionRaycaster.ray.origin.setFromMatrixPosition(controller.matrixWorld); selectionRaycaster.ray.direction.set(0, 0, -1).applyMatrix4(tempMatrix); + // 30b (C3): in Interact/Play a VR press is a CLICK — module handlers, On Click, + // the miss — and selects nothing, ever (the desktop Interact rule); the sweep + // already fired anything it knew was clickable and swallowed this release + if (gameFeelActive()) { + interactClick(selectionRaycaster); + return; + } raycastSelect(); }; // 182: hold-to-move a vertex — grab on trigger press, drop on release diff --git a/src/lib/moduleSDK.js b/src/lib/moduleSDK.js index 9e0d7a12..c6084ed8 100644 --- a/src/lib/moduleSDK.js +++ b/src/lib/moduleSDK.js @@ -89,7 +89,8 @@ export const moduleMenuItems = writable([]); export const moduleEffects = {}; /** @type {Record} node type -> Svelte component */ export const moduleNodeComponents = {}; -/** @type {((object: any) => boolean)[]} */ +/** 30b: a handler also gets `ctx = {source, mode}` (see runClickHandlers) + * @type {((object: any, ctx?: {source: string, mode: string}) => boolean)[]} */ export const moduleClickHandlers = []; /** 30 P1: the three places a viewport click can come from. */ @@ -112,6 +113,11 @@ export function normalizeClickModes(modes) { return known.length ? known : [...DEFAULT_CLICK_MODES]; } +/** 30b (C3): handlers that asked NOT to be swept (`{sweep: false}`) — a knob you drag, a + * dot you carry. They still hear the press itself; only the later entries of a held + * trigger skip them. @type {WeakSet} */ +const noSweepHandlers = new WeakSet(); + /** Does this handler run for a click in `mode`? A null mode is VR's trigger, which has * no editor mode of its own yet and keeps offering every handler, as it always has. * @param {Function} fn @param {string | null} mode */ @@ -125,13 +131,22 @@ export function clickHandlerRunsIn(fn, mode) { * Offer a clicked mesh to every handler that runs in `mode`, in registration order; the * first to return true consumes the click. ONE dispatch for the editor's pick, Interact * and Play's tap, so the three can never disagree about who hears what. - * @param {any} object @param {string | null} mode @returns {boolean} + * + * 30b (C3): every handler also gets `ctx = {source, mode}` — `source` is 'click' (the + * desktop / a VR release), 'trigger' (the VR press itself, fired on the press) or 'sweep' + * (a later entry while the trigger is held). A 'sweep' skips handlers registered with + * `{sweep: false}`. Additive: a one-argument handler is byte-unchanged. + * @param {any} object @param {string | null} mode @param {{source?: string}} [ctx] + * @returns {boolean} */ -export function runClickHandlers(object, mode) { +export function runClickHandlers(object, mode, ctx = {}) { + const source = ctx?.source ?? 'click'; + const info = { source, mode: mode ?? 'vr' }; for (const handler of [...moduleClickHandlers]) { if (!clickHandlerRunsIn(handler, mode)) continue; + if (source === 'sweep' && noSweepHandlers.has(handler)) continue; try { - if (handler(object)) return true; + if (handler(object, info)) return true; } catch (error) { log('warn', 'module', 'click handler failed', String(error)); } @@ -576,11 +591,18 @@ function makeApi(moduleId, moduleName = moduleId) { * a pad, a puzzle piece) no longer eats the editor's select click. A handler * that is an editor TOOL (a toolbox pick) passes {modes: ['edit']}, or all three. * In Edit an 'edit' handler still runs BEFORE the selection, so it can consume. - * @param {(object: any) => boolean} fn - * @param {{modes?: string[]}} [options] + * + * 30b: `fn(object, ctx)` — `ctx.source` is 'click', 'trigger' (the VR press) or + * 'sweep' (VR, Interact/Play: the trigger HELD and the controller tip or laser + * passing into this mesh — each entry clicks once, re-armed when it leaves). Pass + * `{sweep: false}` for a control that must not be swept (a knob you drag, a dot you + * carry); it still hears the press. + * @param {(object: any, ctx?: {source: string, mode: string}) => boolean} fn + * @param {{modes?: string[], sweep?: boolean}} [options] */ registerClickHandler(fn, options = {}) { clickHandlerModes.set(fn, normalizeClickModes(options?.modes)); + if (options?.sweep === false) noSweepHandlers.add(fn); moduleClickHandlers.push(fn); onDispose(() => arrayRemove(moduleClickHandlers, fn)); }, diff --git a/src/lib/vrControls.js b/src/lib/vrControls.js index 4fa27da7..26a76ccc 100644 --- a/src/lib/vrControls.js +++ b/src/lib/vrControls.js @@ -384,9 +384,17 @@ export function registerVRTriggerHooks(hooks) { if (i >= 0) triggerHooks.splice(i, 1); }; } +/** 30b (C3): did SOME hook take the last trigger press on each slot? The sweep starts + * from a non-consuming hook, so it reads this on its next frame to stand down under a + * gesture another feature claimed (a knob drag, a cable, the sleeve). */ +const triggerClaims = [false, false]; +/** @param {number} index */ +export function triggerClaimed(index) { + return !!triggerClaims[index]; +} /** @param {number} index @returns {boolean} */ export function vrModuleTriggerStart(index) { - return triggerHooks.some((h) => { + const claimed = triggerHooks.some((h) => { try { return !!h.start?.(index); } catch (error) { @@ -394,6 +402,8 @@ export function vrModuleTriggerStart(index) { return false; } }); + triggerClaims[index] = claimed; + return claimed; } /** @param {number} index @returns {boolean} */ export function vrModuleTriggerEnd(index) { diff --git a/src/lib/vrGameInput.js b/src/lib/vrGameInput.js index a8e245e4..ec0ff9c8 100644 --- a/src/lib/vrGameInput.js +++ b/src/lib/vrGameInput.js @@ -23,13 +23,13 @@ import * as THREE from 'three'; import { get } from 'svelte/store'; import { objectsGroup, globalScene, globalRenderer } from '../stores/sceneStore'; -import { gameFeelActive } from './gameFeel'; +import { gameFeelActive, gameClickMode } from './gameFeel'; import { sceneHits } from './scenePick'; import { pickStack, primaryIndex } from './selectThrough'; import { topLevelObjectOf } from './objectActions'; -import { objectHasOnClick } from './flowRuntime'; -import { moduleInteractiveGroups } from './moduleSDK'; -import { registerVRFrameHook, registerVRTriggerHooks, registerPanelGroupProvider, controllerIndexFor, hapticPattern } from './vrControls'; +import { objectHasOnClick, fireObjectClick } from './flowRuntime'; +import { moduleInteractiveGroups, runClickHandlers } from './moduleSDK'; +import { registerVRFrameHook, registerVRTriggerHooks, registerPanelGroupProvider, controllerIndexFor, hapticPattern, triggerClaimed } from './vrControls'; // P3 (C2): the game UI in VR — the panel, the wrist card, the strip, the banner import { vrGamePanelFrame, panelTargetAlong, panelHover, pressPanelTarget, pokeFrame, uAcross, vrGameSurface, hideVrGamePanel } from './vrGamePanel'; @@ -126,7 +126,7 @@ export function handOf(index) { /** what each slot's laser pointed at last frame (entry key) — the hover-enter edge */ const hoverKey = /** @type {(string | null)[]} */ ([null, null]); -const stats = { taps: 0, bumps: 0 }; +const stats = { taps: 0, bumps: 0, sweepClicks: 0 }; /** * One frame of the hover edge for slot `index`: a NEW clickable under the laser taps that @@ -162,9 +162,9 @@ export function pressFeedback(index, ray = controllerRayOf(index)) { return true; } -/** @returns {{taps: number, bumps: number, hover: (string | null)[]}} */ +/** @returns {{taps: number, bumps: number, sweepClicks: number, hover: (string | null)[], sweeps: boolean[]}} */ export function vrGameInputDebug() { - return { ...stats, hover: [...hoverKey] }; + return { ...stats, hover: [...hoverKey], sweeps: sweeps.map(Boolean) }; } /* --------------------------------------------------------------- P3 the panel ---- */ @@ -196,26 +196,184 @@ let swallowNext = false; /** was a headset presenting on the last frame (the session-end edge) */ let presenting = false; +/* ---------------------------------------------------------------- P4 the sweep ---- */ + +// C3 — HOLD THE TRIGGER AND SWEEP. The user: "I want to be able to hold trigger and +// automatically press the buttons where I move the controller around. So, for example, I +// can hit different keys on piano holding trigger and on the mixer enable/disable". +// +// While the trigger is held in Interact/Play, every clickable the controller TIP (a +// ~6 cm sphere just ahead of the controller) ENTERS is clicked once, and re-armed when +// the tip leaves it; with the tip touching nothing, the LASER does the same for things +// out of reach (and for the game board's buttons). The first entry is the press itself +// (`source: 'trigger'`), fired on the PRESS rather than the release, so an instrument +// sounds when you hit it; every later one is `source: 'sweep'`, which a handler can opt +// out of (`registerClickHandler(fn, {sweep: false})`). A press another feature CLAIMED +// (a knob drag, a cable, the sleeve) is left alone — read one frame later, because this +// hook does not consume and cannot see the hooks after it. When the sweep clicked +// anything, the press's trailing 'select' is swallowed: that click already happened. + +/** the tip sphere's radius (~6 cm across) and how far ahead of the controller it sits */ +export const TIP_RADIUS = 0.03; +const TIP_AHEAD = 0.03; +/** the laser sweeps only what the tip cannot reach */ +const LASER_MIN = 0.25; + +/** @typedef {{first: boolean, inside: Set, laserKey: string | null, panelKey: string | null, fired: number}} Sweep */ +/** @type {(Sweep | null)[]} */ +const sweeps = [null, null]; + +const _box = new THREE.Box3(); +const _sphere = new THREE.Sphere(); + /** - * A trigger PRESS on slot `index`: a game-panel button under the laser is pressed and the - * press consumed; anything else only gets its haptic bump and falls through untouched. - * Exported for the suites (the live hook calls it). - * @param {number} index @returns {boolean} consumed + * Every clickable a TIP sphere touches (bounding sphere, then the world box — a key, a + * pad, a step cell is a box). Keyed by entry, so a device part is its own control. + * @param {THREE.Vector3} tip @param {number} [radius] @returns {ClickTarget[]} + */ +export function clickTargetsTouching(tip, radius = TIP_RADIUS) { + /** @type {any[]} */ + const roots = []; + const group = get(objectsGroup); + if (group) roots.push(group); + const scene = /** @type {any} */ (get(globalScene)); + for (const name of moduleInteractiveGroups) { + const root = scene?.getObjectByName(name); + if (root) roots.push(root); + } + /** @type {Map} */ + const found = new Map(); + for (const root of roots) + root.traverseVisible((/** @type {any} */ node) => { + if (!node.isMesh || !node.geometry) return; + const geo = node.geometry; + if (!geo.boundingSphere) geo.computeBoundingSphere(); + _sphere.copy(geo.boundingSphere).applyMatrix4(node.matrixWorld); + if (_sphere.distanceToPoint(tip) > radius) return; + if (!geo.boundingBox) geo.computeBoundingBox(); + _box.copy(geo.boundingBox).applyMatrix4(node.matrixWorld); + if (_box.distanceToPoint(tip) > radius) return; + const distance = _box.distanceToPoint(tip); + const target = clickTargetOf({ object: node, point: tip.clone(), distance }); + const known = target ? found.get(target.key) : null; + if (target && (!known || distance < known.distance)) found.set(target.key, target); + }); + return [...found.values()].sort((a, b) => a.distance - b.distance); +} + +/** + * CLICK a target the way Interact/Play's tap does — the module handlers for this mode + * first, then the object's On Click — and bump the hand when something acted. + * @param {ClickTarget} target @param {string} source @param {number} index @returns {boolean} + */ +export function fireClickTarget(target, source, index) { + let acted = runClickHandlers(target.mesh, gameClickMode(), { source }); + if (!acted && target.top) acted = fireObjectClick(target.top.uuid) > 0; + if (acted) { + hapticPattern('bump', handOf(index)); + stats.sweepClicks++; + } + return acted; +} + +/** the tip sphere's centre for slot `index` @param {number} index */ +function tipAhead(index) { + const ray = controllerRayOf(index); + return ray ? ray.ray.origin.clone().addScaledVector(ray.ray.direction, TIP_AHEAD) : null; +} + +/** + * One frame of a held trigger on slot `index`. `opts.tip`/`opts.ray` inject the pose (the + * suites; no headset headless). Returns the entry keys this frame clicked, or null when + * no sweep is live. + * @param {number} index @param {{tip?: THREE.Vector3 | null, ray?: THREE.Raycaster | null}} [opts] + * @returns {string[] | null} + */ +export function sweepFrame(index, opts = {}) { + const sweep = sweeps[index]; + if (!sweep) return null; + if (!gameFeelActive() || (sweep.first && triggerClaimed(index))) { + sweeps[index] = null; + return null; + } + const source = sweep.first ? 'trigger' : 'sweep'; + sweep.first = false; + const tip = opts.tip === undefined ? tipAhead(index) : opts.tip; + const ray = opts.ray === undefined ? controllerRayOf(index) : opts.ray; + /** @type {string[]} */ + const fired = []; + // the tip plays what it is NEAREST to: a 6 cm sphere is wider than a piano key, and + // "every key it touches" would sound clusters. A target fires when it becomes the + // nearest touching one; it stays spent while the tip still touches it, and LEAVING + // re-arms it. + const touching = tip ? clickTargetsTouching(tip) : []; + const nearest = touching[0] ?? null; + const still = new Set([...sweep.inside].filter((key) => touching.some((t) => t.key === key))); + if (nearest && !still.has(nearest.key)) { + if (fireClickTarget(nearest, source, index)) fired.push(nearest.key); + still.add(nearest.key); + } + sweep.inside = still; + if (!touching.length && ray) { + const board = panelTargetAlong(ray); + const boardKey = board?.hit.id ?? null; + if (board && boardKey !== sweep.panelKey && pressPanelTarget(board, { source, u: uAcross(board) })) { + hapticPattern('bump', handOf(index)); + fired.push(board.hit.id); + } + sweep.panelKey = boardKey; + const far = board ? null : clickTargetAlong(ray); + const farKey = far && far.distance > LASER_MIN ? far.key : null; + if (far && farKey && farKey !== sweep.laserKey && fireClickTarget(far, source, index)) fired.push(farKey); + sweep.laserKey = farKey; + } else { + sweep.laserKey = null; + sweep.panelKey = null; + } + if (fired.length) { + sweep.fired += fired.length; + swallowNext = true; + } + return fired; +} + +/** + * A trigger PRESS on slot `index`. In Interact/Play: a game-board button under the laser + * is pressed at once and the press consumed (the sweep carries on across the board); any + * other press starts a SWEEP, which fires on the next frame once it knows no other + * gesture claimed this press. Returns whether the press was consumed. Exported for the + * suites (the live hook calls it). + * @param {number} index @returns {boolean} */ export function triggerStart(index) { swallowNext = false; + sweeps[index] = null; + if (!gameFeelActive()) return false; const ray = controllerRayOf(index); - const target = ray && gameFeelActive() ? panelTargetAlong(ray) : null; + const target = ray ? panelTargetAlong(ray) : null; if (target) { pressPanelTarget(target, { source: 'trigger', u: uAcross(target) }); hapticPattern('bump', handOf(index)); swallowNext = true; + sweeps[index] = { first: false, inside: new Set(), laserKey: null, panelKey: target.hit.id, fired: 1 }; return true; } - pressFeedback(index); + sweeps[index] = { first: true, inside: new Set(), laserKey: null, panelKey: null, fired: 0 }; return false; } +/** The trigger RELEASED on slot `index`: the sweep ends (the swallow stays armed for the + * trailing 'select', which WebXR fires before 'selectend'). @param {number} index */ +export function triggerEnd(index) { + sweeps[index] = null; + return false; +} + +/** is a sweep live on slot `index`? (suites) @param {number} index */ +export function sweepLive(index) { + return !!sweeps[index]; +} + /** @type {(() => void)[]} */ let offs = []; @@ -240,10 +398,12 @@ export function startVrGameInput() { if (panelHover(index, gameFeelActive() ? ray : null)) hoverKey[index] = null; else hoverFrame(index, ray); pokeFrame(index, gameFeelActive() ? tipOf(index) : null); + sweepFrame(index); } }), registerVRTriggerHooks({ start: (/** @type {number} */ index) => triggerStart(index), + end: (/** @type {number} */ index) => triggerEnd(index), swallow: () => { if (!swallowNext) return false; swallowNext = false; @@ -267,5 +427,6 @@ export function stopVrGameInput() { for (const off of offs) off(); offs = []; hoverKey[0] = hoverKey[1] = null; + sweeps[0] = sweeps[1] = null; hideVrGamePanel(); } diff --git a/tests/e2e/vr-sweep.test.cjs b/tests/e2e/vr-sweep.test.cjs new file mode 100644 index 00000000..057353e6 --- /dev/null +++ b/tests/e2e/vr-sweep.test.cjs @@ -0,0 +1,455 @@ +// 30b (vr-play) P4 — C3 HOLD THE TRIGGER AND SWEEP. The user, on a Quest 3: "I want to be able +// to hold trigger and automatically press the buttons where I move the controller around. So, +// for example, I can hit different keys on piano holding trigger and on the mixer +// enable/disable, toggle multiple knobs." +// +// Headless: controllers are posed by hand (matrixAutoUpdate on), the press goes through the +// REAL trigger hooks (vrModuleTriggerStart / End / SelectSwallowed) and each frame through the +// exported sweepFrame, with the tip / laser injected. +// 1 the tip clicks every key it ENTERS, once, the first as the 'trigger' press · 2 leaving +// re-arms · 3 a {sweep: false} handler hears the press and never a sweep · 4 the release is +// swallowed after a sweep, and only then · 5 the LASER sweeps what the tip cannot reach · +// 6 a press another gesture CLAIMED is left alone · 7 Edit mode: no sweep, the old path · +// 8 the game board's buttons sweep too · 9 an Interact release on a plain object CLICKS and +// never selects · 10 the REAL Jam Room: a sweep across piano keys plays each, across drum +// steps flips each. +const h = require('./helpers.cjs'); +const fs = require('fs'); +const path = require('path'); + +const SCENES_REPO = [path.resolve(__dirname, '../../../theprototype.app-scenes'), path.resolve(__dirname, '../../../scenes')].find((p) => + fs.existsSync(p) +); +const JAM = process.env.JAM_ROOM_TPSCENE || (SCENES_REPO && path.join(SCENES_REPO, 'games/jam-room/scene.tpscene')); + +h.run(async () => { + const browser = await h.launch({ args: h.AUDIO_ARGS }); + const A = await h.setupPage(browser, 'A'); + const page = A.page; + + await page.evaluate(async () => { + const s = window.__stores; + window.__log = []; + await s.moduleSDK.initModules([ + { + id: 'sweep30b', + name: 'Sweep test', + version: '1.0.0', + description: 'the 30b sweep', + register(api) { + window.__api = api; + // a keyboard of five keys: this handler plays them, sweeps included + api.registerClickHandler((object, ctx) => { + if (!object.userData?.sweepKey) return false; + window.__log.push({ key: object.userData.sweepKey, source: ctx?.source, mode: ctx?.mode }); + return true; + }); + // a dot you CARRY: it must hear the press and never a sweep + api.registerClickHandler( + (object, ctx) => { + if (!object.userData?.dot) return false; + window.__log.push({ key: 'dot', source: ctx?.source }); + return true; + }, + { sweep: false } + ); + } + } + ]); + let r; + s.globalRenderer.subscribe((v) => (r = v))(); + for (const i of [0, 1]) { + const c = r.xr.getController(i); + c.matrixAutoUpdate = true; + c.userData.handedness = i === 0 ? 'left' : 'right'; + } + // five keys in a row, 12 cm apart, marked clickable, at hand height + const THREE = s.THREE; + let g; + s.objectsGroup.subscribe((v) => (g = v))(); + const keys = []; + for (let i = 0; i < 5; i++) { + const [uuid] = await window.__api.create('/create box 0.08 0.04 0.2', { at: [-0.24 + i * 0.12, 1.0, -0.5] }); + keys.push(uuid); + } + const [dot] = await window.__api.create('/create box 0.08 0.08 0.08', { at: [0.5, 1.0, -0.5] }); + const [plain] = await window.__api.create('/create box 0.3 0.3 0.3', { at: [0, 1.2, -3] }); + await new Promise((res) => setTimeout(res, 600)); + keys.forEach((uuid, i) => { + const o = g.getObjectByProperty('uuid', uuid); + o.userData.clickable = true; + o.traverse((n) => n.isMesh && (n.userData.sweepKey = 'k' + i)); + o.userData.physics = { mode: 'static' }; + }); + const d = g.getObjectByProperty('uuid', dot); + d.userData.clickable = true; + d.traverse((n) => n.isMesh && (n.userData.dot = true)); + g.updateMatrixWorld(true); + window.__ids = { keys, dot, plain }; + s.objectActions.deselectObject?.(); + s.objectActions.setEditorMode('interact'); + window.__T = { + keyAt: (i) => g.getObjectByProperty('uuid', keys[i]).getWorldPosition(new THREE.Vector3()), + aim: (index, from, to) => { + const c = r.xr.getController(index); + c.position.copy(from); + c.lookAt(from.clone().multiplyScalar(2).sub(to)); // a Group's +Z faces the target + c.updateMatrixWorld(true); + }, + // a sweep frame with the TIP at a point and no laser + tipFrame: (index, point) => s.gameKit.vrGameInput.sweepFrame(index, { tip: point, ray: null }), + press: (index) => s.vrControls.vrModuleTriggerStart(index), + release: (index) => ({ swallowed: s.vrControls.vrModuleSelectSwallowed(), end: s.vrControls.vrModuleTriggerEnd(index) }) + }; + }); + const log = () => page.evaluate(() => window.__log.splice(0)); + + console.log('\n=== 1. the tip clicks every key it enters, once ==='); + const run1 = await page.evaluate(() => { + const T = window.__T; + const THREE = window.__stores.THREE; + const above = (i) => T.keyAt(i).add(new THREE.Vector3(0, 0.035, 0)); // just touching the top + T.aim(1, above(0).add(new THREE.Vector3(0, 0.05, 0.2)), above(0)); + const consumed = T.press(1); + const frames = []; + frames.push(T.tipFrame(1, above(0))); // the press itself + frames.push(T.tipFrame(1, above(0))); // still on k0 + frames.push(T.tipFrame(1, above(1))); + frames.push(T.tipFrame(1, above(1))); + frames.push(T.tipFrame(1, above(2))); + frames.push(T.tipFrame(1, above(3))); + return { consumed, frames }; + }); + const l1 = await log(); + h.check(run1.consumed === false, '1.1 the press is NOT consumed by the sweep (other gestures may still take it)'); + h.check(JSON.stringify(l1.map((e) => e.key)) === JSON.stringify(['k0', 'k1', 'k2', 'k3']), '1.2 four keys entered, four clicks, each once (' + JSON.stringify(l1.map((e) => e.key)) + ')'); + h.check(l1[0]?.source === 'trigger' && l1.slice(1).every((e) => e.source === 'sweep'), '1.3 the first is the press (trigger), the rest are sweeps (' + l1.map((e) => e.source) + ')'); + h.check(l1.every((e) => e.mode === 'interact'), '1.4 dispatched as Interact clicks'); + + console.log('\n=== 2. leaving re-arms ==='); + const run2 = await page.evaluate(() => { + const T = window.__T; + const THREE = window.__stores.THREE; + const above = (i) => T.keyAt(i).add(new THREE.Vector3(0, 0.035, 0)); + T.tipFrame(1, above(3).add(new THREE.Vector3(0, 0.3, 0))); // lift off + T.tipFrame(1, above(3)); // back down on k3 + T.tipFrame(1, above(2)); // and k2 again + }); + void run2; + const l2 = await log(); + h.check(JSON.stringify(l2.map((e) => e.key)) === JSON.stringify(['k3', 'k2']), '2.1 lifting off and coming back plays the key again (' + JSON.stringify(l2.map((e) => e.key)) + ')'); + + console.log('\n=== 3. {sweep: false} hears the press, never a sweep ==='); + const dotRun = await page.evaluate(() => { + const s = window.__stores; + const T = window.__T; + const THREE = s.THREE; + const dot = s.gameKit.vrGameInput; + let g; + s.objectsGroup.subscribe((v) => (g = v))(); + const at = g.getObjectByProperty('uuid', window.__ids.dot).getWorldPosition(new THREE.Vector3()); + T.release(1); + window.__log.splice(0); + // a held press that starts on k4 (premise: it plays), then is swept INTO the dot + const k4 = T.keyAt(4).add(new THREE.Vector3(0, 0.035, 0)); + T.aim(1, k4.clone().add(new THREE.Vector3(0, 0.05, 0.2)), k4); + T.press(1); + T.tipFrame(1, k4); + T.tipFrame(1, at.clone()); + const swept = window.__log.splice(0); + T.release(1); + // pressed ON the dot: heard + T.aim(1, at.clone().add(new THREE.Vector3(0, 0.05, 0.2)), at); + T.press(1); + T.tipFrame(1, at.clone()); + const pressed = window.__log.splice(0); + T.release(1); + return { swept, pressed, live: dot.sweepLive(1) }; + }); + h.check( + dotRun.swept.length === 1 && dotRun.swept[0].key === 'k4', + '3.1 a held sweep plays k4, then passes into the {sweep:false} dot without clicking it (' + JSON.stringify(dotRun.swept) + ')' + ); + h.check(dotRun.pressed.length === 1 && dotRun.pressed[0].source === 'trigger', '3.2 a press ON it does (' + JSON.stringify(dotRun.pressed) + ')'); + h.check(dotRun.live === false, '3.3 the release ended the sweep'); + + console.log('\n=== 4. the release is swallowed after a sweep, and only then ==='); + const sw = await page.evaluate(() => { + const T = window.__T; + const THREE = window.__stores.THREE; + const above = (i) => T.keyAt(i).add(new THREE.Vector3(0, 0.035, 0)); + T.press(1); + T.tipFrame(1, above(4)); + const after = T.release(1); + window.__log.splice(0); + T.press(1); + T.tipFrame(1, new THREE.Vector3(3, 3, 3)); // nothing there + const empty = T.release(1); + return { after: after.swallowed, empty: empty.swallowed }; + }); + h.check(sw.after === true, '4.1 a press that swept something swallows its trailing select'); + h.check(sw.empty === false, '4.2 a press that clicked nothing lets its select through'); + + console.log('\n=== 5. the laser sweeps what the tip cannot reach ==='); + const laser = await page.evaluate(() => { + const s = window.__stores; + const T = window.__T; + const THREE = s.THREE; + const k = s.gameKit.vrGameInput; + const from = new THREE.Vector3(0, 1.6, 1.5); + T.aim(1, from, T.keyAt(0)); + T.press(1); + const hits = []; + for (let i = 0; i < 5; i++) { + T.aim(1, from, T.keyAt(i)); + hits.push(k.sweepFrame(1, { tip: from.clone() })); + T.aim(1, from, T.keyAt(i)); // a second frame on the same key + hits.push(k.sweepFrame(1, { tip: from.clone() })); + } + const r = T.release(1); + return { hits, swallowed: r.swallowed, log: window.__log.splice(0) }; + }); + h.check(JSON.stringify(laser.log.map((e) => e.key)) === JSON.stringify(['k0', 'k1', 'k2', 'k3', 'k4']), '5.1 the laser, held and moved along the row from 2 m, clicks each key once (' + JSON.stringify(laser.log.map((e) => e.key)) + ')'); + h.check(laser.swallowed === true, '5.2 and swallows its release'); + + console.log('\n=== 6. a press another gesture claimed is left alone ==='); + const claimed = await page.evaluate(() => { + const s = window.__stores; + const T = window.__T; + const THREE = s.THREE; + const above = (i) => T.keyAt(i).add(new THREE.Vector3(0, 0.035, 0)); + // a knob-drag-like hook registered AFTER ours: it takes this press + const off = s.vrControls.registerVRTriggerHooks({ start: () => true }); + const consumed = T.press(1); + const frame = T.tipFrame(1, above(1)); + const live = s.gameKit.vrGameInput.sweepLive(1); + off(); + T.release(1); + return { consumed, frame, live, log: window.__log.splice(0) }; + }); + h.check(claimed.consumed === true && claimed.log.length === 0 && claimed.live === false, '6.1 the sweep stands down under a claimed gesture (' + JSON.stringify(claimed) + ')'); + + console.log('\n=== 7. Edit mode: no sweep ==='); + const edit = await page.evaluate(() => { + const s = window.__stores; + const T = window.__T; + const THREE = s.THREE; + const above = (i) => T.keyAt(i).add(new THREE.Vector3(0, 0.035, 0)); + s.objectActions.setEditorMode('edit'); + const consumed = T.press(1); + const frame = T.tipFrame(1, above(0)); + const r = T.release(1); + s.objectActions.setEditorMode('interact'); + return { consumed, frame, swallowed: r.swallowed, log: window.__log.splice(0) }; + }); + h.check(edit.consumed === false && edit.frame === null && edit.log.length === 0 && edit.swallowed === false, '7.1 in Edit a held trigger sweeps nothing and the select passes (' + JSON.stringify(edit) + ')'); + + console.log('\n=== 8. the game board\'s buttons sweep too ==='); + const board = await page.evaluate(async () => { + const s = window.__stores; + const T = window.__T; + const THREE = s.THREE; + s.hudDocs.setHudDocFor('scene', { + active: 'pads', + screens: [ + { + id: 'pads', + name: 'Pads', + input: 'menu', + elements: [0, 1, 2].map((i) => ({ id: 'pad' + i, kind: 'toggle', anchor: 'center', x: -220 + i * 220, y: 0, w: 180, h: 80, label: 'Pad ' + (i + 1), value: false })) + } + ] + }); + const k = s.gameKit.vrGamePanel; + const head = { position: new THREE.Vector3(0, 1.6, 0), quaternion: new THREE.Quaternion() }; + k.hideVrGamePanel(); + k.vrGamePanelFrame({ head, hands: [null, null] }); + const surf = k.vrGameSurface('vr-game-panel'); + const pt = (id) => { + const rect = k.vrGamePanelDebug().hits['vr-game-panel'].find((x) => x.id === id); + const g = surf.mesh.geometry.parameters; + return surf.mesh.localToWorld(new THREE.Vector3(((rect.x + rect.w / 2) / surf.canvas.width - 0.5) * g.width, (0.5 - (rect.y + rect.h / 2) / surf.canvas.height) * g.height, 0)); + }; + const from = new THREE.Vector3(0.2, 1.3, -0.3); + T.aim(1, from, pt('pad0')); + const consumed = T.press(1); + for (const id of ['pad0', 'pad1', 'pad1', 'pad2']) { + T.aim(1, from, pt(id)); + s.gameKit.vrGameInput.sweepFrame(1, { tip: from.clone() }); + } + const r = T.release(1); + const values = [0, 1, 2].map((i) => s.hudDocs.hudValueOf('pad' + i, false)); + s.hudDocs.setHudDocFor('scene', null); + k.hideVrGamePanel(); + return { consumed, values, swallowed: r.swallowed }; + }); + h.check(board.consumed === true && JSON.stringify(board.values) === '[true,true,true]', '8.1 one press swept across three board toggles flips each once (' + JSON.stringify(board) + ')'); + + console.log('\n=== 9. an Interact release on a plain object clicks, never selects ==='); + const release = await page.evaluate(async () => { + const s = window.__stores; + const T = window.__T; + const THREE = s.THREE; + let r, g; + s.globalRenderer.subscribe((v) => (r = v))(); + s.objectsGroup.subscribe((v) => (g = v))(); + const target = g.getObjectByProperty('uuid', window.__ids.plain).getWorldPosition(new THREE.Vector3()); + const c = r.xr.getController(1); + T.aim(1, new THREE.Vector3(0, 1.4, 0), target); + s.objectActions.deselectObject?.(); + const sel = () => { + let v; + s.selectedObjects.subscribe((x) => (v = x))(); + return [...v]; + }; + c.dispatchEvent({ type: 'select', target: c }); + const inInteract = sel(); + s.objectActions.setEditorMode('edit'); + c.dispatchEvent({ type: 'select', target: c }); + const inEdit = sel(); + s.objectActions.deselectObject?.(); + s.objectActions.setEditorMode('interact'); + return { inInteract, inEdit, plain: window.__ids.plain }; + }); + h.check(release.inInteract.length === 0, '9.1 in Interact the VR release selected nothing (' + JSON.stringify(release.inInteract) + ')'); + h.check(release.inEdit.includes(release.plain), '9.2 in Edit the same release selects it, as before'); + + console.log('\n=== 11. a real-width keyboard: the tip plays the key it is NEAREST ==='); + const narrow = await page.evaluate(async () => { + const s = window.__stores; + const T = window.__T; + const THREE = s.THREE; + let g; + s.objectsGroup.subscribe((v) => (g = v))(); + // seven keys 2.2 cm wide on a 2.4 cm pitch — narrower than the 6 cm tip, like a piano + const keys = []; + for (let i = 0; i < 7; i++) { + const [uuid] = await window.__api.create('/create box 0.022 0.02 0.12', { at: [-0.072 + i * 0.024, 1.3, -0.8] }); + keys.push(uuid); + } + await new Promise((res) => setTimeout(res, 600)); + keys.forEach((uuid, i) => { + const o = g.getObjectByProperty('uuid', uuid); + o.userData.clickable = true; + o.userData.physics = { mode: 'static' }; + o.traverse((n) => n.isMesh && (n.userData.sweepKey = 'n' + i)); + }); + g.updateMatrixWorld(true); + window.__log.splice(0); + const top = (i) => g.getObjectByProperty('uuid', keys[i]).getWorldPosition(new THREE.Vector3()).add(new THREE.Vector3(0, 0.012, 0)); + T.aim(1, top(1).add(new THREE.Vector3(0, 0.05, 0.2)), top(1)); + T.press(1); + // glide along the keys, several frames per key, centred over keys 1..5 + for (let i = 1; i <= 5; i++) for (let f = 0; f < 3; f++) T.tipFrame(1, top(i)); + T.release(1); + return window.__log.splice(0).map((e) => e.key); + }); + h.check(JSON.stringify(narrow) === JSON.stringify(['n1', 'n2', 'n3', 'n4', 'n5']), '11.1 gliding over five 2.2 cm keys plays those five, one each, no clusters (' + JSON.stringify(narrow) + ')'); + + console.log('\n=== 10. the REAL Jam Room: piano keys and drum steps ==='); + const zip = (id) => h.installModule(A, id); + if (!JAM || !fs.existsSync(JAM)) console.log('SKIP 10: no games/jam-room/scene.tpscene'); + else if (!(await zip('music-lab')) || !(await zip('music-fx'))) console.log('SKIP 10: no music-lab / music-fx zips'); + else { + await page.evaluate(() => window.__stores.modulesOpen.set(false)); + const bytes = Array.from(fs.readFileSync(JAM)); + await page.evaluate(async (arr) => { + const s = window.__stores; + const payload = await s.sessions.readSessionZip(new Uint8Array(arr).buffer); + await s.sessions.applySession(payload, { backup: false }); + }, bytes); + await page.waitForTimeout(3000); + const jam = await page.evaluate(async () => { + const s = window.__stores; + const THREE = s.THREE; + s.objectActions.setEditorMode('interact'); + let g; + s.objectsGroup.subscribe((v) => (g = v))(); + g.updateMatrixWorld(true); + // every piano key, in order of x; every drum STEP cell of the first row + const keys = []; + const steps = []; + g.traverse((o) => { + if (!o.isMesh) return; + if (/^key-\d+$/.test(o.name) || typeof o.userData?.midi === 'number') keys.push(o); + if (/step/i.test(o.name) && o.parent) steps.push(o); + }); + keys.sort((a, b) => a.getWorldPosition(new THREE.Vector3()).x - b.getWorldPosition(new THREE.Vector3()).x); + // spy the dispatch: which handler consumed which mesh, with which source + const consumed = []; + const handlers = s.moduleSDK.moduleClickHandlers; + for (let i = 0; i < handlers.length; i++) { + const fn = handlers[i]; + if (fn.__spied) continue; + const wrapped = (o, ctx) => { + const r = fn(o, ctx); + if (r) consumed.push({ name: o.name, source: ctx?.source }); + return r; + }; + wrapped.__spied = true; + handlers[i] = wrapped; + } + const T = window.__T; + const topOf = (o) => { + const b = new THREE.Box3().setFromObject(o); + return new THREE.Vector3((b.min.x + b.max.x) / 2, b.max.y + 0.01, (b.min.z + b.max.z) / 2); + }; + // three neighbouring WHITE keys: the tip plays whichever it is nearest + const white = keys.filter((k) => ![1, 3, 6, 8, 10].includes(((Number(k.userData?.midi ?? k.name.slice(4)) % 12) + 12) % 12)); + const pick = [white[2], white[3], white[4]].filter(Boolean); + T.aim(1, topOf(pick[0]).add(new THREE.Vector3(0, 0.05, 0.2)), topOf(pick[0])); + T.press(1); + for (const key of pick) { + T.tipFrame(1, topOf(key)); + T.tipFrame(1, topOf(key)); + } + T.release(1); + const keyClicks = consumed.splice(0); + // the drum machine: the step cells of one row, swept left to right + const drums = []; + g.traverse((o) => { + if (o.userData?.device?.kind && /drums/.test(o.userData.device.kind)) drums.push(o); + }); + const drum = drums[0]; + const patternOf = () => String(drum?.userData?.device?.params?.pattern ?? ''); + const cells = []; + drum?.traverse((o) => o.isMesh && /^step-\d+-\d+$/.test(o.name) && cells.push(o)); + // one row (pad 0), every other step, so the tip never sits on two cells at once + const cellPick = cells + .filter((c) => c.name.startsWith('step-0-')) + .sort((a, b) => Number(a.name.split('-')[2]) - Number(b.name.split('-')[2])) + .filter((c) => Number(c.name.split('-')[2]) % 2 === 0) + .slice(0, 3); + const before = patternOf(); + if (cellPick[0]) { + T.aim(1, topOf(cellPick[0]).add(new THREE.Vector3(0, 0.05, 0.2)), topOf(cellPick[0])); + T.press(1); + for (const cell of cellPick) T.tipFrame(1, topOf(cell)); + T.release(1); + } + await new Promise((r) => setTimeout(r, 1500)); // the stroke commits after it idles + const after = patternOf(); + let diff = 0; + for (let i = 0; i < Math.max(before.length, after.length); i++) if (before[i] !== after[i]) diff++; + return { + keys: keys.length, + picked: pick.map((k) => k.name), + keyClicks, + cells: cells.length, + cellNames: cellPick.map((c) => c.name), + stepClicks: consumed.splice(0), + diff, + before: before.slice(0, 20), + after: after.slice(0, 20) + }; + }); + h.check(jam.keys > 10, '10.1 premise: the Jam Room piano is there (' + jam.keys + ' keys)'); + h.check( + jam.keyClicks.length === 3 && jam.keyClicks.every((c, i) => c.name === jam.picked[i]) && jam.keyClicks[0].source === 'trigger' && jam.keyClicks[2].source === 'sweep', + '10.2 one held sweep across three piano keys plays each, once (' + JSON.stringify(jam.keyClicks) + ')' + ); + h.check(jam.stepClicks.length === 3 && jam.diff >= 3, '10.3 a sweep across three drum steps flips each (' + JSON.stringify({ cells: jam.cellNames, clicks: jam.stepClicks.length, diff: jam.diff }) + ')'); + } + + await h.finish(browser); +}); From e89aed5d5bc4411308482adff5c91baa77aeb632 Mon Sep 17 00:00:00 2001 From: AlexZ005 Date: Wed, 23 Sep 2026 17:52:22 +0300 Subject: [PATCH 6/8] [docs] 30b P5: MODULES.md - game feel (sound, music, haptics, effects, banners), the VR board, hold-and-sweep - a "Game feel" section with a ten-line example: api.playSound's game set (and the unknown-name no-op), api.music, api.hapticPattern (and Edit silence + the core defaults), api.effects.burst, api.announce; how a game's HUD reaches the VR board, wrist card and strip with nothing to do on the module side; the hold-and-sweep, what counts as clickable, ctx = {source, mode} and {sweep: false}. - the Interaction section: the VR trigger now follows the editor modes. - Misc: api.hapticPattern beside api.haptic. Co-Authored-By: Claude Opus 5.5 (1M context) --- MODULES.md | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/MODULES.md b/MODULES.md index f72638a3..58c0b736 100644 --- a/MODULES.md +++ b/MODULES.md @@ -243,7 +243,10 @@ api.registerClickHandler(pressKey, { modes: ['interact', 'play'] }); // the defa api.registerClickHandler(pickTool, { modes: ['edit'] }); // an editor TOOL ``` -The VR trigger has no editor mode yet and still offers every handler. +In a headset the same modes apply (1.17, 30b): Play in VR enters Interact, where the +trigger clicks (module handlers, On Click) and selects nothing; in Edit it selects as +before. Handlers also get `ctx = {source, mode}` — see *Game feel* below for the +hold-and-sweep and `{sweep: false}`. `api.editorMode()` reads the mode on this screen (`'edit'` | `'interact'`) — for a module whose own pointer listeners run outside click routing (a drag), so it can stand down in Edit. @@ -478,6 +481,76 @@ if (api.storage) { Feature-detect it; on an older app write the SAME key yourself (`localStorage['tp:mod::'] = JSON.stringify(value)`) so progress carries over. +### Game feel: sound, music, haptics, effects, banners (1.17, roadmap 30b) + +Everything here is LOCAL to the device it runs on — broadcast your own op +(`api.send`) when every peer should hear, feel or see it — and all of it is +feature-detected, so a module written against it still loads on an older app. + +```js +register(api) { + api.registerClickHandler((object, ctx) => { + if (!object.userData.ring) return false; + const at = object.getWorldPosition(new api.THREE.Vector3()).toArray(); + api.playSound?.('success', at); + api.effects?.burst(at, { kind: 'sparkle' }); + api.announce?.('Ring ' + object.userData.ring + ' reached', { sub: '+50' }); + api.hapticPattern?.('success'); // both hands; 'left' | 'right' for one + return true; + }); + api.game.onChange?.(() => (api.game.roundUnderway() ? api.music?.play('arcade') : api.music?.stop())); +} +``` + +- **`api.playSound(name, position?)`** → `true` when a sound started. Twenty + procedural game sounds, no assets: `click` `pop` `whoosh` `success` `fail` `hit` + `kick` `shoot` `laser` `explosion` `coin` `levelup` `goal` `whistle` `cheer` + `step` `ring` `sparkle` `hurt` `portal`, plus the ping chimes `ding` (the default), + `chime`, `pluck`, `bell`. An unknown name is a quiet no-op (it used to play the + ding). `position` spatialises it. Played at the player's "Game sounds" volume. +- **`api.music.play(preset, {volume?})` / `api.music.stop()`** — a looping + procedural track under the effects, at the player's "Music" volume: `arcade` + `ambient` `dungeon` `stadium` `space` `puzzle` `studio`. It is TEMPO-SYNCED to the + session clock (two peers on one preset hear the same bar), plays only in Interact + or Play (`false` from the editor) and stops by itself when the player goes back to + editing. `api.music.current()` names what is playing; `api.music.presets()` lists them. +- **`api.hapticPattern(name, hand?)`** — `tap` `bump` `hit` `success` `fail` + `rumble` `heartbeat` on the VR controllers. `api.haptic(intensity, ms, hand?)` + still exists. BOTH are silent in Edit mode (vibration is for playing), and core + already plays the defaults for you in Interact/Play: a `tap` when the laser enters + something clickable, a `bump` when a press lands, a `hit` on a grab, and a knock + sized by how hard the hand hit. +- **`api.effects.burst([x, y, z], {kind?, color?, count?})`** — a short, pooled + particle burst at the scene root: `sparkle` (default), `confetti`, `smoke`, + `sparks`; `count` 1..96. Nothing is saved or replicated. +- **`api.announce(text, {sub?, ms?, color?})`** — a big centred banner ("GOAL!", + "Level 3"): the desktop HUD, and in a headset a banner fixed in front of the + player. A new one replaces the one showing; `ms` 300..15000 (default 1800). + +**In VR, your game's HUD is in the player's hands.** A screen with `input: 'menu'`, +a control on it, or bound to the `menu` / `paused` / `over` game state is drawn on +a board ~1.2 m in front of the player that follows their head lazily; its buttons +work with the laser and trigger and with a poke. The other screens (score, timer, +level) read as short lines on a card on the left wrist and a strip across the top +of the view. The board and the wrist card always carry an **Edit mode** button. +Nothing to do on your side — author the HUD once and it works on both. + +**Hold the trigger and sweep (VR, Interact/Play).** While the trigger is held, the +controller TIP clicks each clickable it passes over — the one it is NEAREST to, so a +glide along a keyboard plays one key at a time — once, and again after it leaves and +comes back: piano keys, pads, toggles, HUD buttons; the laser does the same for things +out of reach. "Clickable" (also what earns the hover `tap`) means: under a group you +passed to `registerInteractiveGroup`, a part of an audio device (`userData.device` on +an ancestor — each key/pad is its own control), an object an On Click node targets, or +an object you mark `userData.clickable = true`. Your handler gets a second argument +`ctx = {source, mode}`: `source` is `'click'` (desktop), `'trigger'` (the VR press +itself) or `'sweep'` (a later entry while held). A control that must not be swept +(a knob you drag, a dot you carry) opts out: + +```js +api.registerClickHandler(pickDot, { sweep: false }); // presses still reach it; sweeps do not +``` + ### Physics (P-A) All mutations are INITIATOR-ONLY — the peer that started the simulation steps @@ -625,8 +698,9 @@ api.scene(); // THREE.Scene api.objectsGroup(); // the replicated objects root (add scene content here) api.peerId(); // our peer id (undefined before the mesh is up) api.toast('hi'); // toast in the corner -api.haptic(0.6, 60); // buzz the VR controllers (no-op on desktop); +api.haptic(0.6, 60); // buzz the VR controllers (no-op on desktop and in Edit); api.haptic(0.6, 60, 'right'); // optional hand targets one controller (17-A1) +api.hapticPattern('success'); // a named pattern (1.17, see Game feel) api.isVR(); // true inside a VR session api.vrHand('left'); // one hand's WORLD pose + buttons, or null: // {position, quaternion, trigger, gripped, connected} From ee0378a556226f61c61983cdc366efeead71c08f Mon Sep 17 00:00:00 2001 From: AlexZ005 Date: Wed, 23 Sep 2026 17:53:02 +0300 Subject: [PATCH 7/8] [docs] 30b P5: refresh static/llms-full.txt from MODULES.md (npm run sync-llms) Co-Authored-By: Claude Opus 5.5 (1M context) --- static/llms-full.txt | 130 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/static/llms-full.txt b/static/llms-full.txt index cb43b90b..4e569f61 100644 --- a/static/llms-full.txt +++ b/static/llms-full.txt @@ -282,6 +282,31 @@ api.registerClickHandler((object) => { Handlers see desktop clicks and VR trigger presses (the exact mesh hit, not the top-level group). Return `false` to let normal selection continue. +**Modes (1.17).** The editor has two click modes, Edit (every click selects) and +Interact (play-style clicks and drags without starting the game), plus Play. A +handler runs in Interact and Play by default, so an Edit click on your game piece +selects it like any object. Say so when you want something else: + +```js +api.registerClickHandler(pressKey, { modes: ['interact', 'play'] }); // the default, spelled out +api.registerClickHandler(pickTool, { modes: ['edit'] }); // an editor TOOL +``` + +In a headset the same modes apply (1.17, 30b): Play in VR enters Interact, where the +trigger clicks (module handlers, On Click) and selects nothing; in Edit it selects as +before. Handlers also get `ctx = {source, mode}` — see *Game feel* below for the +hold-and-sweep and `{sweep: false}`. +`api.editorMode()` reads the mode on this screen (`'edit'` | `'interact'`) — for a module +whose own pointer listeners run outside click routing (a drag), so it can stand down in Edit. + +**Listed in the object list (1.17).** Content you build at the scene root shows in +the object list's *Module content* section — a read-only row that frames it, hides +it locally, or opens your toolbox. Give it a readable name: + +```js +api.registerListedGroup('piano-module', { label: 'Piano' }); +``` + ```js api.registerFrameTask((time) => { /* runs every frame, synced time */ }); @@ -456,6 +481,14 @@ api.onInput((kind, code) => {}); // 'down'/'up' events; returns unsubscri // 'locomotion' — VR left-stick locomotion api.claimInput('keys'); // ALWAYS release when your mode ends api.releaseInput('keys'); + +// reading a keydown YOURSELF (a toolbox's own handler)? resolve the key the way the +// editor does — `event.key` when it is an ASCII letter/digit, the physical +// `event.code` position otherwise — so it works on a Cyrillic/Greek/Hebrew layout: +window.addEventListener('keydown', (e) => { + if (api.keyOf(e) === 'G') grab(); // 'G' on QWERTY, AZERTY, Dvorak AND ЙЦУКЕН + if (api.letterOf(e) === 'w') drive(); // lowercase form; named keys ('Escape') unchanged +}); ``` ### Pointer ray (190) @@ -473,6 +506,100 @@ api.registerFrameTask(() => { }); ``` +### Free-cursor games (1.17) + +A board, puzzle or instrument game can play with the real mouse cursor and no +pointer lock: publish `userData.play.cursor = 'free'` on your scene-root group +(or the scene's physics block sets `play.cursor: 'free'`). `api.pointerRay()` is +then the cursor's ray in play; in a locked game it is the CROSSHAIR ray (the view's +centre), never the stale mouse position the lock pinned. + +### Storage (1.17) + +```js +// JSON on THIS device, namespaced to your module (tp:mod::), 256 KB per +// module; never replicated, never saved into a scene, survives disable/remove. +if (api.storage) { + const progress = api.storage.get('progress', { unlocked: 1 }); + api.storage.set('progress', progress); // -> true, or false over the quota + api.storage.keys(); api.storage.bytes(); api.storage.remove('progress'); + api.storage.clear(); // your "Reset progress" +} +``` + +Feature-detect it; on an older app write the SAME key yourself +(`localStorage['tp:mod::'] = JSON.stringify(value)`) so progress carries over. + +### Game feel: sound, music, haptics, effects, banners (1.17, roadmap 30b) + +Everything here is LOCAL to the device it runs on — broadcast your own op +(`api.send`) when every peer should hear, feel or see it — and all of it is +feature-detected, so a module written against it still loads on an older app. + +```js +register(api) { + api.registerClickHandler((object, ctx) => { + if (!object.userData.ring) return false; + const at = object.getWorldPosition(new api.THREE.Vector3()).toArray(); + api.playSound?.('success', at); + api.effects?.burst(at, { kind: 'sparkle' }); + api.announce?.('Ring ' + object.userData.ring + ' reached', { sub: '+50' }); + api.hapticPattern?.('success'); // both hands; 'left' | 'right' for one + return true; + }); + api.game.onChange?.(() => (api.game.roundUnderway() ? api.music?.play('arcade') : api.music?.stop())); +} +``` + +- **`api.playSound(name, position?)`** → `true` when a sound started. Twenty + procedural game sounds, no assets: `click` `pop` `whoosh` `success` `fail` `hit` + `kick` `shoot` `laser` `explosion` `coin` `levelup` `goal` `whistle` `cheer` + `step` `ring` `sparkle` `hurt` `portal`, plus the ping chimes `ding` (the default), + `chime`, `pluck`, `bell`. An unknown name is a quiet no-op (it used to play the + ding). `position` spatialises it. Played at the player's "Game sounds" volume. +- **`api.music.play(preset, {volume?})` / `api.music.stop()`** — a looping + procedural track under the effects, at the player's "Music" volume: `arcade` + `ambient` `dungeon` `stadium` `space` `puzzle` `studio`. It is TEMPO-SYNCED to the + session clock (two peers on one preset hear the same bar), plays only in Interact + or Play (`false` from the editor) and stops by itself when the player goes back to + editing. `api.music.current()` names what is playing; `api.music.presets()` lists them. +- **`api.hapticPattern(name, hand?)`** — `tap` `bump` `hit` `success` `fail` + `rumble` `heartbeat` on the VR controllers. `api.haptic(intensity, ms, hand?)` + still exists. BOTH are silent in Edit mode (vibration is for playing), and core + already plays the defaults for you in Interact/Play: a `tap` when the laser enters + something clickable, a `bump` when a press lands, a `hit` on a grab, and a knock + sized by how hard the hand hit. +- **`api.effects.burst([x, y, z], {kind?, color?, count?})`** — a short, pooled + particle burst at the scene root: `sparkle` (default), `confetti`, `smoke`, + `sparks`; `count` 1..96. Nothing is saved or replicated. +- **`api.announce(text, {sub?, ms?, color?})`** — a big centred banner ("GOAL!", + "Level 3"): the desktop HUD, and in a headset a banner fixed in front of the + player. A new one replaces the one showing; `ms` 300..15000 (default 1800). + +**In VR, your game's HUD is in the player's hands.** A screen with `input: 'menu'`, +a control on it, or bound to the `menu` / `paused` / `over` game state is drawn on +a board ~1.2 m in front of the player that follows their head lazily; its buttons +work with the laser and trigger and with a poke. The other screens (score, timer, +level) read as short lines on a card on the left wrist and a strip across the top +of the view. The board and the wrist card always carry an **Edit mode** button. +Nothing to do on your side — author the HUD once and it works on both. + +**Hold the trigger and sweep (VR, Interact/Play).** While the trigger is held, the +controller TIP clicks each clickable it passes over — the one it is NEAREST to, so a +glide along a keyboard plays one key at a time — once, and again after it leaves and +comes back: piano keys, pads, toggles, HUD buttons; the laser does the same for things +out of reach. "Clickable" (also what earns the hover `tap`) means: under a group you +passed to `registerInteractiveGroup`, a part of an audio device (`userData.device` on +an ancestor — each key/pad is its own control), an object an On Click node targets, or +an object you mark `userData.clickable = true`. Your handler gets a second argument +`ctx = {source, mode}`: `source` is `'click'` (desktop), `'trigger'` (the VR press +itself) or `'sweep'` (a later entry while held). A control that must not be swept +(a knob you drag, a dot you carry) opts out: + +```js +api.registerClickHandler(pickDot, { sweep: false }); // presses still reach it; sweeps do not +``` + ### Physics (P-A) All mutations are INITIATOR-ONLY — the peer that started the simulation steps @@ -620,8 +747,9 @@ api.scene(); // THREE.Scene api.objectsGroup(); // the replicated objects root (add scene content here) api.peerId(); // our peer id (undefined before the mesh is up) api.toast('hi'); // toast in the corner -api.haptic(0.6, 60); // buzz the VR controllers (no-op on desktop); +api.haptic(0.6, 60); // buzz the VR controllers (no-op on desktop and in Edit); api.haptic(0.6, 60, 'right'); // optional hand targets one controller (17-A1) +api.hapticPattern('success'); // a named pattern (1.17, see Game feel) api.isVR(); // true inside a VR session api.vrHand('left'); // one hand's WORLD pose + buttons, or null: // {position, quaternion, trigger, gripped, connected} From 770f93e5a93fd687af7c890389c130ded5a06be2 Mon Sep 17 00:00:00 2001 From: AlexZ005 Date: Wed, 23 Sep 2026 18:44:32 +0300 Subject: [PATCH 8/8] [fix] 30b: the first burst of a session was lost; the timing-bound checks wait on the thing - effectsBurst: the Points program was compiled on the FIRST draw that needed it, and on a loaded device that stall ate the first burst of a session whole (measured: the first burst on a fresh page changed 0 pixels, every later one ~1500) - the first "ring reached" sparkle is exactly the one a player would miss. warmUp() compiles the burst material when the pool is built (renderer.compile on one system; compile() walks visible objects only, so it is shown for the call). - game-effects: 1.5 asserts the program exists before any frame is drawn; the pixel check uses a confetti burst (2.2 s of life - a slow frame cannot outlive it) and samples until a frame differs instead of one fixed 120 ms sleep, with a state dump in its message. (While finding this, zero-pixel frames on a long-lived dev server turned out to be the edited-under-a-running-server dual-instance trap; a fresh server reads 3361 px.) - game-haptics: the pattern checks wait for the PULSES (headless timers ran ~2x late on a machine running two e2e slots), not a clock. Counterfactual: warmUp removed -> 1.5 red (16 -> 16 programs). game-effects 24/24, game-haptics 18/18 on a fresh server; svelte-check 334/47; build green. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/lib/effectsBurst.js | 26 ++++++++++++++++- tests/e2e/game-effects.test.cjs | 49 ++++++++++++++++++++++++++++----- tests/e2e/game-haptics.test.cjs | 11 ++++---- 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/src/lib/effectsBurst.js b/src/lib/effectsBurst.js index 4032f4c4..02d98344 100644 --- a/src/lib/effectsBurst.js +++ b/src/lib/effectsBurst.js @@ -21,7 +21,7 @@ // sparks fast orange streaks, gravity, very short import * as THREE from 'three'; import { get } from 'svelte/store'; -import { globalScene } from '../stores/sceneStore'; +import { globalScene, globalRenderer, globalCamera } from '../stores/sceneStore'; import { moduleFrameTasks } from './moduleSDK'; export const BURST_KINDS = ['sparkle', 'confetti', 'smoke', 'sparks']; @@ -128,12 +128,36 @@ function makeSystem(index) { }; } +/** + * Compile the burst material NOW, while nothing is showing. three builds a program on the + * first draw that needs it, and on a loaded device (a headset mid-game) that stall ate the + * first burst of a session whole — measured: the first burst on a page changed 0 pixels, + * every later one ~1500. One material, one program; cheap. + * @param {any} scene + */ +function warmUp(scene) { + const renderer = /** @type {any} */ (get(globalRenderer)); + const camera = /** @type {any} */ (get(globalCamera)); + const probe = pool[0]?.points; + if (!renderer?.compile || !camera || !probe) return; + const was = probe.visible; + probe.visible = true; // compile() walks VISIBLE objects only + try { + renderer.compile(probe, camera, scene); + } catch { + /* a lost context compiles nothing; the first draw will */ + } + probe.visible = was; +} + /** the free system, or the oldest running one @returns {BurstSystem | null} */ function takeSystem() { const scene = /** @type {any} */ (get(globalScene)); if (!scene) return null; + const fresh = pool.length === 0; while (pool.length < POOL_SIZE) pool.push(makeSystem(pool.length)); for (const sys of pool) if (sys.points.parent !== scene) scene.add(sys.points); + if (fresh) warmUp(scene); const idle = pool.find((sys) => !sys.points.visible); if (idle) return idle; stats.recycled++; diff --git a/tests/e2e/game-effects.test.cjs b/tests/e2e/game-effects.test.cjs index 4f21db1b..3103f850 100644 --- a/tests/e2e/game-effects.test.cjs +++ b/tests/e2e/game-effects.test.cjs @@ -37,7 +37,12 @@ h.run(async () => { s.globalScene.subscribe((v) => (scene = v))(); s.objectsGroup.subscribe((v) => (group = v))(); const before = group.children.length; + let rd; + s.globalRenderer.subscribe((v) => (rd = v))(); + const pointsBefore = rd.info.programs.length; const ok = window.__api.effects.burst([0, 1.5, -3], { kind: 'confetti' }); + // synchronously, before ANY frame has drawn the burst + const pointsAfter = rd.info.programs.length; const fx = s.gameKit.effectsBurst.burstDebug(); const live = fx.live[0]; const points = live ? scene.getObjectByName(live.name) : null; @@ -48,13 +53,19 @@ h.run(async () => { inGroup: !!group.getObjectByName(live?.name ?? '-'), groupDelta: group.children.length - before, drawCount: points?.geometry?.drawRange?.count ?? -1, - sent: window.__sent.slice() + sent: window.__sent.slice(), + pointsBefore, + pointsAfter }; }); h.check(fired.ok === true && fired.live?.kind === 'confetti', '1.1 api.effects.burst starts a confetti burst (' + JSON.stringify(fired.live) + ')'); h.check(fired.parentIsScene && !fired.inGroup && fired.groupDelta === 0, '1.2 it lives at the scene root, never in objectsGroup'); h.check(fired.drawCount === 80, '1.3 confetti draws its recipe count (80, got ' + fired.drawCount + ')'); h.check(!fired.sent.length, '1.4 nothing was sent (' + JSON.stringify(fired.sent) + ')'); + h.check( + fired.pointsAfter > fired.pointsBefore, + '1.5 the burst program is compiled when the pool is built, not on the first draw (' + fired.pointsBefore + ' -> ' + fired.pointsAfter + ')' + ); console.log('\n=== 2. each kind moves the way it says ==='); const motion = await page.evaluate(() => { @@ -151,19 +162,43 @@ h.run(async () => { const quiet1 = await h.grabFrame(A); await page.waitForTimeout(150); const quiet2 = await h.grabFrame(A); - await page.evaluate(() => { + const fired6 = await page.evaluate(() => { const s = window.__stores; let cam; s.globalCamera.subscribe((c) => (cam = c))(); const THREE = s.THREE; const at = cam.getWorldPosition(new THREE.Vector3()).add(cam.getWorldDirection(new THREE.Vector3()).multiplyScalar(4)); - window.__api.effects.burst(at.toArray(), { kind: 'sparkle', count: 96, color: '#ffffff' }); + const ok = window.__api.effects.burst(at.toArray(), { kind: 'confetti', count: 96 }); // 2.2 s of life: a slow frame cannot outlive it + return { ok, live: s.gameKit.effectsBurst.burstDebug().live, tasks: s.moduleSDK.moduleFrameTasks.length }; }); - await page.waitForTimeout(120); - const lit = await h.grabFrame(A); + // the burst lives 1.1 s; a loaded machine can take a while to render the next frame, + // so sample until one differs (or the burst is surely gone) const idle = await h.frameDelta(page, quiet1, quiet2); - const burstDelta = await h.frameDelta(page, quiet2, lit); - h.check(burstDelta.changed > idle.changed + 400, '6.1 a sparkle in view changes the frame (' + burstDelta.changed + ' px vs idle ' + idle.changed + ')'); + let burstDelta = { changed: 0 }; + for (let i = 0; i < 6 && burstDelta.changed <= idle.changed + 400; i++) { + await page.waitForTimeout(80); + const frame = await h.grabFrame(A); + burstDelta = await h.frameDelta(page, quiet2, frame); + } + const diag = await page.evaluate(async () => { + const s = window.__stores; + let rd, scene, cam; + s.globalRenderer.subscribe((v) => (rd = v))(); + s.globalScene.subscribe((v) => (scene = v))(); + s.globalCamera.subscribe((v) => (cam = v))(); + const f0 = rd.info.render.frame; + await new Promise((r) => setTimeout(r, 300)); + const live = s.gameKit.effectsBurst.burstDebug().live; + const p = live[0] ? scene.getObjectByName(live[0].name) : null; + return { + frames: rd.info.render.frame - f0, + live, + parent: p?.parent === scene, + cam: cam?.position.toArray().map((v) => +v.toFixed(1)), + pos: p ? Array.from(p.geometry.attributes.position.array.slice(0, 3)).map((v) => +v.toFixed(1)) : null + }; + }); + h.check(burstDelta.changed > idle.changed + 400, '6.1 a burst in view changes the frame (' + burstDelta.changed + ' px vs idle ' + idle.changed + ') ' + JSON.stringify({ fired6, diag })); console.log('\n=== 7. announce ==='); const ann = await page.evaluate(async () => { diff --git a/tests/e2e/game-haptics.test.cjs b/tests/e2e/game-haptics.test.cjs index 28286ab1..18faa841 100644 --- a/tests/e2e/game-haptics.test.cjs +++ b/tests/e2e/game-haptics.test.cjs @@ -61,19 +61,20 @@ h.run(async () => { console.log('\n=== 3. patterns are timed pulses and stop when the player leaves ==='); const ok = await page.evaluate(() => ({ s: window.__api.hapticPattern('success', 'right'), u: window.__api.hapticPattern('zzz') })); - await page.waitForTimeout(400); + // wait on the PULSES, not a clock: headless timers run late under load + await page.waitForFunction(() => window.__buzz.length >= 3, null, { timeout: 5000 }).catch(() => {}); const success = await buzz(); h.check(ok.s === true && ok.u === false, '3.1 a known pattern plays, an unknown one is refused'); h.check( success.length === 3 && success.every((b) => b.hand === 'right') && success[0].i < success[1].i && success[1].i < success[2].i, '3.2 success = three rising pulses on the right (' + JSON.stringify(success.map((b) => b.i)) + ')' ); - h.check(success[2].t - success[0].t > 120, '3.3 they are spaced in time, not fired at once (' + (success[2].t - success[0].t).toFixed(0) + ' ms)'); + h.check(success.length === 3 && success[2].t - success[0].t > 120, '3.3 they are spaced in time, not fired at once (' + (success.length === 3 ? (success[2].t - success[0].t).toFixed(0) : '-') + ' ms)'); await page.evaluate(() => { window.__api.hapticPattern('rumble', 'left'); window.__stores.objectActions.setEditorMode('edit'); }); - await page.waitForTimeout(600); + await page.waitForTimeout(2000); // past the whole rumble even with late timers const cut = await buzz(); h.check(cut.length === 1, '3.4 leaving Interact mid-rumble stops it after the first pulse (' + cut.length + ' of 4)'); await mode('interact'); @@ -161,10 +162,10 @@ h.run(async () => { const v = window.__stores.vrControls; v.hapticKnock(Math.min(1, 0.2 + 0.4 / 10), 30, 'right'); // a brush at 0.4 m/s }); - await page.waitForTimeout(800); // headless timers run late — let the second pulse land + await page.waitForFunction(() => window.__buzz.length >= 2, null, { timeout: 5000 }).catch(() => {}); const soft = await buzz(); await page.evaluate(() => window.__stores.vrControls.hapticKnock(Math.min(1, 0.2 + 6 / 10), 30, 'right')); // a swing at 6 m/s - await page.waitForTimeout(800); + await page.waitForFunction(() => window.__buzz.length >= 2, null, { timeout: 5000 }).catch(() => {}); const hard = await buzz(); h.check(soft.length === 2 && hard.length === 2, '6.1 a knock plays the two-pulse hit (' + soft.length + ', ' + hard.length + ')'); h.check(hard[0].i > soft[0].i * 2 && hard[0].i <= 1, '6.2 a harder knock buzzes harder (' + soft[0].i.toFixed(3) + ' -> ' + hard[0].i.toFixed(3) + ')');