From 7d1e0a1f0c603edf02979dde13109dea5c820cdd Mon Sep 17 00:00:00 2001 From: snow Date: Fri, 17 Jul 2026 18:19:54 +0530 Subject: [PATCH 1/7] fix: apply generated edits directly --- .../src/components/editor-port/AgentChat.tsx | 134 +++++++++++------- 1 file changed, 84 insertions(+), 50 deletions(-) diff --git a/apps/web/src/components/editor-port/AgentChat.tsx b/apps/web/src/components/editor-port/AgentChat.tsx index e1d7033..40dc135 100644 --- a/apps/web/src/components/editor-port/AgentChat.tsx +++ b/apps/web/src/components/editor-port/AgentChat.tsx @@ -8,6 +8,7 @@ import { MessageContent, } from "@/components/ui/message"; import { useAgentStream } from "@/hooks/useAgentStream"; +import { accept } from "@/lib/api"; import { useEDL, totalDuration } from "@/stores/edl"; import { useAgent, @@ -37,6 +38,8 @@ export function AgentChat({ projectId }: AgentChatProps) { } = useAgentStream(projectId); const { state: edlState } = useEDL(); const { dispatch: agentDispatch } = useAgent(); + const [applyingVariant, setApplyingVariant] = useState(null); + const [appliedVariant, setAppliedVariant] = useState(null); const messagesRef = useRef(null); const lastMessage = messages[messages.length - 1]; const lastAgentText = lastMessage?.type === "agent" ? lastMessage.text : ""; @@ -77,23 +80,32 @@ export function AgentChat({ projectId }: AgentChatProps) { [agentDispatch], ); - // Variant card "apply" — user picked a specific finished variant. Ask - // the agent to call accept_variant with that index so the generated - // clip lands on the timeline. Studio listens for the completion event - // (fiebatt:timeline-refresh) and re-hydrates the EDL from the server. + // Applying a finished render is a deterministic editor action. Calling + // the API directly avoids a second model turn guessing whether it should + // invoke accept_variant. Studio consumes the returned authoritative EDL. const handleApplyVariant = useCallback( - (jobId: string, variantIndex: number) => { - if (!projectId) return; - void sendMessage({ - projectId, - message: `Accept variant ${variantIndex} for job ${jobId} and apply it to the timeline. After the tool succeeds, confirm it's applied in one sentence.`, - playheadTs: edlState.playhead, - duration: totalDuration(edlState.clips), - bbox: edlState.bbox ?? null, - selectionId: edlState.mask?.selectionId ?? null, - }); + async (jobId: string, variantIndex: number) => { + if (!projectId || applyingVariant) return; + const key = `${jobId}:${variantIndex}`; + setApplyingVariant(key); + try { + const accepted = await accept(jobId, variantIndex); + window.dispatchEvent( + new CustomEvent("fiebatt:timeline-refresh", { + detail: { tool: "accept_variant", timeline: accepted.timeline }, + }), + ); + setAppliedVariant(key); + } catch (error) { + agentDispatch({ + type: "add_error", + message: error instanceof Error ? error.message : "Could not apply this edit.", + }); + } finally { + setApplyingVariant(null); + } }, - [projectId, sendMessage, edlState.playhead, edlState.clips, edlState.bbox, edlState.mask?.selectionId], + [projectId, applyingVariant, agentDispatch], ); return ( @@ -255,6 +267,10 @@ export function AgentChat({ projectId }: AgentChatProps) { background: rgba(126, 231, 135, 0.15); border-color: rgba(126, 231, 135, 0.35); } + .suggestion-card__btn:disabled { + cursor: default; + opacity: 0.6; + } .suggestion-card__resolved { font-family: var(--f-mono); font-size: 9px; @@ -406,6 +422,8 @@ export function AgentChat({ projectId }: AgentChatProps) { message={msg} onDismissSuggestion={handleDismissSuggestion} onApplyVariant={handleApplyVariant} + applyingVariant={applyingVariant} + appliedVariant={appliedVariant} /> ))} @@ -429,10 +447,14 @@ function MessageRenderer({ message, onDismissSuggestion, onApplyVariant, + applyingVariant, + appliedVariant, }: { message: AgentMessage; onDismissSuggestion: (ts: number) => void; - onApplyVariant: (jobId: string, variantIndex: number) => void; + onApplyVariant: (jobId: string, variantIndex: number) => void | Promise; + applyingVariant: string | null; + appliedVariant: string | null; }) { switch (message.type) { case "user": @@ -481,6 +503,8 @@ function MessageRenderer({ jobId={message.jobId} variants={message.variants} onApply={onApplyVariant} + applyingVariant={applyingVariant} + appliedVariant={appliedVariant} /> ); @@ -597,10 +621,14 @@ function VariantPreviewCard({ jobId, variants, onApply, + applyingVariant, + appliedVariant, }: { jobId: string; variants: VariantPreview[]; - onApply: (jobId: string, variantIndex: number) => void; + onApply: (jobId: string, variantIndex: number) => void | Promise; + applyingVariant: string | null; + appliedVariant: string | null; }) { console.log( `[VariantPreviewCard] render job=${jobId} variants=${variants?.length ?? 0}`, @@ -612,41 +640,47 @@ function VariantPreviewCard({ {variants.length} variant{variants.length !== 1 ? "s" : ""} ready
- {variants.map((v) => ( -
-
- {v.url ? ( -
- ))} + ); + })}
); From bd67e93370979503ba4e4ec07aef46d49ecf0fe0 Mon Sep 17 00:00:00 2001 From: snow Date: Fri, 17 Jul 2026 18:22:30 +0530 Subject: [PATCH 2/7] fix: compare original and edited timelines --- .../components/editor-port/CompareOverlay.tsx | 46 +++++++---- apps/web/tests/e2e/frontend-readiness.spec.ts | 80 ++++++++++++++++++- 2 files changed, 109 insertions(+), 17 deletions(-) diff --git a/apps/web/src/components/editor-port/CompareOverlay.tsx b/apps/web/src/components/editor-port/CompareOverlay.tsx index 2c4df9a..06b22a6 100644 --- a/apps/web/src/components/editor-port/CompareOverlay.tsx +++ b/apps/web/src/components/editor-port/CompareOverlay.tsx @@ -14,11 +14,14 @@ import { export function CompareOverlay({ onClose }: { onClose: () => void }) { const { state } = useEDL(); const original = state.sources.find((asset) => asset.kind === "source") ?? state.sources[0] ?? null; - const total = totalDuration(state.clips); - const [playhead, setPlayhead] = useState(() => Math.min(state.playhead, total || state.playhead)); + const editedDuration = totalDuration(state.clips); + const comparisonDuration = Math.max(original?.duration ?? 0, editedDuration); + const [playhead, setPlayhead] = useState(() => + Math.min(state.playhead, comparisonDuration || state.playhead), + ); const [playing, setPlaying] = useState(false); const [singleView, setSingleView] = useState<"original" | "edited">("original"); - const [sideBySide, setSideBySide] = useState(false); + const [sideBySide, setSideBySide] = useState(true); const originalRef = useRef(null); const editedRef = useRef(null); const editedClipIdRef = useRef(null); @@ -90,7 +93,7 @@ export function CompareOverlay({ onClose }: { onClose: () => void }) { return; } - const restart = total > 0 && playhead >= total - 0.05; + const restart = comparisonDuration > 0 && playhead >= comparisonDuration - 0.05; const nextPlayhead = restart ? 0 : playhead; lastTickRef.current = null; setPlayhead(nextPlayhead); @@ -100,7 +103,7 @@ export function CompareOverlay({ onClose }: { onClose: () => void }) { void Promise.allSettled([originalVideo.play(), editedVideo.play()]); setPlaying(true); }); - }, [playhead, playing, syncVideos, total]); + }, [comparisonDuration, playhead, playing, syncVideos]); useEffect(() => { if (!playing) { @@ -116,10 +119,10 @@ export function CompareOverlay({ onClose }: { onClose: () => void }) { const delta = (now - lastTickRef.current) / 1000; lastTickRef.current = now; setPlayhead((current) => { - const next = Math.min(total, current + delta); - if (next >= total) { + const next = Math.min(comparisonDuration, current + delta); + if (next >= comparisonDuration) { setPlaying(false); - return total; + return comparisonDuration; } return next; }); @@ -130,7 +133,7 @@ export function CompareOverlay({ onClose }: { onClose: () => void }) { return () => { if (rafRef.current != null) cancelAnimationFrame(rafRef.current); }; - }, [playing, total]); + }, [comparisonDuration, playing]); useEffect(() => { const onKey = (event: KeyboardEvent) => { @@ -160,7 +163,7 @@ export function CompareOverlay({ onClose }: { onClose: () => void }) {
Compare
-
Original and edited timeline play together.
+
Original video and full edited timeline play together.
- {formatTime(playhead)} / {formatTime(total)} + {formatTime(playhead)} / {formatTime(comparisonDuration)} + +
+ ) : ( +

{title}

+ )} + {renameError ? ( +

{renameError}

+ ) : null}

{meta}

@@ -71,6 +159,18 @@ export function ProjectCard({

+ {onRename && !editing ? ( + + ) : null} setName(event.target.value)} + onChange={(event) => { + setName(event.target.value); + setError(false); + }} onBlur={() => void save()} onKeyDown={(event) => { if (event.key === "Enter") event.currentTarget.blur(); @@ -45,6 +56,7 @@ export function ProjectNameEditor({ } }} value={name} + title={error ? "Could not save project name" : saving ? "Saving project name" : "Click to rename"} /> ); } diff --git a/apps/web/tests/e2e/frontend-readiness.spec.ts b/apps/web/tests/e2e/frontend-readiness.spec.ts index 9dab564..6c4a3ea 100644 --- a/apps/web/tests/e2e/frontend-readiness.spec.ts +++ b/apps/web/tests/e2e/frontend-readiness.spec.ts @@ -60,6 +60,10 @@ async function installApi( if (path === "/api/health") return json(route, { ok: true, features: {} }); if (path === "/api/upload") return json(route, project); if (path === `/api/projects/${project.project_id}`) { + if (request.method() === "PATCH") { + const body = request.postDataJSON() as { name: string }; + return json(route, { ...project, name: body.name }); + } return json(route, { ...project, segments: [], entities: [] }); } if (path === `/api/timeline/${project.project_id}`) { @@ -90,6 +94,30 @@ test("project library shows only saved projects", async ({ page }) => { await expect(page.locator("article")).toHaveCount(0); }); +test("project can be renamed from the library", async ({ page }) => { + await installApi(page, [project]); + await page.goto("/projects"); + + await page.getByRole("button", { name: "Rename clip" }).click(); + await page.getByLabel("Rename clip").fill("Launch cut"); + await page.getByRole("button", { name: "Save project name" }).click(); + + await expect(page.getByText("Launch cut")).toBeVisible(); + await expect(page.getByRole("button", { name: "Rename Launch cut" })).toBeVisible(); +}); + +test("project can be renamed from the editor", async ({ page }) => { + await installApi(page, [project]); + await page.goto(`/editor?projectId=${project.project_id}`); + + const name = page.getByLabel("Project name"); + await name.fill("Final reel"); + await name.press("Enter"); + + await expect(name).toHaveValue("Final reel"); + await expect(name).toHaveAttribute("aria-invalid", "false"); +}); + test("settings use automatic platform routing", async ({ page }) => { await installApi(page); await page.goto("/settings"); From b63fbaa992479d9c6caa21544213532aced62188 Mon Sep 17 00:00:00 2001 From: snow Date: Fri, 17 Jul 2026 18:38:14 +0530 Subject: [PATCH 5/7] fix: separate original and refined prompts --- apps/api/app/api/routes/agent.py | 4 ++ apps/api/app/services/agent_tools.py | 1 + apps/api/app/workers/generate_job.py | 2 +- apps/api/tests/test_agent.py | 39 +++++++++++++++++++ .../src/components/editor-port/AgentChat.tsx | 4 +- apps/web/src/hooks/useAgentStream.ts | 5 ++- apps/web/src/stores/agent.tsx | 4 +- apps/web/tests/e2e/frontend-readiness.spec.ts | 37 ++++++++++++++++++ 8 files changed, 90 insertions(+), 6 deletions(-) diff --git a/apps/api/app/api/routes/agent.py b/apps/api/app/api/routes/agent.py index d074d77..e1fcd75 100644 --- a/apps/api/app/api/routes/agent.py +++ b/apps/api/app/api/routes/agent.py @@ -1031,6 +1031,10 @@ async def _agent_stream( tool_args = json.loads(tc.function.arguments) if tc.function.arguments else {} if tool_name == "create_edit_plan" and body.selection_id: tool_args.setdefault("selection_id", body.selection_id) + if tool_name == "generate_edit": + # Keep attribution separate from the model-authored generation + # prompt. The worker emits both for the before/refined card. + tool_args["user_prompt"] = body.message if tool_name in {"create_edit_plan", "generate_edit"} and body.video_gen_provider: tool_args["video_gen_provider"] = body.video_gen_provider diff --git a/apps/api/app/services/agent_tools.py b/apps/api/app/services/agent_tools.py index 0268bb4..d7bcf87 100644 --- a/apps/api/app/services/agent_tools.py +++ b/apps/api/app/services/agent_tools.py @@ -655,6 +655,7 @@ async def _generate_edit( reference_frame_ts=reference_frame_ts, payload={ "video_gen_provider": video_gen_provider or "auto", + "user_prompt": str(args.get("user_prompt") or prompt), "plan_id": plan.id if plan else None, "planned_context": plan.range_json if plan else None, "planned_intent": plan.intent_json if plan else None, diff --git a/apps/api/app/workers/generate_job.py b/apps/api/app/workers/generate_job.py index bf6a8de..4fcfda8 100644 --- a/apps/api/app/workers/generate_job.py +++ b/apps/api/app/workers/generate_job.py @@ -588,7 +588,7 @@ async def _do_extract_end_frame() -> tuple[Path, bool]: if planned_intent is not None and planned_intent.grounded_edit is not None else "asking Qwen to structure the edit plan" ), - user_prompt=prompt, + user_prompt=str(payload.get("user_prompt") or prompt), bbox=bbox, duration=segment_duration, ) diff --git a/apps/api/tests/test_agent.py b/apps/api/tests/test_agent.py index e7605cb..f07409d 100644 --- a/apps/api/tests/test_agent.py +++ b/apps/api/tests/test_agent.py @@ -697,3 +697,42 @@ async def test_agent_chat_with_function_call(client: AsyncClient, live_agent_set tc_end = next(e for e in events if e["event"] == "tool_call_end") assert tc_end["data"]["status"] == "done" assert tc_end["data"]["result"]["project_id"] == "test-proj" + + +@pytest.mark.asyncio +async def test_generate_tool_keeps_original_user_prompt(client: AsyncClient, live_agent_settings): + """Model-authored generation text must not replace user attribution.""" + response = _make_mock_agent_response( + "", + tool_calls=[_make_tool_call("generate_edit", { + "project_id": "test-proj", + "start_ts": 0, + "end_ts": 3, + "bbox": {"x": 0, "y": 0, "w": 1, "h": 1}, + "prompt": "A lime-green automobile with stable geometry.", + })], + ) + mock_client = _mock_agent_client(response) + + async def no_plan_events(*_args, **_kwargs): + if False: + yield "" + + with patch("app.api.routes.agent.AsyncOpenAI", return_value=mock_client), \ + patch("app.api.routes.agent.execute_tool", new_callable=AsyncMock) as mock_exec, \ + patch("app.api.routes.agent._bridge_plan_events", no_plan_events): + mock_exec.return_value = {"job_id": "job-1", "status": "pending"} + res = await client.post( + "/api/agent/chat", + json={ + "project_id": "test-proj", + "message": "make the car green", + "conversation_id": "conv-original-prompt", + }, + headers=headers(), + ) + + assert res.status_code == 200 + args = mock_exec.await_args.kwargs["args"] + assert args["prompt"] == "A lime-green automobile with stable geometry." + assert args["user_prompt"] == "make the car green" diff --git a/apps/web/src/components/editor-port/AgentChat.tsx b/apps/web/src/components/editor-port/AgentChat.tsx index 40dc135..d0a475f 100644 --- a/apps/web/src/components/editor-port/AgentChat.tsx +++ b/apps/web/src/components/editor-port/AgentChat.tsx @@ -688,7 +688,7 @@ function VariantPreviewCard({ // ─── prompt plan card ──────────────────────────────────────────────── // -// Surfaces the prompt-rewriting layer so users can SEE what the AI +// Surfaces the prompt-rewriting layer so users can see what the system // turned their one-liner into before it gets dispatched for generation. // Without this the whole "intelligent prompt expansion" value prop is // invisible to the user and the edit feels like a black box. Shows the @@ -734,7 +734,7 @@ function PromptPlanCard({ )}
- qwen + refined {ready ? (

{plan.prompt || plan.prompt_for_veo || plan.description || "(no prompt returned)"} diff --git a/apps/web/src/hooks/useAgentStream.ts b/apps/web/src/hooks/useAgentStream.ts index fe43e92..81aa14c 100644 --- a/apps/web/src/hooks/useAgentStream.ts +++ b/apps/web/src/hooks/useAgentStream.ts @@ -321,7 +321,7 @@ export function useAgentStream(projectId?: string | null) { const data = JSON.parse(dataStr) as Record; eventCount++; if (currentEvent === "done") sawDone = true; - handleSSEEvent(dispatch, currentEvent, data); + handleSSEEvent(dispatch, currentEvent, data, message); if (currentEvent === "suggestion") { const edit = data.edit as { job_id?: string } | undefined; if (edit?.job_id) { @@ -565,6 +565,7 @@ function handleSSEEvent( dispatch: React.Dispatch, event: string, data: Record, + currentUserPrompt = "", ): void { _logSSE(event, data); @@ -679,7 +680,7 @@ function handleSSEEvent( dispatch({ type: "prompt_plan_started", jobId, - userPrompt: (data.user_prompt as string | undefined) ?? "", + userPrompt: currentUserPrompt || (data.user_prompt as string | undefined) || "", }); break; } diff --git a/apps/web/src/stores/agent.tsx b/apps/web/src/stores/agent.tsx index 6dc12bf..f7be3ab 100644 --- a/apps/web/src/stores/agent.tsx +++ b/apps/web/src/stores/agent.tsx @@ -341,7 +341,9 @@ function agentReducer(state: AgentState, action: AgentAction): AgentState { { type: "prompt_plan", jobId: action.jobId, - userPrompt: "", + userPrompt: [...state.messages] + .reverse() + .find((message) => message.type === "user")?.text ?? "", plan: action.plan, vendor: null, ts: now, diff --git a/apps/web/tests/e2e/frontend-readiness.spec.ts b/apps/web/tests/e2e/frontend-readiness.spec.ts index 6c4a3ea..4e4fea8 100644 --- a/apps/web/tests/e2e/frontend-readiness.spec.ts +++ b/apps/web/tests/e2e/frontend-readiness.spec.ts @@ -282,3 +282,40 @@ test("generation preview returns after leaving and reopening editor", async ({ p await expect(page.getByText("finished while away")).toBeVisible(); await expect(page.getByRole("button", { name: "apply" })).toBeVisible(); }); + +test("prompt card separates user request from refined prompt", async ({ page }) => { + const agentSse = [ + "event: prompt_plan_started", + `data: ${JSON.stringify({ + job_id: "job-prompt", + user_prompt: "A polished emerald vehicle with exact color consistency.", + })}`, + "", + "event: prompt_plan", + `data: ${JSON.stringify({ + job_id: "job-prompt", + plan: { + prompt: "A polished emerald vehicle with exact color consistency.", + intent: "appearance change", + }, + })}`, + "", + "event: done", + "data: {}", + "", + "", + ].join("\n"); + await installApi(page, [project], undefined, { + agentSse, + current: () => ({}), + }); + await page.goto(`/editor?projectId=${project.project_id}`); + await page.getByPlaceholder("describe an edit...").fill("make the car green"); + await page.getByRole("button", { name: "send message" }).click(); + + await expect(page.locator(".prompt-plan__user")).toHaveText("make the car green"); + await expect(page.locator(".prompt-plan__content")).toHaveText( + "A polished emerald vehicle with exact color consistency.", + ); + await expect(page.locator(".prompt-plan__lane-k")).toHaveText(["you", "refined"]); +}); From 5d66b4e818298f572ada1ba9caabeac62b117640 Mon Sep 17 00:00:00 2001 From: snow Date: Fri, 17 Jul 2026 18:41:47 +0530 Subject: [PATCH 6/7] refactor: remove editor mode switching --- apps/api/app/api/routes/agent.py | 6 +- apps/web/src/app/editor/Studio.tsx | 23 ++---- apps/web/src/app/editor/studio.css | 82 ++++--------------- .../src/components/editor-port/Inspector.tsx | 16 +--- apps/web/src/components/editor-topbar.tsx | 15 ---- apps/web/tests/e2e/frontend-readiness.spec.ts | 3 + 6 files changed, 28 insertions(+), 117 deletions(-) diff --git a/apps/api/app/api/routes/agent.py b/apps/api/app/api/routes/agent.py index e1fcd75..a6167d4 100644 --- a/apps/api/app/api/routes/agent.py +++ b/apps/api/app/api/routes/agent.py @@ -761,9 +761,9 @@ async def _bridge_plan_events( The ``generate_job`` worker publishes a rich trail of stage events on the per-job event bus (plan_start, plan_done, gen_start, variant_done…). - The Pro-mode reveal panel consumes these directly via - ``/api/jobs/{id}/stream``, but the Vibe-mode agent chat only hears the - agent SSE stream — so by default the user never sees that the model is + The detailed generation panel consumes these directly via + ``/api/jobs/{id}/stream``, while agent chat only hears the agent SSE + stream — so by default the user never sees that the model is quietly rewriting their one-line prompt into a video generation brief. This helper bridges the gap for exactly the events that are interesting diff --git a/apps/web/src/app/editor/Studio.tsx b/apps/web/src/app/editor/Studio.tsx index 07f042a..9efe51d 100644 --- a/apps/web/src/app/editor/Studio.tsx +++ b/apps/web/src/app/editor/Studio.tsx @@ -19,7 +19,6 @@ import { } from "@/stores/edl"; import { Preview } from "@/components/editor-port/Preview"; import { CompareOverlay } from "@/components/editor-port/CompareOverlay"; -import { Inspector } from "@/components/editor-port/Inspector"; import { Timeline } from "@/components/editor-port/Timeline"; import { Library } from "@/components/editor-port/Library"; import { UploadDrop } from "@/components/editor-port/UploadDrop"; @@ -225,7 +224,6 @@ function StudioInner({ const [uploadError, setUploadError] = useState(null); const [hydrateError, setHydrateError] = useState(null); const [hydrateAttempt, setHydrateAttempt] = useState(0); - const [mode, setMode] = useState<'vibe' | 'pro'>('vibe'); const [showShortcuts, setShowShortcuts] = useState(false); const [showCompare, setShowCompare] = useState(false); const [hydratingProject, setHydratingProject] = useState(Boolean(initialProject)); @@ -270,9 +268,9 @@ function StudioInner({ // The "effective" project id is the one AgentChat actually sends to the // backend — it falls back to whatever source the user just uploaded in - // vibe mode so the chat keeps working even before we've persisted a + // focused layout so chat keeps working even before we've persisted a // reopen-able reel. The timeline-refresh listener MUST key on this same - // fallback chain, otherwise uploading-in-vibe-mode silently skips the + // fallback chain, otherwise a newly uploaded project silently skips the // listener attach and ``accept_variant`` lands on the server but the // preview never updates. (This was the "pressing apply does nothing" // bug — initialProject was undefined, so projectId was undefined, so @@ -380,7 +378,7 @@ function StudioInner({ // Prefer the saved initialProject snapshot (from ?reopen=), but // fall back to constructing one from the live EDL source asset — - // this is the vibe-mode upload path where the parent route never + // this is the upload path where the parent route never // provided an initialProject prop. Without this fallback the // handler aborted via the ref mismatch and the preview stayed // frozen on the original clip. @@ -659,7 +657,7 @@ function StudioInner({ : "Export"; return ( -

+
fileInputRef.current?.click()} onCompare={() => { dispatch({ type: "set_playing", playing: false }); @@ -927,16 +923,7 @@ function StudioInner({ /> diff --git a/apps/web/src/app/editor/studio.css b/apps/web/src/app/editor/studio.css index a14abcf..7f62f15 100644 --- a/apps/web/src/app/editor/studio.css +++ b/apps/web/src/app/editor/studio.css @@ -181,58 +181,6 @@ } .topbar__ghost:hover { border-color: var(--edge-2); color: var(--ink); } -/* vibe / pro toggle pill — subtle chrome rim in off state, fully filled - bevel in on state. mirrors the pill-nav aesthetic on the landing page. */ -.topbar__vibe { - position: relative; - padding: 4px 14px; - border-radius: 9999px; - font-family: var(--f-mono); - font-size: 10px; - letter-spacing: 0.18em; - text-transform: uppercase; - color: var(--ink-fade); - background: transparent; - border: 1px solid var(--edge); - transition: - color var(--dur-s) var(--ease), - border-color var(--dur-s) var(--ease), - background var(--dur-m) var(--ease), - box-shadow var(--dur-m) var(--ease); - isolation: isolate; -} -.topbar__vibe:hover { - color: var(--ink); - border-color: var(--edge-2); - box-shadow: 0 0 22px -8px rgba(232, 232, 232, 0.35); -} -.topbar__vibe--on { - color: #0a0a0a; - border-color: transparent; - background: - linear-gradient( - 180deg, - #f4f4f4 0%, - #d0d0d0 46%, - #9a9a9a 50%, - #bcbcbc 54%, - #e8e8e8 100% - ); - text-shadow: 0 1px 0 rgba(255, 255, 255, 0.6); - box-shadow: - inset 0 1px 0 rgba(255, 255, 255, 0.7), - inset 0 -1px 0 rgba(0, 0, 0, 0.35), - 0 0 0 1px rgba(0, 0, 0, 0.45); -} -.topbar__vibe--on:hover { - filter: brightness(1.06); - box-shadow: - inset 0 1px 0 rgba(255, 255, 255, 0.7), - inset 0 -1px 0 rgba(0, 0, 0, 0.35), - 0 0 0 1px rgba(0, 0, 0, 0.45), - 0 0 32px -10px rgba(190, 220, 255, 0.45); -} - .topbar__export { padding: 4px 16px; font-size: 11px; @@ -382,34 +330,34 @@ transform: translate(-50%, -50%); } -/* ── vibe mode ── */ +/* ── focused editor layout ── */ /* 60/40 split: video player left, agent chat right */ -/* hide left sidebar and timeline in vibe mode */ -.studio--vibe .studio__left, -.studio--vibe .studio__bottom { +/* hide low-level media panels from the main workflow */ +.studio--focused .studio__left, +.studio--focused .studio__bottom { display: none !important; } /* hide the left splitter and bottom splitter */ -.studio--vibe .splitter--vertical:first-of-type, -.studio--vibe .studio__h-splitter-wrap { +.studio--focused .splitter--vertical:first-of-type, +.studio--focused .studio__h-splitter-wrap { display: none !important; } /* 60/40 grid: center (video) | splitter | right (chat) */ -.studio--vibe .studio__body { +.studio--focused .studio__body { grid-template-columns: minmax(0, 3fr) var(--split-x) minmax(280px, 2fr); } -/* right sidebar is visible in vibe mode (agent chat fills it) */ -.studio--vibe .studio__right { +/* agent chat fills the right side */ +.studio--focused .studio__right { display: flex !important; flex-direction: column; overflow: hidden; } -.studio--vibe .agent-chat { +.studio--focused .agent-chat { height: 100%; min-height: 0; min-width: 0; @@ -417,27 +365,27 @@ } @media (max-width: 900px) { - .studio--vibe .studio__body { + .studio--focused .studio__body { grid-template-columns: minmax(0, 1fr); grid-template-rows: minmax(260px, 1fr) minmax(300px, 46vh); } - .studio--vibe .splitter--vertical { + .studio--focused .splitter--vertical { display: none !important; } } /* collapse the timeline row */ -.studio--vibe { +.studio--focused { grid-template-rows: 40px 1fr; } /* hide the timeline toolbar row if it exists */ -.studio--vibe .tbar { +.studio--focused .tbar { display: none !important; } /* make the preview video fill the space */ -.studio--vibe .pv { +.studio--focused .pv { height: 100%; } diff --git a/apps/web/src/components/editor-port/Inspector.tsx b/apps/web/src/components/editor-port/Inspector.tsx index 4f1604a..6663141 100644 --- a/apps/web/src/components/editor-port/Inspector.tsx +++ b/apps/web/src/components/editor-port/Inspector.tsx @@ -17,32 +17,20 @@ import "./inspector.css"; type Tab = "ai" | "continuity" | "basic" | "info" | "agent"; export function Inspector({ - mode = "pro", continuity, projectId, - showAiTab = mode === "pro", }: { - mode?: "vibe" | "pro"; continuity: ContinuityDashboardController; projectId: string | null; - showAiTab?: boolean; }) { const [tab, setTab] = useState("ai"); const { state } = useEDL(); const selected = state.clips.find((c) => c.id === state.selectedId) ?? null; - useEffect(() => { - if (!showAiTab && tab === "ai") { - setTab("continuity"); - } - }, [showAiTab, tab]); - return (
- {showAiTab && tab === "ai" && } + {tab === "ai" && } {tab === "continuity" && } {tab === "basic" && } {tab === "info" && } diff --git a/apps/web/src/components/editor-topbar.tsx b/apps/web/src/components/editor-topbar.tsx index 464b8bc..9ba79a6 100644 --- a/apps/web/src/components/editor-topbar.tsx +++ b/apps/web/src/components/editor-topbar.tsx @@ -6,14 +6,11 @@ import type { ReactNode } from "react"; import { ArrowLeft, Download } from "lucide-react"; import { Button } from "@/components/ui/button"; -import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { ProjectNameEditor } from "@/components/project-name-editor"; export function EditorTopbar({ projectId, projectName, - mode = "vibe", - onModeChange, onImport, onCompare, onShowShortcuts, @@ -25,8 +22,6 @@ export function EditorTopbar({ }: { projectId: string | null; projectName: string; - mode?: "vibe" | "pro"; - onModeChange?: (mode: "vibe" | "pro") => void; onImport?: () => void; onCompare?: () => void; onShowShortcuts?: () => void; @@ -76,16 +71,6 @@ export function EditorTopbar({
{statusSlot} - onModeChange?.(value as "vibe" | "pro")}> - - - Vibe - - - Editor - - - diff --git a/apps/web/tests/e2e/frontend-readiness.spec.ts b/apps/web/tests/e2e/frontend-readiness.spec.ts index 4e4fea8..b9297c3 100644 --- a/apps/web/tests/e2e/frontend-readiness.spec.ts +++ b/apps/web/tests/e2e/frontend-readiness.spec.ts @@ -140,6 +140,9 @@ test("upload opens the authoritative project", async ({ page }) => { await expect(page).toHaveURL(/\/editor\?projectId=project-123$/); await expect(page.getByLabel("Project name")).toHaveValue("clip"); + await expect(page.getByPlaceholder("describe an edit...")).toBeVisible(); + await expect(page.getByRole("tab", { name: "Vibe" })).toHaveCount(0); + await expect(page.getByRole("tab", { name: "Editor" })).toHaveCount(0); }); test("login rejects recursive return paths", async ({ page }) => { From c58c9daafa775ce3fe90c1ddc87e21b9e15b2a13 Mon Sep 17 00:00:00 2001 From: snow Date: Fri, 17 Jul 2026 18:46:33 +0530 Subject: [PATCH 7/7] fix: suppress raw tool call markup --- apps/api/app/api/routes/agent.py | 83 ++++++++++++++++++- apps/api/tests/test_agent.py | 64 ++++++++++++++ apps/web/src/hooks/useAgentStream.ts | 3 +- apps/web/src/lib/agent-text.ts | 11 +++ apps/web/src/stores/agent.tsx | 17 +++- apps/web/tests/e2e/frontend-readiness.spec.ts | 26 ++++++ 6 files changed, 199 insertions(+), 5 deletions(-) create mode 100644 apps/web/src/lib/agent-text.ts diff --git a/apps/api/app/api/routes/agent.py b/apps/api/app/api/routes/agent.py index a6167d4..4907131 100644 --- a/apps/api/app/api/routes/agent.py +++ b/apps/api/app/api/routes/agent.py @@ -10,8 +10,10 @@ import asyncio import json import logging +import re import uuid from collections.abc import AsyncGenerator +from dataclasses import dataclass from typing import Any, Literal from fastapi import APIRouter, Depends @@ -34,6 +36,80 @@ log = logging.getLogger("fiebatt.agent") router = APIRouter(prefix="/agent", tags=["agent"]) + +_TOOL_MARKUP_START = re.compile( + r"<\s*(?:(?:[|｜]\s*)?DSML(?:\s*[|｜])?\s*)?" + r"(?:function[\s_]?calls?|tool[\s_]?calls?|invoke\b|parameter\b)", + re.IGNORECASE, +) +_DSML_INVOKE = re.compile( + r'<\s*(?:(?:[|｜]\s*)?DSML(?:\s*[|｜])?\s*)?invoke\s+name="([^"]+)"[^>]*>' + r"(.*?)" + r"", + re.IGNORECASE | re.DOTALL, +) +_DSML_PARAMETER = re.compile( + r'<\s*(?:(?:[|｜]\s*)?DSML(?:\s*[|｜])?\s*)?parameter\s+' + r'name="([^"]+)"(?:\s+string="(true|false)")?[^>]*>' + r"(.*?)" + r"", + re.IGNORECASE | re.DOTALL, +) + + +@dataclass(slots=True) +class _ParsedFunction: + name: str + arguments: str + + +@dataclass(slots=True) +class _ParsedToolCall: + id: str + function: _ParsedFunction + + +def _clean_agent_text(text: str) -> str: + """Return user-facing prose before any leaked tool protocol block.""" + marker = _TOOL_MARKUP_START.search(text) + visible = text[:marker.start()] if marker else text + return visible.strip() + + +def _parse_dsml_tool_calls(text: str) -> list[_ParsedToolCall]: + """Recover DeepSeek-style DSML calls leaked into message content.""" + parsed: list[_ParsedToolCall] = [] + for index, invoke in enumerate(_DSML_INVOKE.finditer(text)): + name, body = invoke.groups() + arguments: dict[str, Any] = {} + for parameter in _DSML_PARAMETER.finditer(body): + key, is_string, raw_value = parameter.groups() + value = raw_value.strip() + if (is_string or "false").lower() != "true": + try: + arguments[key] = json.loads(value) + except json.JSONDecodeError: + arguments[key] = value + else: + arguments[key] = value + if not arguments: + try: + direct = json.loads(body.strip()) + if isinstance(direct, dict): + arguments = direct + except json.JSONDecodeError: + pass + parsed.append( + _ParsedToolCall( + id=f"dsml-{uuid.uuid4().hex[:12]}-{index}", + function=_ParsedFunction( + name=name, + arguments=json.dumps(arguments), + ), + ) + ) + return parsed + # ---- request schema ---- class BBoxParam(BaseModel): @@ -987,8 +1063,11 @@ async def _agent_stream( ) choice = response.choices[0] msg = choice.message - text = msg.content or "" - tool_calls = msg.tool_calls or [] + raw_text = msg.content or "" + tool_calls: list[Any] = list(msg.tool_calls or []) + if not tool_calls: + tool_calls = _parse_dsml_tool_calls(raw_text) + text = _clean_agent_text(raw_text) except Exception as exc: log.exception("[agent.chat] req=%s %s API call failed on turn %d", req_id, provider_name, turn) yield sse_event("error", {"message": f"{provider_name} API error: {exc}"}) diff --git a/apps/api/tests/test_agent.py b/apps/api/tests/test_agent.py index f07409d..f6b37a8 100644 --- a/apps/api/tests/test_agent.py +++ b/apps/api/tests/test_agent.py @@ -23,7 +23,9 @@ from app.workers.runner import JobRunner # noqa: E402 from app.api.routes.agent import ( # noqa: E402 _build_messages, + _clean_agent_text, _detached_agent_relay, + _parse_dsml_tool_calls, sse_event, ) from app.services.agent_tools import execute_tool # noqa: E402 @@ -179,6 +181,29 @@ def test_unicode(self): assert "\u2728" in parsed["text"] +def test_dsml_tool_markup_is_parsed_and_hidden(): + raw = """I will inspect it now. +<｜DSML｜function_calls> +<｜DSML｜invoke name="analyze_video"> +<｜DSML｜parameter name="project_id" string="true">project-1 +<｜DSML｜parameter name="fps" string="false">2 + +""" + + assert _clean_agent_text(raw) == "I will inspect it now." + calls = _parse_dsml_tool_calls(raw) + assert len(calls) == 1 + assert calls[0].function.name == "analyze_video" + assert json.loads(calls[0].function.arguments) == { + "project_id": "project-1", + "fps": 2, + } + + +def test_malformed_dsml_prefix_is_never_user_visible(): + assert _clean_agent_text("Starting +<｜DSML｜invoke name="analyze_video"> +<｜DSML｜parameter name="project_id" string="true">test-proj + +""" + mock_client = _mock_agent_client( + _make_mock_agent_response(raw_call), + _make_mock_agent_response("Inspection complete."), + ) + + with patch("app.api.routes.agent.AsyncOpenAI", return_value=mock_client), \ + patch("app.api.routes.agent.execute_tool", new_callable=AsyncMock) as mock_exec: + mock_exec.return_value = { + "status": "done", + "project_id": "test-proj", + "duration": 5.0, + } + res = await client.post( + "/api/agent/chat", + json={ + "project_id": "test-proj", + "message": "Inspect this video", + "conversation_id": "conv-dsml", + }, + headers=headers(), + ) + + events = _parse_sse_events(res.text) + visible_text = "".join( + event["data"]["text"] for event in events if event["event"] == "token" + ) + assert visible_text == "I will inspect it.Inspection complete." + assert "DSML" not in res.text + assert mock_exec.await_args.kwargs["args"] == {"project_id": "test-proj"} + + @pytest.mark.asyncio async def test_generate_tool_keeps_original_user_prompt(client: AsyncClient, live_agent_settings): """Model-authored generation text must not replace user attribution.""" diff --git a/apps/web/src/hooks/useAgentStream.ts b/apps/web/src/hooks/useAgentStream.ts index 81aa14c..11fb609 100644 --- a/apps/web/src/hooks/useAgentStream.ts +++ b/apps/web/src/hooks/useAgentStream.ts @@ -19,6 +19,7 @@ import { type ChatMessageResp, } from "@/lib/api"; import { redirectToLogin } from "@/lib/auth"; +import { cleanAgentText } from "@/lib/agent-text"; import { useAgent, type AgentAction, @@ -498,7 +499,7 @@ function dbMessageToAgentMessage(msg: ChatMessageResp): AgentMessage | null { case "agent": return { type: "agent", - text: (content.text as string) ?? "", + text: cleanAgentText((content.text as string) ?? ""), ts, streaming: false, }; diff --git a/apps/web/src/lib/agent-text.ts b/apps/web/src/lib/agent-text.ts new file mode 100644 index 0000000..38ddd10 --- /dev/null +++ b/apps/web/src/lib/agent-text.ts @@ -0,0 +1,11 @@ +const TOOL_MARKUP_START = /<\s*(?:(?:[|｜]\s*)?DSML(?:\s*[|｜])?\s*)?(?:function[\s_]?calls?|tool[\s_]?calls?|invoke\b|parameter\b)/i; + +export function hasAgentToolMarkup(text: string): boolean { + return TOOL_MARKUP_START.test(text); +} + +/** Keep model prose while removing raw tool-protocol markup from the UI. */ +export function cleanAgentText(text: string): string { + const marker = TOOL_MARKUP_START.exec(text); + return marker ? text.slice(0, marker.index).trimEnd() : text; +} diff --git a/apps/web/src/stores/agent.tsx b/apps/web/src/stores/agent.tsx index f7be3ab..fb4db42 100644 --- a/apps/web/src/stores/agent.tsx +++ b/apps/web/src/stores/agent.tsx @@ -14,6 +14,7 @@ import { type Dispatch, type ReactNode, } from "react"; +import { cleanAgentText, hasAgentToolMarkup } from "@/lib/agent-text"; // ─── types ──────────────────────────────────────────────────────────── @@ -71,7 +72,13 @@ export interface PromptPlan { export type AgentMessage = | { type: "user"; text: string; ts: number } - | { type: "agent"; text: string; ts: number; streaming?: boolean } + | { + type: "agent"; + text: string; + ts: number; + streaming?: boolean; + toolMarkupSuppressed?: boolean; + } | { type: "tool_call"; id: string; @@ -201,7 +208,13 @@ function agentReducer(state: AgentState, action: AgentAction): AgentState { const msgs = [...state.messages]; const last = msgs[msgs.length - 1]; if (last && last.type === "agent" && last.streaming) { - msgs[msgs.length - 1] = { ...last, text: last.text + action.text }; + if (last.toolMarkupSuppressed) return state; + const combined = last.text + action.text; + msgs[msgs.length - 1] = { + ...last, + text: cleanAgentText(combined), + toolMarkupSuppressed: hasAgentToolMarkup(combined), + }; } return { ...state, messages: msgs }; } diff --git a/apps/web/tests/e2e/frontend-readiness.spec.ts b/apps/web/tests/e2e/frontend-readiness.spec.ts index b9297c3..81282be 100644 --- a/apps/web/tests/e2e/frontend-readiness.spec.ts +++ b/apps/web/tests/e2e/frontend-readiness.spec.ts @@ -322,3 +322,29 @@ test("prompt card separates user request from refined prompt", async ({ page }) ); await expect(page.locator(".prompt-plan__lane-k")).toHaveText(["you", "refined"]); }); + +test("agent reply never shows raw tool markup", async ({ page }) => { + const leaked = `Working on it. +<｜DSML｜function_calls> +<｜DSML｜invoke name="analyze_video"> +<｜DSML｜parameter name="project_id" string="true">project-123`; + const agentSse = [ + "event: token", + `data: ${JSON.stringify({ text: leaked })}`, + "", + "event: done", + "data: {}", + "", + "", + ].join("\n"); + await installApi(page, [project], undefined, { + agentSse, + current: () => ({}), + }); + await page.goto(`/editor?projectId=${project.project_id}`); + await page.getByPlaceholder("describe an edit...").fill("inspect this video"); + await page.getByRole("button", { name: "send message" }).click(); + + await expect(page.getByText("Working on it.")).toBeVisible(); + await expect(page.getByText(/DSML|function_calls|functioncall/i)).toHaveCount(0); +});