fix(gui): repair foreground target discovery and action execution

src/tandem/gui/candidates.py

@@ -91,6 +91,8 @@ def build(
         context: str = "",
     ) -> ActionSpace:
         context_words = set(re.findall(r"[a-z0-9]+", context.lower()))
+        navigation_requested = bool(re.search(
+            r"\b(sidebar|side bar|navigation (menu|panel|bar))\b|侧栏|导航栏", context, re.I))
         hint_words = {
             word
             for item in action_inputs or []
@@ -100,17 +102,26 @@ def build(
             )
         }
 
-        def relevance(target: UITarget) -> tuple[int, int]:
+        def relevance(target: UITarget) -> tuple[int, float, int]:
             words = set(re.findall(r"[a-z0-9]+", target.name.lower()))
             contextual_words = set(
                 re.findall(r"[a-z0-9]+", " ".join(
                     (target.name, *target.parent_context, target.nearby_text)).lower())
             )
             return (
-                len(words & hint_words) * 4 + len(contextual_words & context_words),
+                int(navigation_requested and any(
+                    "navigation" in parent.lower() for parent in target.parent_context)),
+                len(words & hint_words) * 4
+                + len(words & context_words) / max(1, len(words)) * 8
+                + min(2, len(contextual_words & context_words)),
                 -targets.index(target),
             )
 
+        def passive_navigation_text(target: UITarget) -> bool:
+            return (navigation_requested and target.scope == "web_content"
+                    and target.role in {"text", "label", "static", "table_cell", "grid_cell"}
+                    and "click" not in target.operations)
+
         close_browser_requested = any(
             phrase in context.lower()
             for phrase in (
@@ -127,21 +138,23 @@ def relevance(target: UITarget) -> tuple[int, int]:
             (
                 target
                 for target in targets
-                if close_browser_requested
-                or not (
+                if not passive_navigation_text(target)
+                and (close_browser_requested or not (
                     target.name.lower() == "close"
                     and any("google chrome" in item.lower() for item in target.parent_context)
                     and (
                         len(target.parent_context) <= 2 or "memory usage" in target.parent_context[-1].lower()
                     )
-                )
+                ))
             ),
             key=relevance,
             reverse=True,
-        )[: min(max_candidates, 24)]
+        )[:max_candidates]
         editable_targets = [target for target in targets if target.role in EDITABLE_ROLES]
         double_targets = sorted(
-            [target for target in targets if target.role in DOUBLE_CLICK_ROLES], key=relevance, reverse=True
+            [target for target in targets if target.role in DOUBLE_CLICK_ROLES
+             and not passive_navigation_text(target)],
+            key=relevance, reverse=True
         )[:4]
 
         def candidates(operation: str, values: list[UITarget], description: str) -> list[ActionCandidate]:

src/tandem/gui/field_binding.py

@@ -0,0 +1,90 @@
+"""Observed field identity and supplied-value binding; no model or execution calls."""
+
+import re
+
+
+def matches_field(hint, label, owner=""):
+    """Match an intended field to its observed label, not an arbitrary focused control."""
+    def words(value):
+        return set(re.findall(r"\w+", value.casefold())) - {"the", "a", "field", "required"}
+
+    observed = words(label)
+    intended = words(hint)
+    intended -= words(owner) - observed
+    weak_labels = {"name", "search", "value", "title", "date", "number"}
+    if observed <= weak_labels and len(observed) == 1 and intended != observed:
+        return False  # A qualifier distinguishes this field from a different form's generic label.
+    return (bool(observed) and (observed <= intended or len(intended) >= 2 and intended <= observed)
+            and not label.startswith(("Unnamed ", "Focused ")))
+
+
+def launcher_input(value):
+    """A launcher search is not a website's identically named Search field."""
+    return (value.get("scope") == "desktop_launcher"
+            or "desktop search" in value.get("target_hint", value["label"]).casefold())
+
+
+def _input_target(value, targets):
+    """Bind to one best observed field; spatial hints must not override its own label."""
+    scores = []
+    for target in targets:
+        choice = value.get("kind") == "choice" and target.role in {
+            "button", "push_button", "radio_button", "option", "list_item"}
+        editable = (target.role in {"textbox", "entry", "searchbox", "spin_button", "password_text"}
+                    or launcher_input(value) and target.role == "text" and target.focused
+                    or "type_text" in target.operations or target.input_type == "select-one")
+        if not (editable or choice):
+            continue
+        if launcher_input(value) and target.application != "gnome-shell":
+            continue
+        if (target.role == "spin_button" and target.input_type != "date"
+                and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value["text"])):
+            continue  # A complete date is not a Day/Month/Year segment or an ordinary numeric spinner.
+        named = bool(target.name) and not target.name.startswith(("Unnamed ", "Focused "))
+        labels = [target.name] if named else [target.description]
+        if not named and len(target.nearby_text) <= 80 and "\n" not in target.nearby_text:
+            labels.append(target.nearby_text)
+        score = 0
+        if choice:
+            wanted = set(re.findall(r"\w+", value["text"].casefold()))
+            option_name = re.sub(r"\([^)]*\)", "", target.name.casefold())
+            observed = set(re.findall(r"\w+", option_name))
+            hint = set(re.findall(r"\w+", value.get("target_hint", value["label"]).casefold()))
+            if wanted and wanted <= observed <= wanted | hint:
+                score = 6  # Match a specific option, never a container holding the entire form.
+        for label in labels:
+            if not label or not editable:
+                continue
+            hint = value.get("target_hint", value["label"])
+            owner = " ".join((target.application, *target.parent_context))
+            if launcher_input(value):
+                owner += " desktop"
+            if not matches_field(hint, label, owner):
+                continue  # Descriptive context words cannot override the explicit intended field.
+            ignored = {"brief", "required", "enter", "input", "field", "the", "a"}
+            if (set(re.findall(r"\w+", hint.casefold())) - ignored
+                    == set(re.findall(r"\w+", label.casefold())) - ignored):
+                score = max(score, 8)
+            elif matches_field(value["label"], label):
+                score = max(score, 4)
+            elif matches_field(hint, label, owner):
+                score = max(score, 3)
+        if score:
+            scores.append((score, target))
+    best = max((score for score, _ in scores), default=0)
+    matching = [target for score, target in scores if score == best]
+    return matching[0] if len(matching) == 1 else None
+
+
+def input_satisfied(value, target):
+    """Chrome may omit a URL scheme in the displayed address, unlike ordinary text fields."""
+    text = value["text"]
+    if value.get("kind") == "choice" and target.role in {
+            "button", "push_button", "radio_button", "option", "list_item"}:
+        return target.checked is True or target.selected is True
+    if target.input_type == "select-one":
+        return target.value.strip().casefold() == text.strip().casefold()
+    if ("address" in target.name.casefold() and "://" not in target.value
+            and re.match(r"^(?:https?|file)://", text)):
+        text = text.split("://", 1)[1]
+    return target.value == text or value["input_mode"] == "insert" and text in target.value

src/tandem/gui/goal_candidates.py

@@ -8,90 +8,11 @@
 from urllib.parse import unquote, urlsplit
 
 from .candidates import ActionCandidate
+from .field_binding import _input_target, input_satisfied, launcher_input, matches_field
 from .focus import KEYS, input_and_menu_context
 from .targets import _attribute, _local, _pair, _state
 
 
-def matches_field(hint, label):
-    """Match an intended field to its observed label, not an arbitrary focused control."""
-    def words(value):
-        return set(re.findall(r"\w+", value.casefold())) - {"the", "a", "field", "required"}
-
-    observed = words(label)
-    intended = words(hint)
-    return (bool(observed) and (observed <= intended or len(intended) >= 2 and intended <= observed)
-            and not label.startswith(("Unnamed ", "Focused ")))
-
-
-def launcher_input(value):
-    """A launcher search is not a website's identically named Search field."""
-    return (value.get("scope") == "desktop_launcher"
-            or "desktop search" in value.get("target_hint", value["label"]).casefold())
-
-
-def _input_target(value, targets):
-    """Bind to one best observed field; spatial hints must not override its own label."""
-    scores = []
-    for target in targets:
-        choice = value.get("kind") == "choice" and target.role in {
-            "button", "push_button", "radio_button", "option", "list_item"}
-        editable = (target.role in {"textbox", "entry", "searchbox", "spin_button"}
-                    or launcher_input(value) and target.role == "text" and target.focused
-                    or "type_text" in target.operations or target.input_type == "select-one")
-        if not (editable or choice):
-            continue
-        if launcher_input(value) and target.application != "gnome-shell":
-            continue
-        if (target.role == "spin_button" and target.input_type != "date"
-                and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value["text"])):
-            continue  # A complete date is not a Day/Month/Year segment or an ordinary numeric spinner.
-        named = bool(target.name) and not target.name.startswith(("Unnamed ", "Focused "))
-        labels = [target.name] if named else [target.description]
-        if not named and len(target.nearby_text) <= 80 and "\n" not in target.nearby_text:
-            labels.append(target.nearby_text)
-        score = 0
-        if choice:
-            wanted = set(re.findall(r"\w+", value["text"].casefold()))
-            option_name = re.sub(r"\([^)]*\)", "", target.name.casefold())
-            observed = set(re.findall(r"\w+", option_name))
-            hint = set(re.findall(r"\w+", value.get("target_hint", value["label"]).casefold()))
-            if wanted and wanted <= observed <= wanted | hint:
-                score = 6  # Match a specific option, never a container holding the entire form.
-        for label in labels:
-            if not label or not editable:
-                continue
-            hint = value.get("target_hint", value["label"])
-            if not matches_field(hint, label):
-                continue  # Descriptive context words cannot override the explicit intended field.
-            ignored = {"brief", "required", "enter", "input", "field", "the", "a"}
-            if (set(re.findall(r"\w+", hint.casefold())) - ignored
-                    == set(re.findall(r"\w+", label.casefold())) - ignored):
-                score = max(score, 8)
-            elif matches_field(value["label"], label):
-                score = max(score, 4)
-            elif matches_field(hint, label):
-                score = max(score, 3)
-        if score:
-            scores.append((score, target))
-    best = max((score for score, _ in scores), default=0)
-    matching = [target for score, target in scores if score == best]
-    return matching[0] if len(matching) == 1 else None
-
-
-def input_satisfied(value, target):
-    """Chrome may omit a URL scheme in the displayed address, unlike ordinary text fields."""
-    text = value["text"]
-    if value.get("kind") == "choice" and target.role in {
-            "button", "push_button", "radio_button", "option", "list_item"}:
-        return target.checked is True or target.selected is True
-    if target.input_type == "select-one":
-        return target.value.strip().casefold() == text.strip().casefold()
-    if ("address" in target.name.casefold() and "://" not in target.value
-            and re.match(r"^(?:https?|file)://", text)):
-        text = text.split("://", 1)[1]
-    return target.value == text or value["input_mode"] == "insert" and text in target.value
-
-
 def input_frontier(space, targets, inputs):
     """Expose actionable supplied inputs and missing required values on the current page."""
     fields = list(targets)
@@ -226,6 +147,7 @@ def active(element, value):
             elif app is not shell:
                 windows = [e for e in app.iter() if _local(e.tag) in {"frame", "dialog"}]
                 named = [e for e in windows if title and e.get("name") == title]
+                named = [e for e in named if _state(e, "active") is True] or named
                 if named:
                     active(app, False)
                     for window in windows:
@@ -391,10 +313,11 @@ def modal_frontier(space, targets, tree):
     root = ET.fromstring(tree or "<empty/>")
     dialogs = [n for n in root.iter() if _local(n.tag) in {"dialog", "alert"} and _live_dialog(n, root)]
     visual = next((c.metadata for c in space.candidates() if c.metadata.get("blocking_overlay")), {})
-    if not dialogs and not visual:
+    portal = portal_needs_grounding(tree)
+    if not dialogs and not visual and not portal:
         return {}
     dialog = dialogs[-1] if dialogs else None
-    name = dialog.get("name", "Dialog") if dialog is not None else visual["overlay_name"]
+    name = dialog.get("name", "Dialog") if dialog is not None else visual.get("overlay_name", portal)
     position = _pair(_attribute(dialog, "screencoord")) if dialog is not None else None
     size = _pair(_attribute(dialog, "size")) if dialog is not None else None
     bounds = [*position, *size] if position and size else visual.get("overlay_bounds")
@@ -426,7 +349,7 @@ def inside(candidate):
     space.controls = {k: c for k, c in space.controls.items()
                       if k in {"return_to_planner", "visual_fallback"}}
     return {"blocking_overlay": name, "overlay_bounds": bounds,
-            "overlay_kind": visual.get("overlay_kind", "unknown"),
+            "overlay_kind": visual.get("overlay_kind", "task_dialog" if portal else "unknown"),
             "next_intent": f"The active dialog {name!r} owns input; work inside its visible controls."}
 
 

src/tandem/gui/grounding.py

@@ -70,7 +70,8 @@ def ground_visual(client, instruction, plan, screenshot):
                     and re.search(r"date|time|calendar|日期|时间", str(item.get("target_hint", "")), re.I)):
                 continue  # Native date typing requires observed CDP input_type/date_order.
             if ((item.get("text"), item.get("input_mode"), item.get("target_hint")) not in allowed_text
-                    or not matches_field(item.get("target_hint", ""), item.get("target", ""))):
+                    or not matches_field(item.get("target_hint", ""), item.get("target", ""),
+                                         metadata.get("overlay_name", ""))):
                 continue
             metadata.update(text=item["text"], input_mode=item["input_mode"],
                             target_hint=item["target_hint"], field_label=item["target"])

src/tandem/gui/interaction.py

@@ -63,9 +63,12 @@ def target_effect(old, targets):
                     and old["target_semantic"] == t.semantic_key]
         if not intended and old.get("field_label"):
             intended = [t for t in targets if matches_field(old["field_label"], t.name)
-                        and t.role in {"entry", "text", "textbox", "spin_button", "searchbox"}]
+                        and t.role in {"entry", "text", "textbox", "spin_button", "searchbox",
+                                       "password_text"}]
         if intended:
             focus = intended
+        if len(focus) == 1 and focus[0].role == "password_text":
+            return "unknown"  # Login, not a dispatch or masked value, verifies this input.
         if len(focus) == 1 and (focus[0].value or old.get("field_label")):
             if old.get("field_label") and not matches_field(old["field_label"], focus[0].name):
                 return "not_met"