diff --git a/README.md b/README.md index 388ed5e..77e7090 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,8 @@ ebay: stealth: false # true if your eBay account signs in through Google (below) domain: www.ebay.com # www.ebay.co.uk, www.ebay.de, … amazon_domain: www.amazon.com + nav_timeout_s: 35 # per page load; a hung one is retried once in a fresh tab + cache_ttl_min: 360 # reuse a completed search this long (0 = off) ``` Then ask the agent to run `ebay_session_status`, and **sign in once** in the window that @@ -95,6 +97,33 @@ or passkey login needs none of this. | `compare_prices` | eBay sold + eBay active + Amazon active, side by side, each labelled with its basis. | | `price_history` | Where today's price sits against what we've recorded before. | +**How the browser is driven.** On a protoAgent core whose `agent_browser` plugin offers the +`agent_browser.client` service, every command goes through core's hardened browser client. That +client has an output cap, a process-group kill on a timeout, CLI resolution (including core's +first-use download) and typed results, so a stuck browser reads as a timeout, never as a wall. +On an older core, or with `agent_browser` off, the plugin uses its own CLI wrapper +(`browser.py`). Either way it uses the same session, profile, launch options and labelled tab, +so switching never relaunches Chrome or loses the sign-in. Stealth behaviour is the same on +both paths. + +**Failures say what to do next.** Every error carries an `error_kind`, plus a `next_step` the +agent follows: + +- `timeout`: the browser didn't answer. That is a stuck tab or the site not responding, not + a wall. The plugin has already retried once in a fresh tab. +- `wall`: a sign-in wall or a human check. You act in the browser window. +- `page`: the page couldn't be read. That is a bug to report, never "nothing found". +- `unavailable`, `browser` and `bad_input` cover the rest. + +Selector waits carry an explicit 10 s timeout. An un-timed wait that never succeeded used to +cost about 150 s, because the CLI re-sends it five times. + +**Repeat searches are cached.** `ebay_price_check` and `ebay_price_and_profit` answer a repeat +of the same search (query, sold/active, condition) within `cache_ttl_min` from the last +completed check. The answer says `cached`, with its age. `fresh=true` forces a new reading. +The cache lives in the same local store as the price history (a `checks` table). A cache hit +is not counted as a new observation. + A bundled `ebay-pricing` skill teaches the agent how to narrow a query until the comps are actually comparable, and how to report a price *with its basis*. @@ -151,7 +180,7 @@ provided for sales taken *outside* eBay. ```bash pip install -r requirements-dev.txt -pytest -q # 114 tests, no protoAgent host and no browser required +pytest -q # 209 tests, no protoAgent host and no browser required ruff check . && ruff format --check . ``` diff --git a/browser.py b/browser.py index 9d6a804..d328b6a 100644 --- a/browser.py +++ b/browser.py @@ -43,6 +43,7 @@ import contextlib import json import logging +import os import shutil import subprocess import time @@ -66,7 +67,30 @@ class BrowserError(RuntimeError): - """A browser command failed, or the CLI isn't usable. Message is operator-facing.""" + """A browser command failed, or the CLI isn't usable. Message is operator-facing. + + ``kind`` says which failure it is, so a tool can tell the model what to do next instead of + handing it one undifferentiated string: ``"timeout"`` (the browser didn't answer — a stuck + tab or a site not responding, NOT a wall), ``"wall"`` (a bot check / sign-in wall answered), + ``"unavailable"`` (no usable CLI), or ``"browser"`` (anything else the CLI reported).""" + + kind = "browser" + + +class BrowserTimeout(BrowserError): + kind = "timeout" + + +class BrowserWall(BrowserError): + kind = "wall" + + def __init__(self, message: str, wall_kind: str = ""): + super().__init__(message) + self.wall_kind = wall_kind + + +class BrowserUnavailable(BrowserError): + kind = "unavailable" @dataclass @@ -76,17 +100,33 @@ class Result: stderr: str -def _run(args: list[str], *, timeout: float) -> Result: - """Run the CLI once. The single seam every test stubs.""" +#: The CLI gives up reading a reply after 30 s and RE-SENDS the command, up to five times +#: (agent-browser 0.27.1 ``connection.rs``). The daemon's default wait is also 30 s, so a wait +#: that never succeeds (a gate page with no results) cost ~150 s — measured: 152 s for one +#: un-timed ``wait``. Every wait here carries its own timeout, well under that. +_WAIT_TIMEOUT_S = 10.0 +_TIMED_OUT = "timed out after" + + +def _run(args: list[str], *, timeout: float, env: dict | None = None) -> Result: + """Run the CLI once. The single seam every test stubs. ``env`` is merged over this + process's environment for this one call (a wait's ``AGENT_BROWSER_DEFAULT_TIMEOUT``).""" try: - p = subprocess.run(args, capture_output=True, text=True, timeout=timeout) # noqa: S603 + p = subprocess.run( # noqa: S603 + args, capture_output=True, text=True, timeout=timeout, env={**os.environ, **env} if env else None + ) except subprocess.TimeoutExpired: - return Result(False, "", f"timed out after {timeout:.0f}s") + return Result(False, "", f"{_TIMED_OUT} {timeout:.0f}s") except FileNotFoundError: return Result(False, "", f"{args[0]!r} not found on PATH") return Result(p.returncode == 0, p.stdout or "", p.stderr or "") +def _timed_out(res: Result) -> bool: + text = (res.stderr or "") + (res.stdout or "") + return _TIMED_OUT in text or "daemon may be busy or unresponsive" in text + + class Browser: """Drives one named ``agent-browser`` session. @@ -106,6 +146,7 @@ def __init__( stealth: bool = False, timeout_s: float = 60.0, min_interval_s: float = 1.5, + nav_timeout_s: float = 35.0, ): self.binary = binary self.session = session @@ -114,6 +155,10 @@ def __init__( self.stealth = stealth self.timeout_s = timeout_s self.min_interval_s = min_interval_s + # Shorter than the per-command timeout, and just past the CLI's 30 s read timeout: a + # navigation the daemon never answers (a wedged tab) is re-sent by the CLI up to five + # times, so waiting the full per-command timeout only buys more re-sends. + self.nav_timeout_s = min(nav_timeout_s, timeout_s) self._last_nav = 0.0 self._session_ready = False self.tab_label = _TAB_LABEL @@ -163,11 +208,11 @@ def ensure_session(self) -> None: if self._session_ready: return if reason := self.available(): - raise BrowserError(reason) + raise BrowserUnavailable(reason) try: self.focus_own_tab() except BrowserError as exc: - raise BrowserError(f"could not start the browser: {exc}") from None + raise type(exc)(f"could not start the browser: {exc}") from None self._session_ready = True def close(self) -> None: @@ -234,9 +279,16 @@ def focus_own_tab(self) -> str: log.warning("[ebay] the plugin's browser tab would not take focus — replaced it") return "recreated" if not res.ok: - raise BrowserError(self._last_line(res)) + raise (BrowserTimeout if _timed_out(res) else BrowserError)(self._last_line(res)) return "created" + def reset_tab(self) -> None: + """After a navigation that timed out: close Chrome's panels and replace OUR tab (the + only one carrying our label) with a fresh one. Never touches another tab.""" + self.close_panels() + _run(self._tab_cmd("close", self.tab_label), timeout=self.timeout_s) + _run(self._tab_cmd("new", "--label", self.tab_label), timeout=self.timeout_s) + def refocus(self) -> str: """Another target answered instead of our tab: close Chrome's panels, take our tab back.""" self.close_panels() @@ -249,17 +301,30 @@ def open(self, url: str) -> None: try: self.focus_own_tab() except BrowserError as exc: - raise BrowserError(f"could not switch to the plugin's browser tab: {exc}") from None + raise type(exc)(f"could not switch to the plugin's browser tab: {exc}") from None # Flags on every navigation: a daemon that died (an operator's `close`, a newer # CLI's idle timeout) respawns with our profile instead of a blank one. - res = _run(self._open_cmd(url), timeout=self.timeout_s) + res = _run(self._open_cmd(url), timeout=self.nav_timeout_s) if not res.ok and _BLOCKED_MARKER in (res.stderr + res.stdout): # A target took the active slot between our switch and the navigation (the Gemini # panel reopening). Take the tab back and try exactly once more. self.refocus() - res = _run(self._open_cmd(url), timeout=self.timeout_s) + res = _run(self._open_cmd(url), timeout=self.nav_timeout_s) + if not res.ok and _timed_out(res): + # The one automatic recovery: a hung navigation usually means a wedged tab, so + # replace our tab and try once more before reporting a typed timeout. + log.warning("[ebay] opening %s timed out — replacing the plugin's tab and retrying once", url) + self.reset_tab() + with contextlib.suppress(BrowserError): + self.focus_own_tab() + res = _run(self._open_cmd(url), timeout=self.nav_timeout_s) + if not res.ok and _timed_out(res): + raise BrowserTimeout( + f"{url} did not load within {self.nav_timeout_s:.0f}s, twice (the second time in a fresh tab)" + ) if not res.ok: - raise BrowserError(f"could not open {url}: {(res.stderr or res.stdout).strip()}") + cls = BrowserTimeout if _timed_out(res) else BrowserError + raise cls(f"could not open {url}: {(res.stderr or res.stdout).strip()}") def wait_for(self, selector: str, *, timeout_s: float | None = None) -> bool: """Block until ``selector`` appears. ``True`` if it did. @@ -272,7 +337,15 @@ def wait_for(self, selector: str, *, timeout_s: float | None = None) -> bool: and decides what it's looking at. """ self.ensure_session() - res = _run(self._cmd("wait", selector), timeout=timeout_s or self.timeout_s) + secs = min(timeout_s or _WAIT_TIMEOUT_S, 25.0) + # The CLI takes a selector wait's timeout from AGENT_BROWSER_DEFAULT_TIMEOUT (it injects it + # into the command). Without it the daemon waits 30 s — exactly the CLI's read timeout — and + # the CLI re-sends the wait up to five times (see _WAIT_TIMEOUT_S). + res = _run( + self._cmd("wait", selector), + timeout=secs + 10.0, + env={"AGENT_BROWSER_DEFAULT_TIMEOUT": str(int(secs * 1000))}, + ) return res.ok def eval_json(self, script: str): @@ -291,9 +364,9 @@ def eval_json(self, script: str): args, input=script, capture_output=True, text=True, timeout=self.timeout_s ) except subprocess.TimeoutExpired: - raise BrowserError(f"page script timed out after {self.timeout_s:.0f}s") from None + raise BrowserTimeout(f"page script timed out after {self.timeout_s:.0f}s") from None except FileNotFoundError: - raise BrowserError(f"{self.binary!r} not found on PATH") from None + raise BrowserUnavailable(f"{self.binary!r} not found on PATH") from None if p.returncode != 0: raise BrowserError(f"page script failed: {(p.stderr or p.stdout).strip()}") return _parse_cli_json(p.stdout) diff --git a/coreclient.py b/coreclient.py new file mode 100644 index 0000000..6140ce6 --- /dev/null +++ b/coreclient.py @@ -0,0 +1,132 @@ +"""Drive the browser through protoAgent core's browser client when the host offers one. + +Core's ``agent_browser`` plugin offers ``agent_browser.client`` (a plugin service, ADR 0116): its +own hardened path to the same ``agent-browser`` CLI. That path has an output cap, a process-group +kill on timeout, CLI resolution (including the pinned first-use download), the argv option guard, +a lock shared with the browser tools, and typed results that tell a stuck browser from a wall. +:class:`CoreBrowser` adapts it to the small surface the tools use (``open`` / ``wait_for`` / +``eval_json`` / ``refocus``), so ``tools.py`` doesn't care which path runs. + +**The fallback stays.** Fleet members on a core older than the client, or with ``agent_browser`` +disabled, get ``None`` from the lookup, and the tools use :class:`browser.Browser` exactly as +before. The lookup runs on every tool call, never at ``register()``: plugins load in no fixed +order, and a reload can add or remove the provider. + +Same browser either way. The session keeps its configured name verbatim +(``instance_scoped=False``), so the daemon and the profile the operator signed in to are the ones +already running. It sends the same launch options (``--profile``, ``--headed``, and the one +``stealth`` Chrome flag when that's on), so switching paths never relaunches Chrome. It uses the +same labelled tab (``ebaytab``). Stealth and anti-detection behaviour are unchanged: the client +adds no launch options of its own. +""" + +from __future__ import annotations + +import logging + +from .browser import _STEALTH_ARGS, _TAB_LABEL, BrowserError, BrowserTimeout, BrowserUnavailable, BrowserWall + +log = logging.getLogger("protoagent.plugins.ebay") + +#: A selector wait. Short on purpose: a results list that hasn't appeared in 10 s usually never +#: will (a gate page), and the tools' settle loop re-reads a slow page anyway. +WAIT_TIMEOUT_S = 10.0 + + +def core_factory(): + """The host's ``agent_browser.client`` factory, or ``None``: no host (tests, CLI), a core + older than the client or the service seam, or ``agent_browser`` turned off.""" + try: + from graph import sdk + except Exception: # noqa: BLE001 — host-free + return None + lookup = getattr(sdk, "service", None) + if not callable(lookup): + return None + try: + return lookup("agent_browser.client") + except Exception: # noqa: BLE001 — a lookup must never fail a tool call + log.exception("[ebay] looking up the core browser client failed") + return None + + +def _raise_for(result, what: str): + """Map a failed core ``Result`` to this plugin's typed errors.""" + kind = getattr(result, "kind", "cli_error") + error = getattr(result, "error", "") or kind + if kind == "timeout": + raise BrowserTimeout(f"{what}: {error}") + if kind == "wall": + wall = getattr(result, "wall", None) + raise BrowserWall(f"{what}: {error}", getattr(wall, "kind", "")) + if kind == "unavailable": + raise BrowserUnavailable(error) + raise BrowserError(f"{what}: {error}") + + +class CoreBrowser: + """The tools' browser surface over core's client. ``factory`` is the + ``agent_browser.client`` service; options mirror :class:`browser.Browser`'s.""" + + def __init__( + self, + factory, + *, + session: str, + profile: str, + headed: bool, + stealth: bool, + timeout_s: float, + nav_timeout_s: float, + min_interval_s: float, + ): + self.stealth = stealth + self.session = session # tools._session_lock keys the read sequence by it + self.client = factory( + session=session, + instance_scoped=False, + profile=profile, + headed=headed, + browser_args=[_STEALTH_ARGS] if stealth else [], + own_tab=_TAB_LABEL, + timeout_s=timeout_s, + nav_timeout_s=min(nav_timeout_s, timeout_s), + min_interval_s=min_interval_s, + ) + + def open(self, url: str) -> None: + # check_walls=False: the tools' own settle-and-classify loop reads eBay's gate chain + # (search → /splashui/captcha → sign-in → back), which a probe right after the navigation + # would catch mid-chain and wrongly call a wall. The client still recovers a hung + # navigation once (a fresh tab) before reporting a typed timeout. + r = self.client.open(url, check_walls=False) + if not r.ok: + _raise_for(r, f"could not open {url}") + + def wait_for(self, selector: str, *, timeout_s: float | None = None) -> bool: + # Best-effort, as in Browser.wait_for: a timeout is a legitimate answer (a gate page + # never shows results). A stuck browser shows up on the read that follows. + return self.client.wait(selector=selector, timeout_s=timeout_s or WAIT_TIMEOUT_S).ok + + def eval_json(self, script: str): + r = self.client.eval_json(script) + if not r.ok: + _raise_for(r, "page script failed") + return r.value + + def refocus(self) -> str: + self.client.close_panels() + r = self.client.focus_tab() + if not r.ok: + _raise_for(r, "could not switch to the plugin's browser tab") + return r.value + + def check_wall(self, url: str): + """A generic wall (Cloudflare, a 403/429, a vendor CAPTCHA) on the current page, as + core's wall label, or ``""``. Run only when the tools' own read found no results and no + eBay-specific reason, to name what the page actually is.""" + r = self.client.check_wall(url) + return r.wall.label if getattr(r, "kind", "") == "wall" and r.wall is not None else "" + + def close(self) -> None: + self.client.close() diff --git a/extract.py b/extract.py index a50537f..f31a71a 100644 --- a/extract.py +++ b/extract.py @@ -32,6 +32,13 @@ CONDITIONS = {"new": "3", "used": "4", "any": ""} +def effective_condition(condition: object) -> str: + """The condition filter a search really applies: ``new``/``used`` in any case, else ``any``. + The comps cache keys on this too, so it never files unfiltered results under ``used``.""" + c = str(condition or "").strip().lower() + return c if c in CONDITIONS else "any" + + def search_url( query: str, *, @@ -57,7 +64,7 @@ def search_url( params["LH_Complete"] = "1" if sort_code := SORTS.get(sort): params["_sop"] = sort_code - if cond := CONDITIONS.get(condition, ""): + if cond := CONDITIONS[effective_condition(condition)]: params["LH_ItemCondition"] = cond if min_price is not None: params["_udlo"] = f"{min_price:g}" @@ -228,8 +235,10 @@ def summarize(listings: list[Listing]) -> dict: "low": round(totals[0], 2), "high": round(totals[-1], 2), # Quartiles bound "what it usually goes for" without the tails. - "p25": round(totals[max(0, int(n * 0.25) - (1 if n % 4 == 0 else 0))], 2), - "p75": round(totals[min(n - 1, int(n * 0.75))], 2), + # Nearest-rank quartiles (index ceil(q*n) - 1), the same rule for both: [10,20,30,40] + # gives p25 10 and p75 30. p75 used int(n*0.75), which returned the max when 4 | n. + "p25": round(totals[max(0, -(-n // 4) - 1)], 2), + "p75": round(totals[max(0, -(-3 * n // 4) - 1)], 2), "basis": "item price + shipping, where shipping was stated", } if excluded: diff --git a/history.py b/history.py index 327a018..623e3d1 100644 --- a/history.py +++ b/history.py @@ -21,6 +21,7 @@ from __future__ import annotations +import json import re import sqlite3 from contextlib import closing @@ -38,6 +39,20 @@ seen_at TEXT NOT NULL -- ISO-8601 UTC ); CREATE INDEX IF NOT EXISTS idx_obs_key ON observations(key, seen_at); +-- One row per completed search: what the tool read, so a repeat of the same search inside +-- the cache window is answered from here instead of the browser. Separate from +-- `observations` (which carries no sold/active basis or condition, and must not be +-- double-counted by a cache hit). +CREATE TABLE IF NOT EXISTS checks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + key TEXT NOT NULL, -- normalized query + source TEXT NOT NULL, -- ebay@ (the eBay site searched) | amazon + basis TEXT NOT NULL, -- sold | active + condition TEXT NOT NULL, -- any | new | used + seen_at TEXT NOT NULL, -- ISO-8601 UTC + payload TEXT NOT NULL -- JSON: listings, dropped, meta, url +); +CREATE INDEX IF NOT EXISTS idx_checks_key ON checks(key, source, basis, condition, seen_at); """ @@ -90,6 +105,67 @@ def record(self, key: str, source: str, listings, *, now: str | None = None) -> db.commit() return len(rows) + def record_check( + self, + key: str, + source: str, + *, + basis: str, + condition: str, + listings, + dropped: int, + meta: dict, + url: str, + now: str | None = None, + ) -> None: + """Remember one completed search in full, for :meth:`recent_check`. Every listing is + kept (sponsored rows too) so a cached answer is computed exactly as a live one.""" + payload = { + "listings": [x.as_dict() for x in listings], + "dropped": int(dropped or 0), + "meta": dict(meta or {}), + "url": url, + } + with closing(self._connect()) as db: + db.execute( + "INSERT INTO checks (key, source, basis, condition, seen_at, payload) VALUES (?,?,?,?,?,?)", + ( + normalize_key(key), + source, + basis, + (condition or "any").lower(), + now or datetime.now(UTC).isoformat(), + json.dumps(payload), + ), + ) + db.commit() + + def recent_check( + self, key: str, source: str, *, basis: str, condition: str, max_age_min: float, now: datetime | None = None + ) -> dict | None: + """The newest :meth:`record_check` for this exact search within ``max_age_min``, or + ``None``. Returns the stored payload plus ``seen_at`` and ``age_min``.""" + if not max_age_min or max_age_min <= 0: + return None + now = now or datetime.now(UTC) + since = (now - timedelta(minutes=max_age_min)).isoformat() + with closing(self._connect()) as db: + row = db.execute( + "SELECT seen_at, payload FROM checks WHERE key = ? AND source = ? AND basis = ? AND condition = ? " + "AND seen_at >= ? ORDER BY seen_at DESC LIMIT 1", + (normalize_key(key), source, basis, (condition or "any").lower(), since), + ).fetchone() + if row is None: + return None + try: + payload = json.loads(row["payload"]) + except ValueError: + return None + seen = datetime.fromisoformat(row["seen_at"]) + payload["seen_at"] = row["seen_at"] + payload["age_min"] = max(0, int((now - seen).total_seconds() // 60)) + return payload + def summary(self, key: str, *, days: int = 90, source: str = "") -> dict: """Where the record sits over the last ``days``. diff --git a/protoagent.plugin.yaml b/protoagent.plugin.yaml index 42f69cb..eac58c8 100644 --- a/protoagent.plugin.yaml +++ b/protoagent.plugin.yaml @@ -1,6 +1,6 @@ id: ebay name: Marketplace prices (eBay + Amazon, via browser) -version: 0.3.3 +version: 0.4.0 description: >- Price research across eBay and Amazon — what an item actually SOLD for on eBay, what it is being asked on both, and where today's price sits against what we've recorded before. @@ -15,7 +15,11 @@ description: >- you signed in. No API keys, and this plugin never handles your credentials. Requires the `agent-browser` binary on PATH (`npm i -g agent-browser && agent-browser - install`). Ships DISABLED; enable with `plugins: { enabled: [ebay] }`. + install`). On a protoAgent core whose `agent_browser` plugin offers the + `agent_browser.client` service, the browser is driven through core's hardened client (output + cap, process-group kill, CLI resolution, typed timeouts and walls); on an older core, through + this plugin's own CLI wrapper. Same session, profile, launch options and tab either way. + Ships DISABLED; enable with `plugins: { enabled: [ebay] }`. enabled: false repository: https://github.com/protoLabsAI/ebay-plugin min_protoagent_version: "0.27.0" @@ -55,6 +59,15 @@ config: history_db: "" # Per-command subprocess timeout (seconds). timeout_s: 60 + # Per-navigation timeout (seconds). Just past agent-browser's 30 s read timeout: a navigation + # the browser never answers (a wedged tab) is re-sent by the CLI up to five times, so a longer + # wait only buys more re-sends. A timed-out navigation is retried once in a fresh tab, then + # reported as a timeout (never as a wall or a market finding). + nav_timeout_s: 35 + # Minutes a completed eBay search is reused: a repeat of the SAME search (query, sold/active, + # condition) inside this window is answered from the last check, with no browser trip, and + # says so (`cached`). The tools' fresh=true skips it. 0 turns it off. + cache_ttl_min: 360 # Minimum gap between navigations. Politeness, not evasion: it keeps a research loop from # hitting the site faster than a person would. Raise it if eBay starts serving challenges. min_interval_s: 1.5 diff --git a/pyproject.toml b/pyproject.toml index e92414b..a5a3420 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ebay-plugin" -version = "0.3.3" +version = "0.4.0" description = "Marketplace price research (eBay + Amazon) + seller economics for protoAgent, via a browser session" requires-python = ">=3.11" license = "MIT" diff --git a/skills/ebay-pricing/SKILL.md b/skills/ebay-pricing/SKILL.md index c66e64a..6315043 100644 --- a/skills/ebay-pricing/SKILL.md +++ b/skills/ebay-pricing/SKILL.md @@ -70,9 +70,21 @@ market claim. by themselves; if it still fails, retry the call once, then stop and report it — it is not an eBay problem and not a market finding. (A domain allowlist configured on the browser CLI produces the same error; that one is configuration.) +- **`error_kind: "timeout"`** → the browser didn't answer: a stuck tab or eBay not responding. + It is NOT a sign-in wall, NOT a bot check and NOT a market finding. The plugin has already + retried once in a fresh tab. Retry the call at most once more; if it times out again, stop and + tell the operator the browser looks stuck. Never loop on it. +- **`error_kind: "wall"`** → a person must act in the browser window (sign in, or clear a + check). Say which, and wait for the operator. - **couldn't find the results list** → eBay changed its markup. Run `ebay_page_probe` on the URL and report what it shows. This is a bug to file, not a market finding. +## Cached answers + +A repeat of the same search within a few hours comes back with `cached: {age_minutes, …}` and a +note. It is the plugin's own earlier check, not a new reading. Repeat that note. Pass +`fresh=true` when the operator wants today's numbers, or when an earlier answer looked wrong. + ## What this cannot tell you - **Sell-through rate.** Sold comps show what sold, never how many *didn't*. A high median diff --git a/tests/conftest.py b/tests/conftest.py index 816ce53..5530abf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,3 +38,14 @@ def register_skill_dir(self, path): @pytest.fixture def registry(): return FakeRegistry() + + +@pytest.fixture(autouse=True) +def _private_history_db(tmp_path, monkeypatch): + """Every test gets its own price-history / comps-cache file. Host-free, the default path is + ``~/.protoagent/marketplace-prices.db``, so the suite used to write its fixture rows ("x", + "switch oled") into the operator's real home directory, and with the comps cache one test's + search would answer the next one's.""" + import ebay_plugin.tools as tools_mod + + monkeypatch.setattr(tools_mod, "_default_history_db", lambda: str(tmp_path / "prices.db")) diff --git a/tests/test_core_client.py b/tests/test_core_client.py new file mode 100644 index 0000000..7d1340d --- /dev/null +++ b/tests/test_core_client.py @@ -0,0 +1,389 @@ +"""The core-client path (coreclient.py), typed failures, the comps cache, and the fallback's timeouts. + +Host-free: the host's ``agent_browser.client`` factory is replaced by a fake whose client answers +with Result-shaped objects, the way core's ``BrowserClient`` does. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest +from ebay_plugin import browser as browser_mod +from ebay_plugin import coreclient +from ebay_plugin import tools as tools_mod +from ebay_plugin.browser import Browser, BrowserTimeout +from ebay_plugin.history import PriceHistory +from ebay_plugin.tools import build_tools + +_PAGE = { + "found_container": True, + "challenge": False, + "signin_wall": False, + "rows": [ + { + "title": "Switch OLED", + "url": "https://www.ebay.com/itm/1", + "price": "$200.00", + "shipping": "+$10.00 delivery", + }, + { + "title": "Switch OLED bundle", + "url": "https://www.ebay.com/itm/2", + "price": "$300.00", + "shipping": "Free delivery", + }, + ], +} + + +def _r(kind="ok", value=None, error="", wall=None): + return SimpleNamespace(kind=kind, ok=kind == "ok", value=value, error=error, wall=wall) + + +class FakeClient: + """Answers like core's BrowserClient. ``script`` maps a method name to a Result or a list.""" + + def __init__(self, **opts): + self.opts, self.calls = opts, [] + self.answers = {"open": _r(value={"url": "", "title": ""}), "eval_json": _r(value=_PAGE)} + + def _answer(self, name): + a = self.answers.get(name, _r()) + return a.pop(0) if isinstance(a, list) else a + + def open(self, url, **kw): + self.calls.append(("open", url, kw)) + return self._answer("open") + + def wait(self, **kw): + self.calls.append(("wait", kw)) + return self._answer("wait") + + def eval_json(self, script): + self.calls.append(("eval_json",)) + return self._answer("eval_json") + + def close_panels(self): + self.calls.append(("close_panels",)) + return _r(value=[]) + + def focus_tab(self, label=""): + self.calls.append(("focus_tab",)) + return _r(value="switched") + + def check_wall(self, url=""): + self.calls.append(("check_wall", url)) + return self._answer("check_wall") + + def close(self): + return _r() + + +@pytest.fixture +def core(monkeypatch): + """Install a fake ``agent_browser.client`` factory; returns the list of clients it built.""" + built = [] + + def factory(**opts): + c = FakeClient(**opts) + built.append(c) + return c + + monkeypatch.setattr(tools_mod, "core_factory", lambda: factory) + return built + + +def _tool(name, cfg=None): + return {t.name: t for t in build_tools(cfg or {})}[name] + + +# ── which path runs ────────────────────────────────────────────────────────────────────────── + + +def test_the_core_client_is_used_when_the_host_offers_it(core): + out = json.loads( + _tool("ebay_price_check", {"profile": "/p/ebay", "session": "ebay"}).invoke({"query": "switch oled"}) + ) + assert out["ok"] and out["results_found"] == 2 + (c,) = core + # The same browser the fallback drives: the session verbatim, the same options, the same tab, + # and nothing added — no stealth flag unless stealth is on. + assert c.opts["session"] == "ebay" and c.opts["instance_scoped"] is False + assert c.opts["profile"] == "/p/ebay" and c.opts["headed"] is True + assert c.opts["browser_args"] == [] and c.opts["own_tab"] == "ebaytab" + assert c.opts["nav_timeout_s"] == 35 and c.opts["min_interval_s"] == 1.5 + # The tools' own settle-and-classify loop reads eBay's gate chain, not core's wall probe. + assert c.calls[0][0] == "open" and c.calls[0][2] == {"check_walls": False} + assert ("wait", {"selector": "li.s-card, li.s-item", "timeout_s": coreclient.WAIT_TIMEOUT_S}) in c.calls + + +def test_stealth_sends_exactly_the_one_flag_it_always_did(core): + _tool("ebay_price_check", {"stealth": True}).invoke({"query": "x"}) + assert core[0].opts["browser_args"] == ["--disable-blink-features=AutomationControlled"] + + +def test_the_fallback_runs_when_there_is_no_core_client(monkeypatch): + monkeypatch.setattr(tools_mod, "core_factory", lambda: None) + used = [] + + class Stub: + session, stealth = "ebay", False + + def open(self, url): + used.append(url) + + def wait_for(self, selector, timeout_s=None): + return True + + def eval_json(self, script): + return _PAGE + + monkeypatch.setattr(tools_mod, "Browser", lambda **kw: Stub()) + assert json.loads(_tool("ebay_price_check").invoke({"query": "x"}))["ok"] and used + + +def test_a_new_factory_after_a_reload_builds_a_new_client(monkeypatch): + factories = [] + + def make_factory(): + def factory(**opts): + return FakeClient(**opts) + + factories.append(factory) + return factory + + current = {"f": make_factory()} + monkeypatch.setattr(tools_mod, "core_factory", lambda: current["f"]) + check = _tool("ebay_price_check", {"cache_ttl_min": 0}) + check.invoke({"query": "x"}) + check.invoke({"query": "y"}) + current["f"] = make_factory() + assert json.loads(check.invoke({"query": "z"}))["ok"] + + +def test_no_host_means_no_factory(): + assert coreclient.core_factory() is None + + +# ── typed failures ─────────────────────────────────────────────────────────────────────────── + + +def test_a_timeout_is_typed_with_a_next_step(core, monkeypatch): + def factory(**opts): + c = FakeClient(**opts) + c.answers["open"] = _r("timeout", error="did not load within 35s, twice") + return c + + monkeypatch.setattr(tools_mod, "core_factory", lambda: factory) + out = json.loads(_tool("ebay_price_check").invoke({"query": "x"})) + assert out["ok"] is False and out["error_kind"] == "timeout" + assert "NOT a sign-in wall" in out["next_step"] and "at most once more" in out["next_step"] + + +def test_a_core_wall_and_an_unavailable_cli_are_typed(monkeypatch): + for kind, expect in (("wall", "wall"), ("unavailable", "unavailable"), ("cli_error", "browser")): + + def factory(_kind=kind, **opts): + c = FakeClient(**opts) + c.answers["open"] = _r(_kind, error="nope", wall=SimpleNamespace(kind="cloudflare", label="x")) + return c + + monkeypatch.setattr(tools_mod, "core_factory", lambda f=factory: f) + out = json.loads(_tool("ebay_price_check").invoke({"query": "x"})) + assert out["error_kind"] == expect, (kind, out) + + +def _core_with(monkeypatch, **answers): + """A fake factory whose clients start with ``answers`` (method name → Result).""" + built = [] + + def factory(**opts): + c = FakeClient(**opts) + c.answers.update(answers) + built.append(c) + return c + + monkeypatch.setattr(tools_mod, "core_factory", lambda: factory) + return built + + +def test_ebays_own_walls_are_typed_as_walls(monkeypatch): + for flag in ("signin_wall", "challenge"): + _core_with(monkeypatch, eval_json=_r(value={**_PAGE, "rows": [], flag: True})) + out = json.loads(_tool("ebay_price_check").invoke({"query": "x"})) + assert out["error_kind"] == "wall" and "browser window" in out["next_step"], flag + + +def test_a_generic_wall_is_named_before_blaming_the_markup(monkeypatch): + built = _core_with( + monkeypatch, + eval_json=_r(value={**_PAGE, "found_container": False, "rows": []}), + check_wall=_r("wall", wall=SimpleNamespace(kind="http_403", label="an HTTP 403 Forbidden response")), + ) + out = json.loads(_tool("ebay_price_check").invoke({"query": "x"})) + assert out["error_kind"] == "wall" and "an HTTP 403 Forbidden response" in out["error"] + assert any(call[0] == "check_wall" for call in built[0].calls) + + +def test_a_page_with_no_container_and_no_wall_is_still_the_markup_error(monkeypatch): + _core_with(monkeypatch, eval_json=_r(value={**_PAGE, "found_container": False, "rows": []}), check_wall=_r()) + out = json.loads(_tool("ebay_price_check").invoke({"query": "x"})) + assert out["error_kind"] == "page" and "results list" in out["error"] + + +def test_bad_input_is_typed(core): + out = json.loads(_tool("ebay_price_check").invoke({"query": " "})) + assert out["ok"] is False and out["error_kind"] == "bad_input" + + +# ── the comps cache ────────────────────────────────────────────────────────────────────────── + + +def _opens(core): + return sum(1 for c in core for call in c.calls if call[0] == "open") + + +def test_a_repeat_search_is_answered_from_the_cache(core, tmp_path): + db = tmp_path / "h.db" + tool = _tool("ebay_price_check", {"history_db": str(db)}) + first = json.loads(tool.invoke({"query": "Switch OLED"})) + second = json.loads(tool.invoke({"query": "switch oled"})) + assert _opens(core) == 1 + assert "cached" not in first and second["cached"]["age_minutes"] == 0 + assert second["stats"] == first["stats"] and second["results_found"] == first["results_found"] + assert any("no browser trip" in n for n in second["notes"]) + # A cache hit is not a second observation in the price history. + assert PriceHistory(db).summary("switch oled")["observations"] == 2 + + +def test_fresh_and_a_different_search_go_to_the_browser(core, tmp_path): + tool = _tool("ebay_price_check", {"history_db": str(tmp_path / "h.db")}) + tool.invoke({"query": "x"}) + tool.invoke({"query": "x", "fresh": True}) + tool.invoke({"query": "x", "sold": False}) + tool.invoke({"query": "x", "condition": "used"}) + assert _opens(core) == 4 + + +def test_the_cache_keys_on_the_filter_actually_applied_and_the_ebay_site(core, tmp_path): + # QA (#8): condition='Used' used to search unfiltered but cache under 'used'; and a repeat on + # another eBay domain was answered with the first site's comps. + db = str(tmp_path / "h.db") + tool = _tool("ebay_price_check", {"history_db": db}) + tool.invoke({"query": "x", "condition": "Used"}) + assert json.loads(tool.invoke({"query": "x", "condition": "used"}))["cached"] # same filter + assert "cached" not in json.loads(tool.invoke({"query": "x", "condition": "any"})) + uk = _tool("ebay_price_check", {"history_db": db, "domain": "www.ebay.co.uk"}) + assert "cached" not in json.loads(uk.invoke({"query": "x", "condition": "used"})) + assert _opens(core) == 3 + + +def test_search_url_honours_condition_in_any_case(): + from ebay_plugin import extract as from_extract + + assert "LH_ItemCondition=4" in from_extract.search_url("x", condition="Used") + assert "LH_ItemCondition" not in from_extract.search_url("x", condition="refurbished") + assert from_extract.effective_condition(" NEW ") == "new" + assert from_extract.effective_condition("refurbished") == "any" + + +def test_price_and_profit_shares_the_sold_cache(core, tmp_path): + cfg = {"history_db": str(tmp_path / "h.db")} + _tool("ebay_price_check", cfg).invoke({"query": "x"}) + out = json.loads(_tool("ebay_price_and_profit", cfg).invoke({"query": "x", "item_cost": 50})) + assert out["ok"] and out["cached"] and out["at_market_prices"] + assert _opens(core) == 1 + + +def test_the_cache_can_be_turned_off(core, tmp_path): + tool = _tool("ebay_price_check", {"history_db": str(tmp_path / "h.db"), "cache_ttl_min": 0}) + tool.invoke({"query": "x"}) + tool.invoke({"query": "x"}) + assert _opens(core) == 2 + + +def test_a_failed_search_is_never_cached(core, tmp_path): + tool = _tool("ebay_price_check", {"history_db": str(tmp_path / "h.db")}) + tool.invoke({"query": "warmup"}) + core[0].answers["open"] = [_r("timeout", error="stuck"), _r(value={})] + assert json.loads(tool.invoke({"query": "x"}))["ok"] is False + assert "cached" not in json.loads(tool.invoke({"query": "x"})) + + +def test_recent_check_respects_the_window(tmp_path): + from datetime import UTC, datetime, timedelta + + from ebay_plugin.extract import normalize + + h = PriceHistory(tmp_path / "h.db") + listings, _ = normalize(_PAGE["rows"], sold=True) + old = (datetime.now(UTC) - timedelta(hours=7)).isoformat() + h.record_check("x", "ebay", basis="sold", condition="any", listings=listings, dropped=0, meta={}, url="u", now=old) + assert h.recent_check("x", "ebay", basis="sold", condition="any", max_age_min=360) is None + hit = h.recent_check("x", "ebay", basis="sold", condition="any", max_age_min=480) + assert hit and hit["age_min"] >= 419 and len(hit["listings"]) == 2 + assert h.recent_check("x", "ebay", basis="active", condition="any", max_age_min=480) is None + + +# ── the fallback's timeouts ────────────────────────────────────────────────────────────────── + + +def _fallback(monkeypatch, answer): + monkeypatch.setattr(browser_mod.shutil, "which", lambda b: "/usr/bin/agent-browser") + calls = [] + + def fake(args, timeout, env=None): + calls.append((args, timeout, env)) + return answer(args) + + monkeypatch.setattr(browser_mod, "_run", fake) + return Browser(session="ebay", min_interval_s=0), calls + + +def test_the_fallback_wait_carries_an_explicit_timeout(monkeypatch): + b, calls = _fallback(monkeypatch, lambda args: browser_mod.Result(True, "", "")) + b._session_ready = True + b.wait_for("li.s-card") + args, timeout, env = calls[-1] + assert args[:2] == ["agent-browser", "wait"] and env == {"AGENT_BROWSER_DEFAULT_TIMEOUT": "10000"} + assert timeout < 30 # the subprocess never outlives the CLI's first 30 s read + + +def test_a_hung_fallback_navigation_recovers_once_in_a_fresh_tab(monkeypatch): + state = {"opens": 0} + + def answer(args): + if args[1] == "open": + state["opens"] += 1 + if state["opens"] == 1: + return browser_mod.Result(False, "", "timed out after 35s") + return browser_mod.Result(True, '{"success":true,"data":{"tabs":[]}}', "") + + b, calls = _fallback(monkeypatch, answer) + b.open("https://www.ebay.com/sch/i.html?_nkw=x") + verbs = [a[1:3] for a, _, _ in calls] + assert ["tab", "close"] in verbs and state["opens"] == 2 + assert all(t == 35 for a, t, _ in calls if a[1] == "open") # nav_timeout_s, not timeout_s + + +def test_a_second_fallback_timeout_is_typed(monkeypatch): + def answer(args): + if args[1] == "open": + return browser_mod.Result(False, "", "timed out after 35s") + return browser_mod.Result(True, '{"success":true,"data":{"tabs":[]}}', "") + + b, _ = _fallback(monkeypatch, answer) + with pytest.raises(BrowserTimeout) as exc: + b.open("https://www.ebay.com/") + assert exc.value.kind == "timeout" and "twice" in str(exc.value) + + +def test_quartiles_use_one_nearest_rank_rule(): + # QA (#8): p75 used int(n*0.75), which returned the max whenever 4 divided n. + from ebay_plugin.extract import Listing, summarize + + rows = [Listing("t", "u", float(p), None, "$", 0.0, "", "", True) for p in (10, 20, 30, 40)] + stats = summarize(rows) + assert (stats["p25"], stats["p75"], stats["high"]) == (10, 30, 40) diff --git a/tools.py b/tools.py index eaa1180..d4dbd9c 100644 --- a/tools.py +++ b/tools.py @@ -32,8 +32,9 @@ from urllib.parse import parse_qs, unquote_plus, urlparse from . import amazon -from .browser import Browser, BrowserError -from .extract import RESULT_JS, SESSION_JS, normalize, search_url, summarize +from .browser import Browser, BrowserError, BrowserWall +from .coreclient import CoreBrowser, core_factory +from .extract import RESULT_JS, SESSION_JS, Listing, effective_condition, normalize, search_url, summarize from .fees import Costs, breakeven_price, net_proceeds, schedule_from_config log = logging.getLogger("protoagent.plugins.ebay") @@ -57,7 +58,44 @@ def _default_history_db() -> str: class EbayError(RuntimeError): - """An operator-facing failure. Message is safe to show verbatim.""" + """An operator-facing failure. Message is safe to show verbatim. ``kind`` is ``"wall"`` + (sign-in wall / human check) or ``"page"`` (the page couldn't be read).""" + + def __init__(self, message: str, kind: str = "page"): + super().__init__(message) + self.kind = kind + + +#: What to do next, per failure kind — given to the model with every error so it can tell a +#: stuck browser (retry once, then stop) from a wall (a human must act) from a page it can't +#: read (a bug to report). The audit behind it: timeouts used to read like any other error, so +#: the agent retried a stuck browser for minutes, or reported it as a market finding. +_NEXT_STEP = { + "timeout": ( + "The browser did not respond in time — a stuck browser tab or the site not answering. This is " + "NOT a sign-in wall, a bot check or a market finding. The plugin already retried once in a fresh " + "tab: retry this call at most once more, and if it times out again stop and tell the operator " + "the browser looks stuck." + ), + "wall": ( + "A person has to act in the browser window (sign in, or complete the check). Tell the operator; " + "don't retry until they say it's done." + ), + "unavailable": "The browser isn't set up on this machine. Tell the operator; retrying won't help.", + "browser": "Retry once; if the same error comes back, report it verbatim — it is not a market finding.", + "page": "This is not an empty result. Report it as a read failure, never as 'nothing found'.", + "bad_input": "Fix the arguments and call again.", +} + + +def _failure(exc: Exception) -> dict: + """The JSON a tool returns for a failure: the message, a stable ``error_kind`` and the + ``next_step`` for that kind.""" + kind = "bad_input" if isinstance(exc, ValueError) else (getattr(exc, "kind", "") or "browser") + out = {"ok": False, "error": str(exc), "error_kind": kind, "next_step": _NEXT_STEP.get(kind, _NEXT_STEP["browser"])} + if isinstance(exc, BrowserWall) and exc.wall_kind: + out["wall"] = exc.wall_kind + return out #: What "the results loaded" looks like — either result layout. Waited for after every @@ -87,23 +125,45 @@ def _is_undecided(data, url: str = "") -> bool: _AMAZON_SELECTOR = 'div[data-component-type="s-search-result"]' -def _classify(data, url: str, *, marketplace: str) -> None: +def _name_generic_wall(browser, url: str, marketplace: str) -> None: + """No results and no eBay-specific reason: before calling it a markup change, ask core's + generic wall check (Cloudflare, a 403/429, a vendor CAPTCHA) what the page is. Only the core + client offers it; the fallback browser skips straight to the markup error.""" + check = getattr(browser, "check_wall", None) + if not callable(check): + return + try: + label = check(url) + except Exception: # noqa: BLE001 — a diagnostic must never replace the real error + return + if label: + raise BrowserWall( + f"{marketplace} answered with {label} instead of results. Complete or clear it in the browser " + "window, then retry — this plugin hands that to you rather than working around it.", + "generic", + ) + + +def _classify(data, url: str, *, marketplace: str, browser=None) -> None: """Raise the right error for a page that carries no results. Shared by both sources because both gate the same three ways — sign-in, bot check, or a page we can't read — and all three arrive as a card-less DOM.""" if data.get("signin_wall"): raise EbayError( f"{marketplace} redirected to its sign-in page. Run the session-status tool, sign in " - "once in the browser window, and retry. The profile keeps you signed in after that." + "once in the browser window, and retry. The profile keeps you signed in after that.", + "wall", ) if data.get("challenge"): raise EbayError( f"{marketplace} is asking for human verification (a CAPTCHA / bot check) instead of " "returning results. Complete it in the browser window, then retry — this plugin " "deliberately hands that to you rather than trying to work around it. If it keeps " - "recurring, raise min_interval_s to slow the search cadence." + "recurring, raise min_interval_s to slow the search cadence.", + "wall", ) if not data.get("found_container"): + _name_generic_wall(browser, url, marketplace) landed = str(data.get("url") or "").strip() where = f" The browser ended up at: {landed}" if landed and landed not in url else "" raise EbayError( @@ -172,13 +232,14 @@ def attempt(): browser.refocus() data = attempt() if not isinstance(data, dict): - raise EbayError(f"unexpected response while reading {url}") + raise EbayError(f"unexpected response while reading {url}", "page") if strict and _hijacked(data, url): raise EbayError( f"the browser answered from {data.get('url')} instead of the requested page, even after taking " "the plugin's tab back — Chrome's Gemini side panel, another tab, or another search is holding " "the active slot. Reporting that page's listings would price the wrong thing. Close the panel or " - "tab in the browser window and retry." + "tab in the browser window and retry.", + "browser", ) return data @@ -190,7 +251,7 @@ def _read(browser: Browser, url: str, script: str, selector: str): def _fetch_amazon(browser: Browser, url: str): data = _read(browser, url, amazon.RESULT_JS, _AMAZON_SELECTOR) - _classify(data, url, marketplace="Amazon") + _classify(data, url, marketplace="Amazon", browser=browser) return amazon.normalize(data.get("rows") or []) @@ -208,16 +269,19 @@ def _fetch(browser: Browser, url: str, *, sold: bool): raise EbayError( "eBay redirected to its sign-in page. The sold-listings view needs a signed-in " "session — run ebay_session_status, sign in once in the browser window, and retry. " - "The profile keeps you signed in after that." + "The profile keeps you signed in after that.", + "wall", ) if data.get("challenge"): raise EbayError( "eBay is asking for human verification (its 'Security Measure' / CAPTCHA page) " "instead of returning results. Complete it in the browser window, then retry — " "this plugin deliberately hands that to you rather than trying to work around it. " - "If it keeps recurring, raise ebay.min_interval_s to slow the search cadence." + "If it keeps recurring, raise ebay.min_interval_s to slow the search cadence.", + "wall", ) if not data.get("found_container"): + _name_generic_wall(browser, url, "eBay") # Name where the browser actually ENDED UP. Without it this error is unfalsifiable — # "the markup changed" reads the same whether eBay redesigned the page or simply # bounced you somewhere else, and only one of those is a bug to file. @@ -364,6 +428,14 @@ def _page_notes(meta: dict, found: int) -> list[str]: return notes +def _with_cached(cached: dict | None, payload: dict) -> dict: + """Mark an answer served from the comps cache, and say so in ``notes`` the model repeats.""" + if cached: + payload["cached"] = cached + payload.setdefault("notes", []).append(f"These comps are {cached['note']}.") + return payload + + def _with_meta(payload: dict, meta: dict, found: int) -> dict: payload["headline_count"] = meta.get("headline_count") payload["related_rows_excluded"] = meta.get("related_rows_excluded", 0) @@ -405,21 +477,91 @@ def build_tools(cfg: dict): # THIS PLUGIN had just started and tripped the "someone else owns the browser" guard on # itself — the first search worked and every one after it failed. Sharing the instance # also keeps the pacing clock honest across calls, which a per-call object silently reset. - shared = Browser( - binary=cfg.get("binary") or "agent-browser", - session=cfg.get("session") or "ebay", - profile=cfg.get("profile") or "", - headed=_as_bool(cfg.get("headed"), True), - stealth=_as_bool(cfg.get("stealth"), False), - timeout_s=float(cfg.get("timeout_s", 60)), - min_interval_s=float(cfg.get("min_interval_s", 1.5)), - ) - - def _browser() -> Browser: - return shared + opts = { + "session": cfg.get("session") or "ebay", + "profile": cfg.get("profile") or "", + "headed": _as_bool(cfg.get("headed"), True), + "stealth": _as_bool(cfg.get("stealth"), False), + "timeout_s": float(cfg.get("timeout_s", 60)), + "nav_timeout_s": float(cfg.get("nav_timeout_s", 35)), + "min_interval_s": float(cfg.get("min_interval_s", 1.5)), + } + shared = Browser(binary=cfg.get("binary") or "agent-browser", **opts) + # The core-client path, built once per factory: the pacing clock lives on it, and a plugin + # reload hands out a NEW factory (with core's new lock table), which must not keep the old one. + core: dict = {"factory": None, "browser": None} + + def _browser(): + """Core's hardened browser client when the host offers it (``agent_browser.client``), + else this plugin's own CLI wrapper — looked up per call (coreclient module doc).""" + factory = core_factory() + if factory is None: + return shared + if core["factory"] is not factory: + try: + core["browser"] = CoreBrowser(factory, **opts) + core["factory"] = factory + except Exception: # noqa: BLE001 — a client that won't build must not take the tools down + log.exception("[ebay] the core browser client could not be built; using the plugin's own") + return shared + log.info("[ebay] driving the browser through core's agent_browser.client") + return core["browser"] domain = cfg.get("domain") or "www.ebay.com" max_results = int(cfg.get("max_results", 60)) + # A repeat of the same search inside this window is answered from the plugin's own record of + # the last check instead of the browser (0 = off). Sold comps move over days, not minutes, and + # every browser trip costs seconds and an eBay page view. + cache_ttl_min = float(cfg.get("cache_ttl_min", 360) or 0) + + def _comps(query: str, *, sold: bool, condition: str, fresh: bool): + """eBay comps for a search — from a recent check of the exact same search when there is + one (``cached`` says so), else from the browser (and remembered). Returns + ``(listings, dropped, meta, url, cached)``.""" + url = search_url(query, domain=domain, sold=sold, condition=condition) + basis = "sold" if sold else "active" + # Key on what the search really is: the filter search_url applied and the eBay site it + # ran on (www.ebay.com and www.ebay.co.uk comps are different markets). + cache_source = f"ebay@{str(domain).strip().lower()}" + cache_condition = effective_condition(condition) + h = _history() if cache_ttl_min > 0 else None + if h is not None and not fresh: + try: + hit = h.recent_check( + query, cache_source, basis=basis, condition=cache_condition, max_age_min=cache_ttl_min + ) + except Exception: # noqa: BLE001 — the cache is an optimisation, never a failure + log.exception("[ebay] reading the comps cache failed") + hit = None + if hit is not None: + try: + listings = [Listing(**row) for row in hit.get("listings") or []] + except TypeError: # a row from a different Listing shape — read it fresh instead + listings = None + if listings is not None: + cached = { + "checked_at": hit["seen_at"], + "age_minutes": hit["age_min"], + "note": f"from this plugin's own check {hit['age_min']} min ago (no browser trip); " + "pass fresh=true for a new reading", + } + return listings, int(hit.get("dropped") or 0), hit.get("meta") or {}, url, cached + listings, dropped, meta = _fetch(_browser(), url, sold=sold) + if h is not None: + try: + h.record_check( + query, + cache_source, + basis=basis, + condition=cache_condition, + listings=listings, + dropped=dropped, + meta=meta, + url=url, + ) + except Exception: # noqa: BLE001 + log.exception("[ebay] recording the comps cache failed") + return listings, dropped, meta, url, None @tool def ebay_price_check( @@ -427,6 +569,7 @@ def ebay_price_check( condition: str = "any", sold: bool = True, limit: int = 0, + fresh: bool = False, ) -> str: """What does this item actually go for on eBay? Returns median/quartile price statistics plus a sample of matching listings. @@ -437,42 +580,44 @@ def ebay_price_check( condition: "any", "new" or "used". Statistics cover every result found; only a sample is listed. limit sets how many listings that sample shows (default 8) and - never changes the statistics or results_found. Prices include stated shipping. + never changes the statistics or results_found. Prices include stated shipping. A repeat + of the same search within a few hours answers from the last check (`cached` says so); + fresh=True forces a new one. """ - b = _browser() try: - url = search_url(query, domain=domain, sold=sold, condition=condition) - listings, dropped, meta = _fetch(b, url, sold=sold) - except (BrowserError, EbayError) as exc: - return json.dumps({"ok": False, "error": str(exc)}) - except ValueError as exc: - return json.dumps({"ok": False, "error": str(exc)}) + listings, dropped, meta, url, cached = _comps(query, sold=sold, condition=condition, fresh=fresh) + except (BrowserError, EbayError, ValueError) as exc: + return json.dumps(_failure(exc)) # `limit` sizes the SAMPLE only. It used to truncate the listings before the # statistics were taken, so limit=1 reported one result and a one-row "median". listings = listings[:max_results] - _record(query, "ebay", listings) + if cached is None: + _record(query, "ebay", listings) # a cache hit is not a new observation stats = summarize(listings) shown = _sample_size(limit, max_results) return json.dumps( - _with_meta( - { - "ok": True, - "query": query, - "basis": "sold listings — what buyers paid" - if sold - else "active listings — asking prices, NOT sale prices", - "condition": condition, - "url": url, - "stats": stats, - # Surfaced so the model can qualify a thin result instead of treating three - # comps with the same confidence as sixty. - "results_found": len(listings), - "unparseable_rows_skipped": dropped, - "sample": [x.as_dict() for x in listings[:shown]], - }, - meta, - len(listings), + _with_cached( + cached, + _with_meta( + { + "ok": True, + "query": query, + "basis": "sold listings — what buyers paid" + if sold + else "active listings — asking prices, NOT sale prices", + "condition": condition, + "url": url, + "stats": stats, + # Surfaced so the model can qualify a thin result instead of treating three + # comps with the same confidence as sixty. + "results_found": len(listings), + "unparseable_rows_skipped": dropped, + "sample": [x.as_dict() for x in listings[:shown]], + }, + meta, + len(listings), + ), ) ) @@ -491,7 +636,7 @@ def ebay_search(query: str, sort: str = "best_match", condition: str = "any", so url = search_url(query, domain=domain, sold=sold, sort=sort, condition=condition) listings, dropped, meta = _fetch(b, url, sold=sold) except (BrowserError, EbayError, ValueError) as exc: - return json.dumps({"ok": False, "error": str(exc)}) + return json.dumps(_failure(exc)) return json.dumps( _with_meta( { @@ -521,7 +666,7 @@ def ebay_session_status() -> str: try: data = _read_page(b, home, SESSION_JS, settle=False, strict=False) except (BrowserError, EbayError) as exc: - return json.dumps({"ok": False, "error": str(exc)}) + return json.dumps(_failure(exc)) if _hijacked(data, home): # Another tab answered twice. Saying "not signed in" here would send the operator to # sign in again for nothing — say what actually happened instead. @@ -565,7 +710,7 @@ def ebay_page_probe(url: str) -> str: try: data = _read_page(b, url, RESULT_JS, settle=False, strict=False) except (BrowserError, EbayError) as exc: - return json.dumps({"ok": False, "error": str(exc)}) + return json.dumps(_failure(exc)) rows = (data or {}).get("rows") or [] return json.dumps( { @@ -627,7 +772,7 @@ def ebay_net_proceeds( costs=_costs(item_cost, shipping_cost, other_costs, labor_hours, hourly_rate), ) except ValueError as exc: - return json.dumps({"ok": False, "error": str(exc)}) + return json.dumps(_failure(exc)) return json.dumps({"ok": True, **result}) @tool @@ -658,7 +803,7 @@ def ebay_breakeven( promoted=bool(promoted), ) except ValueError as exc: - return json.dumps({"ok": False, "error": str(exc)}) + return json.dumps(_failure(exc)) return json.dumps( { "ok": True, @@ -680,6 +825,7 @@ def ebay_price_and_profit( condition: str = "any", labor_hours: float = 0.0, hourly_rate: float = 0.0, + fresh: bool = False, ) -> str: """Price an item against real sold comps AND show what you'd net at those prices. The one-call answer to "what should I list this at?". @@ -687,21 +833,27 @@ def ebay_price_and_profit( through eBay's fees and your costs, so the answer is net profit at each price rather than a headline number that fees quietly eat. """ - b = _browser() try: - url = search_url(query, domain=domain, sold=True, condition=condition) - listings, dropped, meta = _fetch(b, url, sold=True) + listings, dropped, meta, url, cached = _comps(query, sold=True, condition=condition, fresh=fresh) except (BrowserError, EbayError, ValueError) as exc: - return json.dumps({"ok": False, "error": str(exc)}) + return json.dumps(_failure(exc)) listings = listings[:max_results] stats = summarize(listings) if not stats.get("count"): return json.dumps( - _with_meta( - {"ok": True, "query": query, "stats": stats, "note": "no sold comps found — try a broader query"}, - meta, - 0, + _with_cached( + cached, + _with_meta( + { + "ok": True, + "query": query, + "stats": stats, + "note": "no sold comps found — try a broader query", + }, + meta, + 0, + ), ) ) @@ -720,25 +872,28 @@ def ebay_price_and_profit( } return json.dumps( - _with_meta( - { - "ok": True, - "query": query, - "basis": "sold listings — what buyers paid", - "url": url, - "stats": stats, - "results_found": len(listings), - "unparseable_rows_skipped": dropped, - "breakeven_price": breakeven_price(schedule=schedule, costs=costs), - "at_market_prices": scenarios, - "your_costs": costs.total, - "schedule": schedule.as_dict(), - "caveat": "" - if schedule.verified - else "Fee schedule is UNVERIFIED — confirm against a real eBay invoice.", - }, - meta, - len(listings), + _with_cached( + cached, + _with_meta( + { + "ok": True, + "query": query, + "basis": "sold listings — what buyers paid", + "url": url, + "stats": stats, + "results_found": len(listings), + "unparseable_rows_skipped": dropped, + "breakeven_price": breakeven_price(schedule=schedule, costs=costs), + "at_market_prices": scenarios, + "your_costs": costs.total, + "schedule": schedule.as_dict(), + "caveat": "" + if schedule.verified + else "Fee schedule is UNVERIFIED — confirm against a real eBay invoice.", + }, + meta, + len(listings), + ), ) ) @@ -779,7 +934,7 @@ def amazon_price_check(query: str, sort: str = "relevance", limit: int = 0) -> s url = amazon.search_url(query, domain=amazon_domain, sort=sort) listings, dropped = _fetch_amazon(b, url) except (BrowserError, EbayError, ValueError) as exc: - return json.dumps({"ok": False, "error": str(exc)}) + return json.dumps(_failure(exc)) listings = listings[:max_results] # `limit` sizes the sample only — see ebay_price_check _record(query, "amazon", listings) @@ -887,7 +1042,7 @@ def _run(key, basis, fetch): except (BrowserError, EbayError, ValueError) as exc: # Named, not omitted: a missing source that looks like an absent one turns a # half-answer into a confident whole one. - out["sources"][key] = {"ok": False, "error": str(exc), "basis": basis} + out["sources"][key] = {**_failure(exc), "basis": basis} _run( "ebay_sold",