diff --git a/tests/test_plugin.py b/tests/test_plugin.py index e405dfe..31502c6 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -6,7 +6,10 @@ from __future__ import annotations import json +import threading +import time from pathlib import Path +from urllib.parse import parse_qs, urlparse import ebay_plugin import pytest @@ -751,9 +754,11 @@ def test_a_reclaimed_read_goes_through_the_settle_loop(self, monkeypatch): """Review finding: the first fix re-read ONCE after repairing, so a re-read that landed mid-redirect (eBay's captcha hop) failed as "couldn't find the results list".""" import ebay_plugin.tools as tools_mod + from ebay_plugin.extract import search_url - mid_redirect = {"url": _EBAY, "found_container": False, "count": 0} - b = _Reads(_PANEL_READ, mid_redirect, dict(_GOOD_PAGE, url=_EBAY)) + ours = search_url("x", sold=True) # the read must be OUR search, filters and all + mid_redirect = {"url": ours, "found_container": False, "count": 0} + b = _Reads(_PANEL_READ, mid_redirect, dict(_GOOD_PAGE, url=ours)) monkeypatch.setattr(tools_mod, "Browser", lambda **kw: b) monkeypatch.setattr(tools_mod, "_SETTLE_S", 0) out = json.loads({t.name: t for t in build_tools({})}["ebay_price_check"].invoke({"query": "x"})) @@ -908,3 +913,155 @@ def test_a_same_site_misfire_still_settles(monkeypatch): loading = {"url": "https://www.ebay.com/sch/i.html?_nkw=other", "found_container": False, "count": 0} assert tools_mod._is_undecided(loading, _EBAY) is True assert tools_mod._is_undecided(_PANEL_READ, _EBAY) is False + + +class TestLimitSizesOnlyTheSample: + """`limit` used to truncate the listings BEFORE the statistics: limit=1 reported + results_found=1 and a one-row median, though the docs say statistics cover every result.""" + + def test_ebay_limit_trims_the_sample_not_the_statistics(self, monkeypatch, tmp_path): + import ebay_plugin.tools as tools_mod + from ebay_plugin.history import PriceHistory + + monkeypatch.setattr(tools_mod, "Browser", lambda **kw: _StubBrowser(_GOOD_PAGE)) + db = tmp_path / "h.db" + tools = {t.name: t for t in build_tools({"history_db": str(db)})} + out = json.loads(tools["ebay_price_check"].invoke({"query": "switch oled", "limit": 1})) + assert out["results_found"] == 2 and out["stats"]["count"] == 2 + assert len(out["sample"]) == 1 + # the history log records what the statistics were taken over, not the trimmed sample + assert PriceHistory(db).summary("switch oled")["observations"] == 2 + + def test_ebay_no_limit_is_the_default_sample(self, monkeypatch): + rows = [dict(_GOOD_PAGE["rows"][0], url=f"https://www.ebay.com/itm/{i}") for i in range(12)] + tools, _ = _tools(monkeypatch, {**_GOOD_PAGE, "rows": rows}) + out = json.loads(tools["ebay_price_check"].invoke({"query": "switch"})) + assert out["results_found"] == 12 and len(out["sample"]) == 8 + + def test_amazon_limit_trims_the_sample_not_the_statistics(self, monkeypatch, tmp_path): + import ebay_plugin.tools as tools_mod + from ebay_plugin.amazon import normalize + + rows, _ = normalize( + [{"title": f"t{i}", "url": f"https://www.amazon.com/dp/{i}", "price": f"${10 + i}.00"} for i in range(5)] + ) + monkeypatch.setattr(tools_mod, "Browser", lambda **kw: _StubBrowser(_GOOD_PAGE)) + monkeypatch.setattr(tools_mod, "_fetch_amazon", lambda *a, **k: (rows, 0)) + tools = {t.name: t for t in build_tools({"history_db": str(tmp_path / "h.db")})} + out = json.loads(tools["amazon_price_check"].invoke({"query": "x", "limit": 1})) + assert out["results_found"] == 5 and out["stats"]["count"] == 5 + assert len(out["sample"]) == 1 + + +class TestSecondHijackIsAnError: + def test_a_read_still_hijacked_after_the_retry_is_reported_not_priced(self, monkeypatch): + """The docstring promised "a second hijack is reported"; the code returned the foreign + page's data, which then priced whatever that tab showed.""" + import ebay_plugin.tools as tools_mod + + other = dict(_GOOD_PAGE, url="https://www.ebay.com/sch/i.html?_nkw=something+else&LH_Sold=1&LH_Complete=1") + b = _Reads(other) + monkeypatch.setattr(tools_mod, "Browser", lambda **kw: b) + out = json.loads({t.name: t for t in build_tools({})}["ebay_price_check"].invoke({"query": "x"})) + assert out["ok"] is False and "instead of the requested page" in out["error"] + assert b.refocused == 1 and len(b.opened) == 2 # retried exactly once, then reported + + def test_the_probe_still_reports_where_it_landed_rather_than_failing(self, monkeypatch): + import ebay_plugin.tools as tools_mod + + b = _Reads(_PANEL_READ) + monkeypatch.setattr(tools_mod, "Browser", lambda **kw: b) + out = json.loads({t.name: t for t in build_tools({})}["ebay_page_probe"].invoke({"url": _EBAY})) + assert out["ok"] is True and out["not_the_requested_site"] is True + + +_SOLD_X = "https://www.ebay.com/sch/i.html?_nkw=x&LH_Sold=1&LH_Complete=1&_sop=12" + + +@pytest.mark.parametrize( + "landed,requested,hijacked", + [ + # a concurrent ACTIVE search for the same words answering a SOLD request + ("https://www.ebay.com/sch/i.html?_nkw=x&_sop=12", _SOLD_X, True), + # and the reverse + (_SOLD_X, "https://www.ebay.com/sch/i.html?_nkw=x&_sop=12", True), + # same words, another condition + (_SOLD_X + "&LH_ItemCondition=4", _SOLD_X + "&LH_ItemCondition=3", True), + (_SOLD_X + "&LH_ItemCondition=4", _SOLD_X, True), + # same words, another price band + (_SOLD_X + "&_udhi=50", _SOLD_X, True), + # same filters in another order, extra tracking params: our search + ("https://www.ebay.com/sch/i.html?_sop=12&LH_Complete=1&_nkw=X&LH_Sold=1&rt=nc", _SOLD_X, False), + # a page that leaves the sort implicit is not judged on sort + ("https://www.ebay.com/sch/i.html?_nkw=x&LH_Sold=1&LH_Complete=1", _SOLD_X, False), + # but two explicit, different sorts are different searches + (_SOLD_X.replace("_sop=12", "_sop=15"), _SOLD_X, True), + # Amazon: same words, another price band / sort + ("https://www.amazon.com/s?k=x&high-price=20", "https://www.amazon.com/s?k=x", True), + ("https://www.amazon.com/s?k=x&s=price-asc-rank", "https://www.amazon.com/s?k=x&s=price-desc-rank", True), + ("https://www.amazon.com/s?k=x&ref=nb_sb_noss", "https://www.amazon.com/s?k=x", False), + ], +) +def test_a_search_must_match_on_its_filters_too(landed, requested, hijacked): + from ebay_plugin.tools import _hijacked + + assert _hijacked({"url": landed}, requested) is hijacked + + +class _SharedTab(_StubBrowser): + """One tab shared by every caller, like the real labelled tab: a read returns whatever was + navigated to LAST, and the navigate → read gap is wide enough for another call to land in it.""" + + def __init__(self): + super().__init__(None) + self.current, self.active, self.max_active = "", 0, 0 + self._count = threading.Lock() + + def refocus(self): + return "switched" + + def open(self, url): + with self._count: + self.active += 1 + self.max_active = max(self.max_active, self.active) + self.current = url + time.sleep(0.05) + + def eval_json(self, script): + time.sleep(0.05) + url = self.current + with self._count: + self.active -= 1 + terms = parse_qs(urlparse(url).query)["_nkw"][0] + return { + "url": url, + "found_container": True, + "count": 1, + "rows": [{"title": terms, "url": "https://www.ebay.com/itm/1", "price": "$10.00", "shipping": ""}], + } + + +def test_concurrent_price_checks_never_read_each_others_page(monkeypatch, tmp_path): + """Parallel tool calls run on worker threads against ONE tab. Unserialised, call A navigated, + call B navigated, and A read B's results — another query's comps reported as A's.""" + import ebay_plugin.tools as tools_mod + + tab = _SharedTab() + monkeypatch.setattr(tools_mod, "Browser", lambda **kw: tab) + check = {t.name: t for t in build_tools({"history_db": str(tmp_path / "h.db")})}["ebay_price_check"] + queries = [f"item {i}" for i in range(6)] + results: dict[str, dict] = {} + + def run(q): + results[q] = json.loads(check.invoke({"query": q})) + + threads = [threading.Thread(target=run, args=(q,)) for q in queries] + for t in threads: + t.start() + for t in threads: + t.join() + + assert tab.max_active == 1 # one navigate → read sequence at a time + for q in queries: + assert results[q]["ok"] is True, results[q] + assert [row["title"] for row in results[q]["sample"]] == [q] diff --git a/tools.py b/tools.py index cda4f9d..eaa1180 100644 --- a/tools.py +++ b/tools.py @@ -27,6 +27,7 @@ import json import logging +import threading import time from urllib.parse import parse_qs, unquote_plus, urlparse @@ -123,13 +124,40 @@ def _settle(browser: Browser, script: str, data, url: str): return data -def _read_page(browser: Browser, url: str, script: str, selector: str | None = None, *, settle: bool = True): +#: One lock per browser session. Every tool drives the SAME labelled tab, and a read is a +#: navigate → wait → eval sequence: two tool calls interleaving it (the host runs sync tools on +#: worker threads, so parallel tool calls really are concurrent) read each other's page — one +#: query's comps reported as another's. Serialising the sequence is the fix; pacing between +#: navigations already makes the browser a one-at-a-time resource. +_SESSION_LOCKS: dict[str, threading.Lock] = {} +_SESSION_LOCKS_GUARD = threading.Lock() + + +def _session_lock(browser) -> threading.Lock: + key = str(getattr(browser, "session", "") or "") + with _SESSION_LOCKS_GUARD: + return _SESSION_LOCKS.setdefault(key, threading.Lock()) + + +def _read_page( + browser: Browser, + url: str, + script: str, + selector: str | None = None, + *, + settle: bool = True, + strict: bool = True, +): """Navigate, wait, read and settle — the one read path every tool uses. - If the read came from a tab that is not showing the site we navigated to (Chrome's Gemini - side panel, or an operator's tab that took the active slot), take the plugin's tab back, - navigate again and read again through the SAME wait + settle steps. Once: a second - hijack is reported, not chased.""" + The whole sequence holds the session's lock, so concurrent tool calls take turns on the + shared tab instead of reading each other's pages. + + If the read came from a tab that is not showing what we navigated to (Chrome's Gemini + side panel, an operator's tab that took the active slot, or another search), take the + plugin's tab back, navigate again and read again through the SAME wait + settle steps. + Once: a second hijack is reported, not chased — ``strict`` (the default) raises it as an + error; ``strict=False`` returns the read for a caller that reports it itself.""" def attempt(): browser.open(url) @@ -138,12 +166,20 @@ def attempt(): data = browser.eval_json(script) return _settle(browser, script, data, url) if settle else data - data = attempt() - if _hijacked(data, url): - browser.refocus() + with _session_lock(browser): data = attempt() + if _hijacked(data, url): + browser.refocus() + data = attempt() if not isinstance(data, dict): raise EbayError(f"unexpected response while reading {url}") + if strict and _hijacked(data, url): + raise EbayError( + f"the browser answered from {data.get('url')} instead of the requested page, even after taking " + "the plugin's tab back — Chrome's Gemini side panel, another tab, or another search is holding " + "the active slot. Reporting that page's listings would price the wrong thing. Close the panel or " + "tab in the browser window and retry." + ) return data @@ -205,6 +241,17 @@ def _site(host: str) -> str: #: The query parameter that carries a search's terms, per marketplace search path. _SEARCH_TERM = {"/sch/": "_nkw", "/s": "k"} +#: The filter parameters, per marketplace search path, that change WHICH listings a search +#: returns (sold vs active, condition, price band, sort). Two searches for the same words with +#: different filters are different searches: sold comps read off an active-listings page are +#: asking prices reported as sale prices. +_SEARCH_FILTERS = { + "/sch/": ("LH_Sold", "LH_Complete", "LH_ItemCondition", "_udlo", "_udhi", "_sop"), + "/s": ("s", "low-price", "high-price"), +} +#: Of those, the ones that only order the results. +_SORT_KEYS = ("_sop", "s") + def _search_term(parsed) -> tuple[str, str] | None: """``(search path, normalized terms)`` for an eBay or Amazon search URL, else ``None``.""" @@ -215,6 +262,18 @@ def _search_term(parsed) -> tuple[str, str] | None: return None +def _search_filters(parsed, path: str) -> dict[str, frozenset[str]]: + """The filter parameters a search URL states, each as a set of values (eBay joins several + condition ids with ``|``). A parameter that is absent is simply not in the dict.""" + query = parse_qs(parsed.query) + out: dict[str, frozenset[str]] = {} + for key in _SEARCH_FILTERS.get(path, ()): + values = {v.strip() for raw in query.get(key) or [] for v in unquote_plus(raw).split("|") if v.strip()} + if values: + out[key] = frozenset(values) + return out + + def _foreign_host(data, url: str) -> bool: """The read came from a different SITE than the one requested (the Gemini panel, about:blank, an unrelated tab). Only this counts as "decided" for the settle loop: a same-site page that @@ -250,7 +309,20 @@ def _hijacked(data, url: str) -> bool: got = _search_term(got_p) # Judge only a search page that states its own terms: an operator's search always does, # while an eBay rewrite that drops them is not evidence of another tab. - return got is not None and got[1] != "" and got[1] != wanted[1] + if got is None or got[1] == "": + return False + if got[1] != wanted[1]: + return True + # Same words — the filters must match too: a concurrent active-listings search for the + # same item, or the same words filtered to another condition, is not our search. + got_f, want_f = _search_filters(got_p, wanted[0]), _search_filters(want_p, wanted[0]) + for key in _SORT_KEYS: + # A sort changes the order, not the set: judge it only when both pages state one, so a + # page that leaves the default sort implicit is not mistaken for another search. + if key not in got_f or key not in want_f: + got_f.pop(key, None) + want_f.pop(key, None) + return got_f != want_f def _page_meta(data: dict) -> dict: @@ -303,6 +375,16 @@ def _with_meta(payload: dict, meta: dict, found: int) -> dict: return payload +def _sample_size(limit, max_results: int) -> int: + """Rows to list inline: ``limit`` when the caller set one (bounded by ``max_results``), + else the default sample. Never applied to the statistics.""" + try: + n = int(limit or 0) + except (TypeError, ValueError): + n = 0 + return min(n, max_results) if n > 0 else _SAMPLE + + def _as_bool(value, default: bool) -> bool: """A YAML/console flag: real bools pass through; the strings "false"/"no"/"0"/"off" mean False.""" if value is None: @@ -354,7 +436,8 @@ def ebay_price_check( asking price as a sale price. condition: "any", "new" or "used". Statistics cover every result found; only a - sample is listed. Prices include stated shipping. + sample is listed. limit sets how many listings that sample shows (default 8) and + never changes the statistics or results_found. Prices include stated shipping. """ b = _browser() try: @@ -365,10 +448,12 @@ def ebay_price_check( except ValueError as exc: return json.dumps({"ok": False, "error": str(exc)}) - cap = limit if limit and limit > 0 else max_results - listings = listings[:cap] + # `limit` sizes the SAMPLE only. It used to truncate the listings before the + # statistics were taken, so limit=1 reported one result and a one-row "median". + listings = listings[:max_results] _record(query, "ebay", listings) stats = summarize(listings) + shown = _sample_size(limit, max_results) return json.dumps( _with_meta( { @@ -384,7 +469,7 @@ def ebay_price_check( # comps with the same confidence as sixty. "results_found": len(listings), "unparseable_rows_skipped": dropped, - "sample": [x.as_dict() for x in listings[:_SAMPLE]], + "sample": [x.as_dict() for x in listings[:shown]], }, meta, len(listings), @@ -434,7 +519,7 @@ def ebay_session_status() -> str: b = _browser() home = f"https://{domain}" try: - data = _read_page(b, home, SESSION_JS, settle=False) + data = _read_page(b, home, SESSION_JS, settle=False, strict=False) except (BrowserError, EbayError) as exc: return json.dumps({"ok": False, "error": str(exc)}) if _hijacked(data, home): @@ -478,7 +563,7 @@ def ebay_page_probe(url: str) -> str: """ b = _browser() try: - data = _read_page(b, url, RESULT_JS, settle=False) + data = _read_page(b, url, RESULT_JS, settle=False, strict=False) except (BrowserError, EbayError) as exc: return json.dumps({"ok": False, "error": str(exc)}) rows = (data or {}).get("rows") or [] @@ -685,6 +770,9 @@ def amazon_price_check(query: str, sort: str = "relevance", limit: int = 0) -> s buyers actually paid. Use eBay's sold comps for that. Sponsored placements are flagged and excluded from the statistics: an ad is what a seller paid to be shown, not what the market charges. + + limit sets how many listings the sample shows (default 8); it never changes the + statistics or results_found. """ b = _browser() try: @@ -693,9 +781,9 @@ def amazon_price_check(query: str, sort: str = "relevance", limit: int = 0) -> s except (BrowserError, EbayError, ValueError) as exc: return json.dumps({"ok": False, "error": str(exc)}) - cap = limit if limit and limit > 0 else max_results - listings = listings[:cap] + listings = listings[:max_results] # `limit` sizes the sample only — see ebay_price_check _record(query, "amazon", listings) + shown = _sample_size(limit, max_results) return json.dumps( { "ok": True, @@ -706,7 +794,7 @@ def amazon_price_check(query: str, sort: str = "relevance", limit: int = 0) -> s "stats": summarize(listings), "results_found": len(listings), "unparseable_rows_skipped": dropped, - "sample": [x.as_dict() for x in listings[:_SAMPLE]], + "sample": [x.as_dict() for x in listings[:shown]], } )