diff --git a/README.md b/README.md index c0741fc..388ed5e 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ hand, once, in a browser window. | **Headed only** | eBay answers headless browsers with an error page. | | **Rate-limited** | Hit either site hard and you get verification pages. `min_interval_s` paces requests; raise it if that happens. | | **Markup isn't a contract** | Selectors were verified live and will eventually break. The tools distinguish "no results" from "couldn't read the page" and report the second as a bug. | +| **Thin searches are padded** | A search with few exact matches gets a "Results matching fewer words" river of loosely related items. The tools count only the exact matches and report the rest as `related_rows_excluded`, with eBay's own `headline_count` alongside. | This plugin makes **no attempt to defeat bot detection** — no proxy rotation, no fingerprint spoofing, no CAPTCHA solving. It drives a normal browser at a human pace. When eBay asks for diff --git a/extract.py b/extract.py index d59c6d1..a50537f 100644 --- a/extract.py +++ b/extract.py @@ -263,12 +263,38 @@ def summarize(listings: list[Listing]) -> dict: for (const sel of CARD) { const f = document.querySelectorAll(sel); if (f.length) { cards = [...f]; break; } } const container = document.querySelector("ul.srp-results, .srp-river-results, .srp-results__list, [data-testid='search-results']"); + // A search with few (or no) exact matches is PADDED: eBay renders the exact matches, then + // a divider reading "Results matching fewer words", then dozens of loosely related items + // (other teams' dice, complete boxes, $1 transfer sheets). Verified live 2026-09-12 on a + // query whose headline said "0 results": 2 filler cards, the divider, then 54 cards — + // which this script used to report as 54 sold comps with a median. Everything below the + // divider is excluded from `rows` and counted in `related_rows_excluded` instead. + const leafText = (el) => (el.children.length === 0 ? (el.textContent || "").trim() : ""); + const divider = [...document.querySelectorAll("li, div, h2, h3, h4, span, p")].find((el) => + /^results matching fewer words$|^results? for similar (searches|items)$/i.test(leafText(el)) + ) || null; + const afterDivider = (c) => !!divider && !!(divider.compareDocumentPosition(c) & Node.DOCUMENT_POSITION_FOLLOWING); + // eBay's own count of exact matches ("1,234 results for …", or "0 results for …" on a + // padded page). Read so the caller can tell "few comps" from "none, and the rest is filler". + const headText = pick(document, [".srp-controls__count-heading", "h1.srp-controls__count-heading", + ".srp-controls__count", "[data-testid='count-heading']"]) || + (document.body ? document.body.innerText.slice(0, 3000) : ""); + const headMatch = headText.match(/([\d,]+)\+?\s*results?\b/i); + const headline_count = headMatch ? parseInt(headMatch[1].replace(/,/g, ""), 10) : null; + // "Showing results for X — Search instead for Y": eBay rewrote the query, so EVERY card is + // for its query, not the caller's. + const controlsText = pick(document, [".srp-controls", ".srp-rewrite", ".srp-save-null-search"]) || + (document.body ? document.body.innerText.slice(0, 3000) : ""); + const query_rewritten = /search instead for|showing results for/i.test(controlsText); + const MONEYISH = /[$£€¥]\s*[\d.,]+/; const SHIPPING = /(delivery|shipping|postage|freight)/i; const rows = []; + let related_rows_excluded = 0; for (const c of cards) { const title = pick(c, [".s-item__title", ".s-card__title", "[role='heading']", "h3"]); if (!title || /^shop on ebay$/i.test(title)) continue; // eBay's own filler card + if (afterDivider(c)) { related_rows_excluded += 1; continue; } const a = c.querySelector("a.s-item__link, a.s-card__link, a[href*='/itm/']"); let price = pick(c, [".s-item__price", ".s-card__price"]); @@ -313,6 +339,9 @@ def summarize(listings: list[Listing]) -> dict: ), signin_wall: /signin\.ebay/.test(location.href + " " + (document.referrer || "")), count: rows.length, + headline_count, + related_rows_excluded, + query_rewritten, rows, }); })() diff --git a/protoagent.plugin.yaml b/protoagent.plugin.yaml index 41d0fb5..8a87151 100644 --- a/protoagent.plugin.yaml +++ b/protoagent.plugin.yaml @@ -1,6 +1,6 @@ id: ebay name: Marketplace prices (eBay + Amazon, via browser) -version: 0.3.0 +version: 0.3.1 description: >- Price research across eBay and Amazon — what an item actually SOLD for on eBay, what it is being asked on both, and where today's price sits against what we've recorded before. diff --git a/pyproject.toml b/pyproject.toml index 7b81cc7..9f81fdc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ebay-plugin" -version = "0.3.0" +version = "0.3.1" description = "Marketplace price research (eBay + Amazon) + seller economics for protoAgent, via a browser session" requires-python = ">=3.11" diff --git a/skills/ebay-pricing/SKILL.md b/skills/ebay-pricing/SKILL.md index cddb6c0..c905172 100644 --- a/skills/ebay-pricing/SKILL.md +++ b/skills/ebay-pricing/SKILL.md @@ -19,8 +19,9 @@ current asking prices" are different claims and must never be stated the same wa 1. **Search the item as a buyer would type it.** Include the details that change the price — model number, capacity, edition, bundle — and nothing that doesn't. -2. **Read `results_found` before the statistics.** Under ~10 comps is a weak signal; say so - rather than quoting a median to the cent off five data points. +2. **Read `results_found`, `headline_count` and `notes` before the statistics.** Under ~10 + comps is a weak signal; say so rather than quoting a median to the cent off five data + points. A non-empty `notes` list is part of the answer, not a footnote. 3. **Sanity-check the spread.** If `high` is many times `low`, or `p25` and `p75` are far apart, the query is catching different things — a lot of 10 next to a single unit, or a different model. Narrow it and search again. A tight p25–p75 band is what makes a @@ -38,6 +39,18 @@ current asking prices" are different claims and must never be stated the same wa $20 item shipped free, and comparing item prices alone gets that backwards. - `unparseable_rows_skipped` is rows with no readable price. A handful is normal. A large number next to a small `results_found` means treat the statistics as provisional. +- **`headline_count` and `related_rows_excluded`.** eBay pads a search with few exact matches: + it shows the matches, then a divider reading "Results matching fewer words", then dozens of + loosely related items. Only the exact matches are counted in `results_found` and the + statistics; the padding is reported as `related_rows_excluded`. `headline_count` is eBay's + own count of exact matches. `results_found: 0` with `related_rows_excluded: 54` means eBay + found NOTHING for that query — broaden it; never price from the excluded rows. +- **`query_rewritten: true`** means eBay silently searched for something else ("Showing results + for …"). Every number on that page is for its query, not yours. Say so and re-query. +- **The same quartiles showing up for different items** (five items all at p25 $20.22) is the + signature of padding or a rewrite, not a market fact. Stop and look at the page. +- Repeat every entry in `notes` when you report a number. They exist because a padded page once + came back as "54 sold comps, median $31" for a query eBay matched to nothing. ## When a tool returns an error diff --git a/tests/test_plugin.py b/tests/test_plugin.py index b053db1..98b61e7 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -532,3 +532,91 @@ def test_session_status_has_no_next_step_when_signed_in(self, monkeypatch): out = json.loads(by_name["ebay_session_status"].invoke({})) assert out["signed_in"] is True assert out["next_step"] == "" + + +_PADDED_PAGE = { + "found_container": True, + "challenge": False, + "signin_wall": False, + "count": 0, + "headline_count": 0, + "related_rows_excluded": 54, + "query_rewritten": False, + "rows": [], +} + + +class TestPaddedResults: + """eBay pads a search with few exact matches: the matches, a 'Results matching fewer words' + divider, then dozens of loosely related items. Through 0.3.0 those were counted as comps — + a query eBay itself matched to NOTHING came back as '54 sold, median $31'.""" + + def test_a_zero_match_page_reports_no_comps_and_says_why(self, monkeypatch): + by_name, _ = _tools(monkeypatch, _PADDED_PAGE) + out = json.loads(by_name["ebay_price_check"].invoke({"query": "x"})) + assert out["ok"] is True + assert out["results_found"] == 0 + assert out["stats"]["count"] == 0 + assert out["headline_count"] == 0 + assert out["related_rows_excluded"] == 54 + assert any("54" in n and "fewer words" in n for n in out["notes"]) + assert any("do not price from those" in n.lower() or "broaden" in n.lower() for n in out["notes"]) + + def test_price_and_profit_names_the_padding_when_there_are_no_comps(self, monkeypatch): + by_name, _ = _tools(monkeypatch, _PADDED_PAGE) + out = json.loads(by_name["ebay_price_and_profit"].invoke({"query": "x", "item_cost": 10})) + assert out["ok"] is True + assert out["stats"]["count"] == 0 + assert out["related_rows_excluded"] == 54 + assert any("fewer words" in n for n in out["notes"]) + + def test_a_thin_match_next_to_padding_is_flagged(self, monkeypatch): + page = {**_GOOD_PAGE, "headline_count": 2, "related_rows_excluded": 40} + by_name, _ = _tools(monkeypatch, page) + out = json.loads(by_name["ebay_price_check"].invoke({"query": "x"})) + assert out["results_found"] == 2 + assert out["stats"]["count"] == 2 # statistics over the exact matches only + assert any("only 2" in n and "40" in n for n in out["notes"]) + + def test_a_healthy_page_with_some_padding_just_says_so(self, monkeypatch): + rows = [dict(_GOOD_PAGE["rows"][0], url=f"https://www.ebay.com/itm/{i}") for i in range(12)] + page = {**_GOOD_PAGE, "rows": rows, "count": 12, "headline_count": 12, "related_rows_excluded": 5} + by_name, _ = _tools(monkeypatch, page) + out = json.loads(by_name["ebay_price_check"].invoke({"query": "x"})) + assert out["results_found"] == 12 + assert out["notes"] == [ + "5 loosely related listings below eBay's 'Results matching fewer words' divider were excluded from the statistics." + ] + + def test_a_clean_page_has_no_notes(self, monkeypatch): + by_name, _ = _tools(monkeypatch, _GOOD_PAGE) + out = json.loads(by_name["ebay_price_check"].invoke({"query": "x"})) + assert "notes" not in out + assert out["related_rows_excluded"] == 0 + assert out["headline_count"] is None # older payloads / unknown markup: not claimed + + def test_a_rewritten_query_is_called_out(self, monkeypatch): + page = {**_GOOD_PAGE, "query_rewritten": True} + by_name, _ = _tools(monkeypatch, page) + out = json.loads(by_name["ebay_search"].invoke({"query": "x"})) + assert out["query_rewritten"] is True + assert any("rewrote" in n for n in out["notes"]) + + def test_compare_carries_the_notes_per_ebay_source(self, monkeypatch): + import ebay_plugin.tools as tools_mod + + b = _StubBrowser(_PADDED_PAGE) + monkeypatch.setattr(tools_mod, "Browser", lambda **kw: b) + monkeypatch.setattr(tools_mod, "_fetch_amazon", lambda *a, **k: ([], 0)) + out = json.loads({t.name: t for t in build_tools({})}["compare_prices"].invoke({"query": "x"})) + assert out["sources"]["ebay_sold"]["related_rows_excluded"] == 54 + assert any("fewer words" in n for n in out["sources"]["ebay_sold"]["notes"]) + assert "notes" not in out["sources"]["amazon_active"] # the Amazon reader has no padding meta + + def test_the_page_script_reports_the_new_fields(self): + """The JS itself can't run here; pin that it emits the keys the tools read.""" + from ebay_plugin.extract import RESULT_JS + + for key in ("headline_count", "related_rows_excluded", "query_rewritten"): + assert key in RESULT_JS + assert "Results matching fewer words" in RESULT_JS or "results matching fewer words" in RESULT_JS diff --git a/tools.py b/tools.py index acb4643..e3d3136 100644 --- a/tools.py +++ b/tools.py @@ -176,7 +176,57 @@ def _fetch(browser: Browser, url: str, *, sold: bool): f"matches'.{where} Run ebay_page_probe on the URL for the details." ) listings, dropped = normalize(data.get("rows") or [], sold=sold) - return listings, dropped + return listings, dropped, _page_meta(data) + + +def _page_meta(data: dict) -> dict: + """What the page said about itself beyond the rows: eBay's own match count, how many + padded "fewer words" rows were left out, and whether it rewrote the query.""" + head = data.get("headline_count") + return { + "headline_count": int(head) if isinstance(head, (int, float)) else None, + "related_rows_excluded": int(data.get("related_rows_excluded") or 0), + "query_rewritten": bool(data.get("query_rewritten")), + } + + +def _page_notes(meta: dict, found: int) -> list[str]: + """Plain-language qualifiers the model must repeat. A padded page used to come back as + "54 sold comps, median $31" for a query eBay itself matched to nothing.""" + notes: list[str] = [] + related = int(meta.get("related_rows_excluded") or 0) + head = meta.get("headline_count") + divider = "below eBay's 'Results matching fewer words' divider" + if found == 0 and (related or head == 0): + notes.append( + f"eBay found no listings matching all the words in this query (its own count: {head}); " + f"{related} loosely related listings {divider} were excluded. Broaden the query — do not " + "price from those." + ) + elif related and found < 5: + notes.append( + f"only {found} listings matched the whole query; {related} loosely related listings {divider} " + "were excluded. A thin result — broaden the query before quoting a number." + ) + elif related: + notes.append(f"{related} loosely related listings {divider} were excluded from the statistics.") + if meta.get("query_rewritten"): + notes.append( + "eBay rewrote this search ('Showing results for …'); every result is for ITS query, not " + "yours — check the page before trusting these numbers." + ) + return notes + + +def _with_meta(payload: dict, meta: dict, found: int) -> dict: + payload["headline_count"] = meta.get("headline_count") + payload["related_rows_excluded"] = meta.get("related_rows_excluded", 0) + if meta.get("query_rewritten"): + payload["query_rewritten"] = True + notes = _page_notes(meta, found) + if notes: + payload["notes"] = notes + return payload def _as_bool(value, default: bool) -> bool: @@ -235,7 +285,7 @@ def ebay_price_check( b = _browser() try: url = search_url(query, domain=domain, sold=sold, condition=condition) - listings, dropped = _fetch(b, url, sold=sold) + listings, dropped, meta = _fetch(b, url, sold=sold) except (BrowserError, EbayError) as exc: return json.dumps({"ok": False, "error": str(exc)}) except ValueError as exc: @@ -246,21 +296,25 @@ def ebay_price_check( _record(query, "ebay", listings) stats = summarize(listings) return json.dumps( - { - "ok": True, - "query": query, - "basis": "sold listings — what buyers paid" - if sold - else "active listings — asking prices, NOT sale prices", - "condition": condition, - "url": url, - "stats": stats, - # Surfaced so the model can qualify a thin result instead of treating three - # comps with the same confidence as sixty. - "results_found": len(listings), - "unparseable_rows_skipped": dropped, - "sample": [x.as_dict() for x in listings[:_SAMPLE]], - } + _with_meta( + { + "ok": True, + "query": query, + "basis": "sold listings — what buyers paid" + if sold + else "active listings — asking prices, NOT sale prices", + "condition": condition, + "url": url, + "stats": stats, + # Surfaced so the model can qualify a thin result instead of treating three + # comps with the same confidence as sixty. + "results_found": len(listings), + "unparseable_rows_skipped": dropped, + "sample": [x.as_dict() for x in listings[:_SAMPLE]], + }, + meta, + len(listings), + ) ) @tool @@ -276,19 +330,23 @@ def ebay_search(query: str, sort: str = "best_match", condition: str = "any", so b = _browser() try: url = search_url(query, domain=domain, sold=sold, sort=sort, condition=condition) - listings, dropped = _fetch(b, url, sold=sold) + listings, dropped, meta = _fetch(b, url, sold=sold) except (BrowserError, EbayError, ValueError) as exc: return json.dumps({"ok": False, "error": str(exc)}) return json.dumps( - { - "ok": True, - "query": query, - "sold": sold, - "url": url, - "count": len(listings), - "unparseable_rows_skipped": dropped, - "listings": [x.as_dict() for x in listings[:max_results]], - } + _with_meta( + { + "ok": True, + "query": query, + "sold": sold, + "url": url, + "count": len(listings), + "unparseable_rows_skipped": dropped, + "listings": [x.as_dict() for x in listings[:max_results]], + }, + meta, + len(listings), + ) ) @tool @@ -461,7 +519,7 @@ def ebay_price_and_profit( b = _browser() try: url = search_url(query, domain=domain, sold=True, condition=condition) - listings, dropped = _fetch(b, url, sold=True) + listings, dropped, meta = _fetch(b, url, sold=True) except (BrowserError, EbayError, ValueError) as exc: return json.dumps({"ok": False, "error": str(exc)}) @@ -469,7 +527,11 @@ def ebay_price_and_profit( stats = summarize(listings) if not stats.get("count"): return json.dumps( - {"ok": True, "query": query, "stats": stats, "note": "no sold comps found — try a broader query"} + _with_meta( + {"ok": True, "query": query, "stats": stats, "note": "no sold comps found — try a broader query"}, + meta, + 0, + ) ) schedule = schedule_from_config(fee_cfg) @@ -487,22 +549,26 @@ def ebay_price_and_profit( } return json.dumps( - { - "ok": True, - "query": query, - "basis": "sold listings — what buyers paid", - "url": url, - "stats": stats, - "results_found": len(listings), - "unparseable_rows_skipped": dropped, - "breakeven_price": breakeven_price(schedule=schedule, costs=costs), - "at_market_prices": scenarios, - "your_costs": costs.total, - "schedule": schedule.as_dict(), - "caveat": "" - if schedule.verified - else "Fee schedule is UNVERIFIED — confirm against a real eBay invoice.", - } + _with_meta( + { + "ok": True, + "query": query, + "basis": "sold listings — what buyers paid", + "url": url, + "stats": stats, + "results_found": len(listings), + "unparseable_rows_skipped": dropped, + "breakeven_price": breakeven_price(schedule=schedule, costs=costs), + "at_market_prices": scenarios, + "your_costs": costs.total, + "schedule": schedule.as_dict(), + "caveat": "" + if schedule.verified + else "Fee schedule is UNVERIFIED — confirm against a real eBay invoice.", + }, + meta, + len(listings), + ) ) amazon_domain = cfg.get("amazon_domain") or "www.amazon.com" @@ -625,19 +691,25 @@ def compare_prices(query: str, condition: str = "any") -> str: def _run(key, basis, fetch): try: - listings, dropped = fetch() + fetched = fetch() + listings, dropped = fetched[0], fetched[1] + meta = fetched[2] if len(fetched) > 2 else {} # the Amazon reader has no padding meta # Record here too. Only the single-source checks logged at first, so the # comparison — the tool most likely to be run repeatedly on a watched item — # built no history at all. _record(query, "amazon" if key.startswith("amazon") else "ebay", listings) - out["sources"][key] = { - "ok": True, - "basis": basis, - "stats": summarize(listings), - "results_found": len(listings), - "unparseable_rows_skipped": dropped, - "sample": [x.as_dict() for x in listings[:3]], - } + out["sources"][key] = _with_meta( + { + "ok": True, + "basis": basis, + "stats": summarize(listings), + "results_found": len(listings), + "unparseable_rows_skipped": dropped, + "sample": [x.as_dict() for x in listings[:3]], + }, + meta, + len(listings), + ) except (BrowserError, EbayError, ValueError) as exc: # Named, not omitted: a missing source that looks like an absent one turns a # half-answer into a confident whole one.