diff --git a/.dockerignore b/.dockerignore index bd37e7c..878a8dc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,5 @@ tests *.pdf .pytest_cache + +.devpost-profile diff --git a/.gitignore b/.gitignore index 24527f8..39d1aaf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ __pycache__ *.log *.db *.db-journal -*.csv \ No newline at end of file +*.csv +.devpost-profile diff --git a/README.md b/README.md index ac0cf3c..5800613 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,16 @@ Errors are JSON: `{"detail":{"code":"..."}}`. Rate limits add `retry_after_ms` a 1. Upload the Devpost projects export with `POST /datasets` as multipart form data, file field `entities_csv`. The CSV must be UTF-8 with unique header names, and must have the columns `Project Title`, `Submission Url`, `M Hacks Main Track`, and `Sponsor Opt In Prizes`; a missing one is `422` `INVALID_COLUMNS` with `detail.names`. Rows with an empty `Submission Url` (drafts) are dropped, and at least two must remain. Columns with a blank header, which spreadsheet apps add when they re-save the export, are dropped. Devpost headers only the first team member, so cells past the last header are dropped and short rows are padded with empty strings. Row ids are the zero-based index among the kept rows. Cell values are returned later as strings. - Before storing anything, the upload follows each `Submission Url` to the public page it redirects to and stores it on that row as a `Project Url` column, which is appended to the headers. Every hop must stay on `https` `devpost.com`. If any row does not resolve, nothing is stored and the upload is `422` `DEVPOST_UNRESOLVED` with `detail.failures`, one `{ "title", "submission_url", "code" }` per failed row. `code` is `INVALID_DEVPOST_URL`, `DEVPOST_LOGIN_REQUIRED`, `DEVPOST_NOT_FOUND`, `DEVPOST_REDIRECTED_OFFSITE`, `DEVPOST_TOO_MANY_REDIRECTS`, or `DEVPOST_UNAVAILABLE`. While the hackathon's submissions are private, Devpost sends anonymous requests to its login page (`DEVPOST_LOGIN_REQUIRED`); set `MDREDD_DEVPOST_COOKIE` to the `Cookie` header of an organizer's logged-in Devpost session to resolve them anyway. Resolution makes one request per row, `MDREDD_DEVPOST_CONCURRENCY` at a time, so a large upload can take a while. + Before storing anything, the upload follows each `Submission Url` to the public page it redirects to and stores it on that row as a `Project Url` column, which is appended to the headers. Every hop must stay on `https` `devpost.com`. If any row does not resolve, nothing is stored and the upload is `422` `DEVPOST_UNRESOLVED` with `detail.failures`, one `{ "title", "submission_url", "code" }` per failed row. `code` is `INVALID_DEVPOST_URL`, `DEVPOST_LOGIN_REQUIRED`, `DEVPOST_NOT_FOUND`, `DEVPOST_REDIRECTED_OFFSITE`, `DEVPOST_TOO_MANY_REDIRECTS`, or `DEVPOST_UNAVAILABLE`. While the hackathon's submissions are private, Devpost sends anonymous requests to its login page (`DEVPOST_LOGIN_REQUIRED`); set `MDREDD_DEVPOST_COOKIE` to the `Cookie` header of an organizer's logged-in Devpost session to resolve them anyway. Resolution makes one request per row, `MDREDD_DEVPOST_CONCURRENCY` at a time, so a large upload can take a while. To skip those requests, upload the export with a `Project Url` column already filled in: rows with a value keep it (it must be an `https` `devpost.com` link, or the row fails as `INVALID_DEVPOST_URL`), and only blank rows are resolved. + + To fill that column in locally, run the preprocessor on the export. It opens each `Submission Url` in a real Chromium (via Playwright), needs no `MDREDD_API_TOKEN`, keeps only submitted rows, and writes the same columns plus `Project Url`: + + ```bash + uv run --group preprocess playwright install chromium # once + uv run --group preprocess python -m app.preprocess projects-export.csv -o resolved.csv --login + ``` + + `--login` opens a browser window to sign in to Devpost once; the session is kept in `--profile` (default `.devpost-profile`) for later runs. `--cookie` (default `$MDREDD_DEVPOST_COOKIE`) is an alternative to signing in. `--headed` shows the browser while resolving, e.g. to pass a Cloudflare check, and `--concurrency` caps how many pages are open at once (default: every row). Rows that do not resolve are printed with their `code` (the codes above, plus `DEVPOST_NOT_PROJECT_PAGE` when the link lands on a Devpost page that is not a project), left blank, and the exit status is `1`. Run it again on `resolved.csv` (as both input and output) to retry only those rows. Upload `resolved.csv` once it exits `0`. 2. A successful upload is `201` and turns judging on: ```json diff --git a/app/devpost.py b/app/devpost.py index d206596..46d6263 100644 --- a/app/devpost.py +++ b/app/devpost.py @@ -3,8 +3,6 @@ import httpx -from app.settings import settings - MAX_REDIRECTS = 10 TIMEOUT_SECONDS = 10 # Devpost sits behind Cloudflare, which rejects bare or library user agents. @@ -31,10 +29,11 @@ def is_devpost_url(url: str) -> bool: ) -def _is_login_page(url: str) -> bool: +def is_login_page(url: str) -> bool: + # Devpost sends signed-out visitors to /users/login or /users/register. parts = urlsplit(url) return parts.hostname == "secure.devpost.com" and parts.path.startswith( - "/users/login" + ("/users/login", "/users/register") ) @@ -49,7 +48,7 @@ def resolve(client: httpx.Client, url: str) -> str: current = url try: for _ in range(MAX_REDIRECTS + 1): - if _is_login_page(current): + if is_login_page(current): raise DevpostError("DEVPOST_LOGIN_REQUIRED") with client.stream("GET", current) as response: if response.is_redirect: @@ -67,11 +66,16 @@ def resolve(client: httpx.Client, url: str) -> str: raise DevpostError("DEVPOST_TOO_MANY_REDIRECTS") -def resolve_all(urls: list[str]) -> list[str | DevpostError]: - """Resolve each link, in order. A failed link is its DevpostError.""" +def resolve_all( + urls: list[str], cookie: str, concurrency: int +) -> list[str | DevpostError]: + """Resolve each link, in order. A failed link is its DevpostError. + + `cookie` is sent to Devpost when set, so private submissions resolve. + """ headers = {"User-Agent": USER_AGENT} - if settings.DEVPOST_COOKIE: - headers["Cookie"] = settings.DEVPOST_COOKIE + if cookie: + headers["Cookie"] = cookie with httpx.Client( headers=headers, timeout=TIMEOUT_SECONDS, follow_redirects=False @@ -83,5 +87,5 @@ def attempt(url: str) -> str | DevpostError: except DevpostError as exc: return exc - with ThreadPoolExecutor(max_workers=settings.DEVPOST_CONCURRENCY) as pool: + with ThreadPoolExecutor(max_workers=concurrency) as pool: return list(pool.map(attempt, urls)) diff --git a/app/preprocess.py b/app/preprocess.py new file mode 100644 index 0000000..61c1334 --- /dev/null +++ b/app/preprocess.py @@ -0,0 +1,274 @@ +"""Resolve a Devpost projects export's submission URLs ahead of upload. + + uv run --group preprocess python -m app.preprocess export.csv -o resolved.csv + +Each submission URL is opened in a real Chromium, all rows at once by default, +and the public project page it lands on is written as the Project Url column, +so POST /datasets stores the output without contacting Devpost. While +submissions are private, pass --login once to sign in to Devpost in a browser +window; the session is kept in --profile for later runs. Rows that do not +resolve are listed and left blank; run the script on its own output to retry +only those rows. Needs no MDREDD_API_TOKEN. +""" + +import argparse +import asyncio +import csv +import os +import sys +from pathlib import Path +from urllib.parse import urlsplit + +from playwright.async_api import ( + BrowserContext, + Playwright, + async_playwright, +) +from playwright.async_api import ( + Error as PlaywrightError, +) +from playwright.async_api import ( + TimeoutError as PlaywrightTimeoutError, +) + +from app import devpost, project +from app.entity import Entity +from app.exceptions import JudgingFailure + +LOGIN_URL = "https://secure.devpost.com/users/login" +TIMEOUT_MS = 30_000 +# Time Cloudflare gets to clear its challenge page before a row fails. +CHALLENGE_TIMEOUT_MS = 20_000 +# Only the redirects matter, so skip what the page would render. +SKIPPED_RESOURCES = {"image", "media", "font", "stylesheet"} + + +async def _launch( + playwright: Playwright, profile: Path, headed: bool, cookie: str +) -> BrowserContext: + context = await playwright.chromium.launch_persistent_context( + profile, headless=not headed, user_agent=devpost.USER_AGENT + ) + if cookie: + await context.add_cookies( + [ + {"name": name, "value": value, "domain": ".devpost.com", "path": "/"} + for name, _, value in ( + part.strip().partition("=") for part in cookie.split(";") + ) + if name + ] + ) + return context + + +async def _login(playwright: Playwright, profile: Path, cookie: str) -> None: + context = await _launch(playwright, profile, headed=True, cookie=cookie) + page = context.pages[0] if context.pages else await context.new_page() + await page.goto(LOGIN_URL) + await asyncio.to_thread( + input, "Log in to Devpost in the browser window, then press Enter here. " + ) + await context.close() + + +async def _resolve(context: BrowserContext, url: str) -> str: + """Open a submission URL and return the project page it lands on.""" + if not devpost.is_devpost_url(url): + raise devpost.DevpostError("INVALID_DEVPOST_URL") + page = await context.new_page() + try: + await page.route( + "**/*", + lambda route: ( + route.abort() + if route.request.resource_type in SKIPPED_RESOURCES + else route.continue_() + ), + ) + try: + response = await page.goto( + url, wait_until="domcontentloaded", timeout=TIMEOUT_MS + ) + if response is not None and response.status in (403, 429, 503): + # Cloudflare's challenge reloads the page once it passes. + async with page.expect_navigation( + wait_until="domcontentloaded", timeout=CHALLENGE_TIMEOUT_MS + ) as navigation: + pass + response = await navigation.value + except PlaywrightTimeoutError as exc: + raise devpost.DevpostError("DEVPOST_UNAVAILABLE") from exc + except PlaywrightError as exc: + raise devpost.DevpostError("DEVPOST_UNAVAILABLE") from exc + landed = page.url + finally: + await page.close() + if devpost.is_login_page(landed): + raise devpost.DevpostError("DEVPOST_LOGIN_REQUIRED") + if not devpost.is_devpost_url(landed): + raise devpost.DevpostError("DEVPOST_REDIRECTED_OFFSITE") + if response is not None and response.status == 404: + raise devpost.DevpostError("DEVPOST_NOT_FOUND") + if response is None or not response.ok: + raise devpost.DevpostError("DEVPOST_UNAVAILABLE") + if not urlsplit(landed).path.startswith("/software/"): + raise devpost.DevpostError("DEVPOST_NOT_PROJECT_PAGE") + return landed + + +async def _resolve_all( + entities: list[Entity], + known: list[str], + profile: Path, + headed: bool, + cookie: str, + concurrency: int, +) -> list[str | devpost.DevpostError]: + """Each row's Devpost URL, in order. Rows with a known URL are not opened.""" + invalid = devpost.DevpostError("INVALID_DEVPOST_URL") + results: list[str | devpost.DevpostError] = [ + url if devpost.is_devpost_url(url) else invalid for url in known + ] + pending = [index for index, url in enumerate(known) if not url] + print( + f"{len(entities)} submitted projects, {len(pending)} to resolve", + file=sys.stderr, + ) + if not pending: + return results + + limit = asyncio.Semaphore(concurrency) + done = 0 + async with async_playwright() as playwright: + context = await _launch(playwright, profile, headed, cookie) + + async def attempt(index: int) -> None: + nonlocal done + attributes = entities[index].attributes + async with limit: + try: + results[index] = await _resolve( + context, attributes[project.SUBMISSION_URL].strip() + ) + except devpost.DevpostError as exc: + results[index] = exc + done += 1 + outcome = results[index] + shown = ( + outcome.code if isinstance(outcome, devpost.DevpostError) else outcome + ) + print( + f"[{done}/{len(pending)}] {attributes[project.TITLE]}: {shown}", + file=sys.stderr, + ) + + try: + await asyncio.gather(*(attempt(index) for index in pending)) + finally: + await context.close() + return results + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="python -m app.preprocess", + description="Fill in each row's Project Url so the upload skips Devpost.", + ) + parser.add_argument("export", type=Path, help="Devpost projects export (CSV)") + parser.add_argument( + "-o", "--output", type=Path, required=True, help="where to write the CSV" + ) + parser.add_argument( + "--login", + action="store_true", + help="open a browser window to sign in to Devpost before resolving", + ) + parser.add_argument( + "--profile", + type=Path, + default=Path(".devpost-profile"), + help="browser profile that keeps the Devpost session (default: .devpost-profile)", + ) + parser.add_argument( + "--headed", + action="store_true", + help="show the browser while resolving, e.g. to pass a Cloudflare check", + ) + parser.add_argument( + "--cookie", + default=os.environ.get("MDREDD_DEVPOST_COOKIE", ""), + help=( + "Cookie header of a logged-in Devpost session, an alternative to " + "--login (default: $MDREDD_DEVPOST_COOKIE)" + ), + ) + parser.add_argument( + "--concurrency", + type=int, + default=0, + help="pages open at once (default: every row at once)", + ) + args = parser.parse_args() + if args.concurrency < 0: + parser.error("--concurrency must not be negative") + + try: + headers, entities = Entity.list_from_csv(args.export.read_bytes()) + entities = project.submitted(headers, entities) + except JudgingFailure as exc: + names = getattr(exc, "names", []) + print(f"{exc.code}: {', '.join(names)}" if names else exc.code, file=sys.stderr) + return 2 + + known = [ + entity.attributes.get(project.PROJECT_URL, "").strip() for entity in entities + ] + headers = [name for name in headers if name != project.PROJECT_URL] + + async def run() -> list[str | devpost.DevpostError]: + if args.login: + async with async_playwright() as playwright: + await _login(playwright, args.profile, args.cookie) + return await _resolve_all( + entities, + known, + args.profile, + args.headed, + args.cookie, + args.concurrency or len(entities), + ) + + results = asyncio.run(run()) + + failures = [ + project.failure(entity, result) + for entity, result in zip(entities, results, strict=True) + if isinstance(result, devpost.DevpostError) + ] + with args.output.open("w", newline="", encoding="utf-8") as file: + writer = csv.writer(file) + writer.writerow([*headers, project.PROJECT_URL]) + for entity, result in zip(entities, results, strict=True): + url = "" if isinstance(result, devpost.DevpostError) else result + writer.writerow([*(entity.attributes[name] for name in headers), url]) + + for row in failures: + print( + f"{row['code']}: {row['title']} ({row['submission_url']})", + file=sys.stderr, + ) + if failures: + print( + f"{len(failures)} of {len(entities)} left blank in {args.output}. " + "Run again on that file to retry them; add --login if " + "DEVPOST_LOGIN_REQUIRED, or --headed if DEVPOST_UNAVAILABLE.", + file=sys.stderr, + ) + return 1 + print(f"Wrote {args.output}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app/project.py b/app/project.py index 430e7dc..211c7a3 100644 --- a/app/project.py +++ b/app/project.py @@ -22,29 +22,52 @@ def submitted(headers: list[str], entities: list[Entity]) -> list[Entity]: """Check the export's columns and drop drafts, which have no submission URL.""" missing = [name for name in REQUIRED if name not in headers] - if PROJECT_URL in headers: - missing.append(PROJECT_URL) if missing: raise InvalidColumnsException(missing) - kept = [ - entity for entity in entities if entity.attributes[SUBMISSION_URL].strip() - ] + kept = [entity for entity in entities if entity.attributes[SUBMISSION_URL].strip()] if len(kept) < 2: raise TooFewEntitiesException() return kept -def with_project_urls(entities: list[Entity]) -> list[Entity]: - """Add each project's resolved Devpost URL, or fail listing every row that did not resolve.""" - results = devpost.resolve_all( - [entity.attributes[SUBMISSION_URL].strip() for entity in entities] +def project_urls( + entities: list[Entity], known: list[str], cookie: str, concurrency: int +) -> list[str | devpost.DevpostError]: + """Each row's Devpost URL, in order. A row that did not resolve is its DevpostError. + + `known` is each row's Project Url from the upload, looked up ahead of time. + Only rows where it is blank are resolved here. + """ + pending = [index for index, url in enumerate(known) if not url] + resolved = devpost.resolve_all( + [entities[index].attributes[SUBMISSION_URL].strip() for index in pending], + cookie, + concurrency, ) + invalid = devpost.DevpostError("INVALID_DEVPOST_URL") + results: list[str | devpost.DevpostError] = [ + url if devpost.is_devpost_url(url) else invalid for url in known + ] + for index, result in zip(pending, resolved, strict=True): + results[index] = result + return results + + +def failure(entity: Entity, error: devpost.DevpostError) -> dict[str, str]: + return { + "title": entity.attributes[TITLE], + "submission_url": entity.attributes[SUBMISSION_URL], + "code": error.code, + } + + +def with_project_urls( + entities: list[Entity], known: list[str], cookie: str, concurrency: int +) -> list[Entity]: + """Add each project's Devpost URL, or fail listing every row that did not resolve.""" + results = project_urls(entities, known, cookie, concurrency) failures = [ - { - "title": entity.attributes[TITLE], - "submission_url": entity.attributes[SUBMISSION_URL], - "code": result.code, - } + failure(entity, result) for entity, result in zip(entities, results, strict=True) if isinstance(result, devpost.DevpostError) ] diff --git a/app/session.py b/app/session.py index 93062d7..5f929ef 100644 --- a/app/session.py +++ b/app/session.py @@ -35,12 +35,22 @@ def close(self) -> None: def start(self, entity_csv: bytes) -> list[str]: headers, entities = Entity.list_from_csv(entity_csv) entities = project.submitted(headers, entities) + # A Project Url column, filled in ahead of time, saves those rows a + # Devpost request. It is moved to the end like a resolved one. + known = [ + entity.attributes.get(project.PROJECT_URL, "").strip() + for entity in entities + ] + headers = [name for name in headers if name != project.PROJECT_URL] + entities = [project.without_project_url(entity) for entity in entities] # Answer a repeat or a conflict before spending a Devpost request per row. current = self.worker.check_replaceable(entities, headers) if current is not None: return current # Resolve here, off the worker thread, which would time out on Devpost. - entities = project.with_project_urls(entities) + entities = project.with_project_urls( + entities, known, settings.DEVPOST_COOKIE, settings.DEVPOST_CONCURRENCY + ) return self.worker.replace_entities(entities, [*headers, project.PROJECT_URL]) def _watch(self) -> None: diff --git a/pyproject.toml b/pyproject.toml index 36312ae..db762fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,3 +24,6 @@ typeCheckingMode = "standard" dev = [ "pandas-stubs>=3.0.5.260914", ] +preprocess = [ + "playwright>=1.63.0", +] diff --git a/uv.lock b/uv.lock index 3d9a522..51144f2 100644 --- a/uv.lock +++ b/uv.lock @@ -194,6 +194,53 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/85/11/0aa8455af26f0ae89e42be67f3a874255ee5d7f0f026fc86e8d56f76b428/fastar-0.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:e59673307b6a08210987059a2bdea2614fe26e3335d0e5d1a3d95f49a05b1418", size = 460467, upload-time = "2025-11-26T02:36:07.978Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/6e/0091f175ccd02b02bc8811bbcbcc6ac2e980be116e3b2f7a736ca322bf84/greenlet-3.5.6.tar.gz", hash = "sha256:8e67c43bdfc88d5fee6db0d3e40175b362fc95fb85f0412d233b9b203c53a575", size = 207653, upload-time = "2026-09-14T15:42:51.806Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/c0/d254544ae2b8bdd311aef000fafc02828c2771b17d994b3075620ea7cc6e/greenlet-3.5.6-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8cddea1b8339451c2fb3388e138347b6126744f33b611bdb55b7357361cfef46", size = 295221, upload-time = "2026-09-14T14:25:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/18/18/eb54be16b9cc3971e09ca5b73334e1b8c804a4630d9addaaf218a4fe300f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c59acfa8eb73a1e0d484392dc002bdf001fd4ce73394e0132df3d1ab6093d7cb", size = 660992, upload-time = "2026-09-14T15:12:04.876Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b4/e193efe65671dcf294bc51fcc59efb52d154adf8612c4ea016da0d2c486c/greenlet-3.5.6-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3b4a01c6da07ef9f80d4fe8933b994bc99747bcea3eab0330a9c34d3c12655b", size = 673428, upload-time = "2026-09-14T15:20:45.756Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/631bb45fafde1dca782152377c0676d182ec924820064047f533a3627b28/greenlet-3.5.6-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd0b83bed3405b586a3133629f1d1a5bc7bfd64822a3b7ab342bdc68e6dbc61b", size = 677688, upload-time = "2026-09-14T15:25:09.279Z" }, + { url = "https://files.pythonhosted.org/packages/45/ac/28fa7a9e50f2859466214c4ac584d776db52c1604ad4dd158960a5af2a1f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a09d59bef1db94f384b5bcc2d523694d338f3df6b757aeeaf7baca5d0c0be88", size = 670773, upload-time = "2026-09-14T14:36:02.577Z" }, + { url = "https://files.pythonhosted.org/packages/40/30/2b0a73e68e1e18e30b601d0d183cfdfc2beca4de5a6843c630f0fc9fb90c/greenlet-3.5.6-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:fdacf26402389bdd89857ad3c045a26fe8f3314f9a8b28226f82f88463a65b77", size = 480475, upload-time = "2026-09-14T15:28:40.741Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cd/fb7d6cdd86ff3427c1494854f0e35437eba05142be91f530f6da75e09e19/greenlet-3.5.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8b7c73d1cef3d9ae963e9ff03f6222df43efbb9054ffd2f1969c935b7fc84c02", size = 1631900, upload-time = "2026-09-14T15:10:09.745Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/143bdbb20a516628cb15074ae52ed17d850b450292609c7a6fccac6dbece/greenlet-3.5.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8b27df301f56e3b3d2298095c8f7d6b68f2521f6b1693e901fa039bdbae34424", size = 1693740, upload-time = "2026-09-14T14:35:52.959Z" }, + { url = "https://files.pythonhosted.org/packages/c9/9e/019642432e6ae283301df1361227d47610709d2dc69a38f95edef266d713/greenlet-3.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:f8f0bd690e1a41294ac87905e8121c81a3761ec2583c768f13467428606c8c7a", size = 327473, upload-time = "2026-09-14T14:28:12.948Z" }, + { url = "https://files.pythonhosted.org/packages/e9/7f/8aafc7bf70c948786dba7221d0dc0838e5329bebc6d434ef2208b4f0e760/greenlet-3.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:8cda13494d86a4f12429641117cb6ac4bbbc9c30a33f711f7d3a2e5fbe4b0b7e", size = 311095, upload-time = "2026-09-14T14:28:00.7Z" }, + { url = "https://files.pythonhosted.org/packages/14/7e/7a205688a5b3074933b18a906608d46d106e9a79d776bdab5a4abf4b4feb/greenlet-3.5.6-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:97c5a53e8c1754df58e73f047a99e287d4da1bdfe64b0072fb25c87000897951", size = 305352, upload-time = "2026-09-14T14:21:31.962Z" }, + { url = "https://files.pythonhosted.org/packages/78/cb/9c4a57a9d9dd0256e20b8f7f4f06554c2c92badebf0ab73ce344321b78b9/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fea4427d1ffdb3b523d7daa6712038428a4c16c450b9777bdd1221cfee0eab49", size = 672671, upload-time = "2026-09-14T15:12:06.347Z" }, + { url = "https://files.pythonhosted.org/packages/97/52/c6729681ebbd298f4decd28746815acc8a0b0a0fde21d2df33776fd4d042/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73a29b5ba642e35433166a03a3e02935e7238c4b3467fbd77523b99edea23e5b", size = 679489, upload-time = "2026-09-14T15:20:47.291Z" }, + { url = "https://files.pythonhosted.org/packages/71/76/3c11c21e0716b1f1dc7c1a4b3d690abb1d3b448c69a9d32049fecb64010a/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:61a61b4a95a4f97922c3a6f5606d3e360851584bd47e500a5161373c53810e3d", size = 681303, upload-time = "2026-09-14T15:25:11.088Z" }, + { url = "https://files.pythonhosted.org/packages/58/c5/2b6c721ba8b8963da42d5a0f57f25b8aaeb1fe9bdd156875e57f3be648a2/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:460e70b033aba8ed47e2ac9b5d0d2157b05a34fbfa30a241400aef4118902cdc", size = 676608, upload-time = "2026-09-14T14:36:03.959Z" }, + { url = "https://files.pythonhosted.org/packages/3f/26/3ae402202452cd5941bbbd483e5a74297e2397e7aa3182c2a5e3ab7d5666/greenlet-3.5.6-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:fe3170a69fe039b18ad18171e66faa9a75f6fe9d78f968fd9b54e09fbd714d81", size = 510112, upload-time = "2026-09-14T15:28:42.112Z" }, + { url = "https://files.pythonhosted.org/packages/b2/04/0d018e0d05bcdde19a0fcb907834155f1fc853a9bedd3f3f5e6acadcae19/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ca80a49b53ed1d22f7282da7255f7bb2fd1935fd0f623d8613fda38745f18961", size = 1641479, upload-time = "2026-09-14T15:10:11.216Z" }, + { url = "https://files.pythonhosted.org/packages/59/bb/f02ef9073919158f6403fe3701d4ed4403d646720e7201dfc6e9d264bac3/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:916f92f2a8db10508f739d0b5e00b83defe5d1115a997c54532a6d7cf8c95404", size = 1698758, upload-time = "2026-09-14T14:35:54.336Z" }, + { url = "https://files.pythonhosted.org/packages/08/a5/1f48fe647473a2dcccfd1839b2ff2c78eb57009be776b4da071e901c9bff/greenlet-3.5.6-cp314-cp314t-win_amd64.whl", hash = "sha256:886bcf1870af74c32bc310fd00a6b803445e17e51b7d5a107c7b35c0f362cc16", size = 331574, upload-time = "2026-09-14T14:27:18.451Z" }, + { url = "https://files.pythonhosted.org/packages/cd/72/3882855a75838faeb54a58aeef4fd77d20b2a86d4bad570c70d41b565dcf/greenlet-3.5.6-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3ac3494c381dab876cad7d0b22f3a722f3e0c8deb3a65b9e7f35ad7f58b8fcb3", size = 295926, upload-time = "2026-09-14T14:27:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/10/1f/be4d957d8a9b90bcbe8db206548a42134d96222d43e5ed3fc4708fb6e24b/greenlet-3.5.6-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:602024dae6d77e161f4b89491b62ca1d4f19949d79d47b2db057e476d21179d6", size = 666910, upload-time = "2026-09-14T15:12:07.901Z" }, + { url = "https://files.pythonhosted.org/packages/a1/af/60d62571a7d6de961e4ce7625d6c2faf359345659fc782d2cdf517c34577/greenlet-3.5.6-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8e63209c3e1e828ee6a457529b4a6d8b05d050fe0ae03a7ae49e967c5d312e0", size = 677415, upload-time = "2026-09-14T15:20:48.817Z" }, + { url = "https://files.pythonhosted.org/packages/f5/41/b3114c97c10e796010f00a30f51c81470072bca4b53e396ccca87484fcf7/greenlet-3.5.6-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9133d68624b1f2e89ec2f554d56aea8a5b0d7168cd9320200ba58d4d794845a4", size = 681337, upload-time = "2026-09-14T15:25:12.812Z" }, + { url = "https://files.pythonhosted.org/packages/fb/16/ac9e547b611539aaed1870eb1d6ddc57abdd5924b3a99bb9b5f0b44176b8/greenlet-3.5.6-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccadce0130fd813ec86ebfe969a6c58b42acc1d0fe55a47525375b740e07b605", size = 675927, upload-time = "2026-09-14T14:36:05.34Z" }, + { url = "https://files.pythonhosted.org/packages/48/1b/d41861c2fa00968e39e467a495ca8db9ce9b6310a5d9b57561b3d0dc48fa/greenlet-3.5.6-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:5adcbbfe78bdc242c71740a02e0991cc1b2f34d33c8bb15ca45eee8fd1140942", size = 487339, upload-time = "2026-09-14T15:28:43.497Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/b7ba08d6431121741f1d30be0d5d292e76873325179a63586cd9217b62f6/greenlet-3.5.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9297fb9c39b9a2c039dbcd306c410bd6906b95244dec3bba4318d36c718c164c", size = 1637236, upload-time = "2026-09-14T15:10:12.442Z" }, + { url = "https://files.pythonhosted.org/packages/af/c5/3b1cbc68f0c082022fc8717f7fe4b8b13b8d583c52352be37f4e9f55bcd2/greenlet-3.5.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b374e79ffa7511afc11773aef40a4ccea6191fba1c856ea2f9c56738dca69d7a", size = 1698526, upload-time = "2026-09-14T14:35:56.039Z" }, + { url = "https://files.pythonhosted.org/packages/de/56/12941ed2711400451c89d544e10f831800a2770f19dd55eac8f0f7f2003b/greenlet-3.5.6-cp315-cp315-win_amd64.whl", hash = "sha256:7969bffa322c097bd46ae595ada6a931cefda613f18ba64587e9cff4cb320756", size = 327739, upload-time = "2026-09-14T14:23:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3b/576b9ed5ac929252e340cf60b4bcb6a8515350dc20797064b1922dc4ea75/greenlet-3.5.6-cp315-cp315-win_arm64.whl", hash = "sha256:8dba0129b93e7091dfefaf4cf7000172741bff7f47bf6326fcf17f32fbb54d6b", size = 311715, upload-time = "2026-09-14T14:28:25.154Z" }, + { url = "https://files.pythonhosted.org/packages/16/c2/86cfc5555a98e12b86966ddbd24fd39af32f71f2f785c6595b7feb2db156/greenlet-3.5.6-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:de3de000d459402cda015068fd135aa50c0bf6f2477a80d4da1e646f123b4e78", size = 306244, upload-time = "2026-09-14T14:27:57.565Z" }, + { url = "https://files.pythonhosted.org/packages/14/6d/83ffc9d05a75a80ab3a7595dbb1d9604e5d4fc2996d73a8ae2dbd1284900/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45663c01a4de48b9a64a2ee1509d92d1dfd3afb02b2ccfc9333029d11aef996a", size = 676578, upload-time = "2026-09-14T15:12:09.468Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d6/c2cf684810e5caded075970aaadea654ecb58b8382b9aecf1d231b936894/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3deccbb57a481e3a408fe61cdfd5c13e0678fc0a30fdd09597917ca87b4be877", size = 684203, upload-time = "2026-09-14T15:20:50.261Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d1/039c353d5593a97a89699e989324c9bc86af499e6c6152fe0180f5742204/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:63aff70fe5aac59c72215f42ec39fcb59ff46774fa966e717f8ecb6ee2273577", size = 686023, upload-time = "2026-09-14T15:25:14.528Z" }, + { url = "https://files.pythonhosted.org/packages/62/19/00e1bee5d2af890dc8f400b54d0b0f9b489965f92bc12b407ff72cc6f469/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:311018b46472fb26ee85870847fb89eb64cc8aaddb617400789d87076f7cfeec", size = 681253, upload-time = "2026-09-14T14:36:06.742Z" }, + { url = "https://files.pythonhosted.org/packages/8a/62/97ceb8e0b2ea96046cdf8e95b042715020ebb12d83ea0690db80a8f03d23/greenlet-3.5.6-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:520648db8fb92eef7b3e6013f5a6f901cdf0d6685f639c2f7a245879f865bef7", size = 517058, upload-time = "2026-09-14T15:28:44.924Z" }, + { url = "https://files.pythonhosted.org/packages/89/58/c9275fd0ca195d1d3402931bcce8cfcc74726ff76efb1883d229e6e1a3d7/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7f924a5a9d5890649566f2f6682e0d8ad8ca23028bacffbbac36dbd7fd680176", size = 1645988, upload-time = "2026-09-14T15:10:13.758Z" }, + { url = "https://files.pythonhosted.org/packages/e0/36/b35747582fa4f1a5453f8f3002405dbac788e450cec7674dc2d204b6ccb5/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:de9923832f2d8c1a5ecd8d7260465a6ca5a86888a0d129e3bd5cf0406d2fc5bf", size = 1702383, upload-time = "2026-09-14T14:35:58.143Z" }, + { url = "https://files.pythonhosted.org/packages/ed/69/6ec22ac9351e474d2a134d0ff9400dc80362d1c20f0721088ffffdfc205b/greenlet-3.5.6-cp315-cp315t-win_amd64.whl", hash = "sha256:2ab5f42ac6c238eb71770715e6e909ad9a1a92b6c681ccb64cd5a0f07edb953f", size = 331845, upload-time = "2026-09-14T14:27:41.723Z" }, + { url = "https://files.pythonhosted.org/packages/30/cf/697c051fd534e223461fb8b523890e21a24eeca229cd50624cff6f02fabd/greenlet-3.5.6-cp315-cp315t-win_arm64.whl", hash = "sha256:f9fe868463ec7e1363733af77e38a5fda3e9b63940337048c945d69e0c80ff24", size = 315070, upload-time = "2026-09-14T14:22:21.476Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -362,6 +409,9 @@ dependencies = [ dev = [ { name = "pandas-stubs" }, ] +preprocess = [ + { name = "playwright" }, +] [package.metadata] requires-dist = [ @@ -376,6 +426,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "pandas-stubs", specifier = ">=3.0.5.260914" }] +preprocess = [{ name = "playwright", specifier = ">=1.63.0" }] [[package]] name = "mdurl" @@ -495,6 +546,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/90/3ddf6ef69090c63b6fe0d42b41e5dd3323d95dfe01eda6766305337be862/peewee-4.5.1-py3-none-any.whl", hash = "sha256:dbbdc93e9be08d1df49ceed48dc5d609bfeda6112848086f1f6c1f1debe70975", size = 193883, upload-time = "2026-09-08T01:20:09.027Z" }, ] +[[package]] +name = "playwright" +version = "1.63.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/dd/fbb3d34228ad753bc14464d5f2252585b73367539775a88d0a3949cf5a45/playwright-1.63.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:84c540759e8e7f01e690e197e04060f48899d37cea322299f255843273d3385d", size = 44278407, upload-time = "2026-09-15T16:49:06.045Z" }, + { url = "https://files.pythonhosted.org/packages/f5/9a/948b930b1a8c4ee869e5a139a2b7747caa06aab56a3f09a2f0abdcbda221/playwright-1.63.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fd1aa00631d44d55e56e0975bf3f3f285fac4a9fd2813183f0c12a488f1a1b24", size = 42944679, upload-time = "2026-09-15T16:49:10.039Z" }, + { url = "https://files.pythonhosted.org/packages/94/11/dc5c13fa1602371603acd461be47529c1b3513815d3a0dc98f642c291a10/playwright-1.63.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:c89fc4736502a1f0fac2c8ca5d10c0cbc1c669f1f4774a2d8507a43140e4d53f", size = 44278410, upload-time = "2026-09-15T16:49:13.861Z" }, + { url = "https://files.pythonhosted.org/packages/27/9c/103a5037789062bdab27c7dca53f3ca6b075b572ab2cd96eec825b3aec4e/playwright-1.63.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:ad21bc07516b187965a7521c5cf0df0bd657b17482eaad74335272d35a2b07de", size = 48217159, upload-time = "2026-09-15T16:49:17.404Z" }, + { url = "https://files.pythonhosted.org/packages/f3/82/3d85505284c5a210f2da6c07b8f757524e79d1fba9cfdafe1eafb766ae59/playwright-1.63.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:354e15b29503565fc598b89f16fbe070459343bef9d7498a93e304864000c6a7", size = 47902708, upload-time = "2026-09-15T16:49:21.055Z" }, + { url = "https://files.pythonhosted.org/packages/69/8d/f74ff6b52751859f4b69caf66aa7d4a3c8d6c1f0c7dc9920da103612ee39/playwright-1.63.0-py3-none-win32.whl", hash = "sha256:660c00c62639e31b16700ba5456b351ddba55bb766b7ce8261223aa34928e482", size = 38606075, upload-time = "2026-09-15T16:49:29.546Z" }, + { url = "https://files.pythonhosted.org/packages/76/eb/d6b8d92658038e260dbc7dd69fb3fdbe245aac283953de8b50b02bfe5f61/playwright-1.63.0-py3-none-win_amd64.whl", hash = "sha256:2f9a707a6c6c91157ed77bff2b8caeb04b3c8d46e70d585fc134298cbe4b5cc6", size = 38606082, upload-time = "2026-09-15T16:49:32.838Z" }, + { url = "https://files.pythonhosted.org/packages/be/75/2432b3e3c7c62103b72d4c4cc8b16a56383ada372bbb0d1278591e988f71/playwright-1.63.0-py3-none-win_arm64.whl", hash = "sha256:1e4a3a838ce22fb68ad17193fcd142a19610d9d70fb9966d2239f5dddc0cc05b", size = 34523554, upload-time = "2026-09-15T16:49:36.455Z" }, +] + [[package]] name = "pydantic" version = "2.12.5" @@ -581,6 +651,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/1a/dd1b9d7e627486cf8e7523d09b70010e05a4bc41414f4ae6ce184cf0afb6/pydantic_settings-2.13.0-py3-none-any.whl", hash = "sha256:d67b576fff39cd086b595441bf9c75d4193ca9c0ed643b90360694d0f1240246", size = 58429, upload-time = "2026-02-15T12:11:22.133Z" }, ] +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + [[package]] name = "pygments" version = "2.19.2"