diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2db62db --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + # Called by release.yml as its gate, so the bar for tagging is exactly the bar for + # merging rather than a second copy that drifts. + workflow_call: + +permissions: + contents: read + +# Superseded runs on the same ref are pointless once a newer commit exists. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - run: go build ./... + + - name: gofmt + run: | + unformatted=$(gofmt -l .) + if [ -n "$unformatted" ]; then + echo "unformatted files:" + echo "$unformatted" + exit 1 + fi + + - run: go vet ./... + + # The syncer fans out downloads under a semaphore and persists the lockfile from + # each goroutine, so the suite is worth running under the detector. No network and + # no session are needed: everything runs against httptest servers and the + # committed, scrubbed fixtures. + - run: go test -race ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..eb3c7c9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,74 @@ +name: Release + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + +jobs: + check-branch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Verify tag is on main + run: | + git fetch origin main + git merge-base --is-ancestor ${{ github.sha }} origin/main || { + echo "Error: tag must point to a commit on main" + exit 1 + } + + test: + needs: check-branch + uses: ./.github/workflows/ci.yml + + release: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Get version from tag + id: version + run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + + - name: Build release artifacts + env: + VERSION: ${{ steps.version.outputs.VERSION }} + run: | + mkdir -p dist + + platforms=( + "darwin/amd64/mac-intel" + "darwin/arm64/mac-apple" + "linux/amd64/linux-intel" + "linux/arm64/linux-arm64" + "windows/amd64/win" + ) + + for p in "${platforms[@]}"; do + IFS='/' read -r goos goarch label <<< "$p" + bin="unity-sync" + if [ "$goos" = "windows" ]; then bin="unity-sync.exe"; fi + + CGO_ENABLED=0 GOOS=$goos GOARCH=$goarch go build \ + -ldflags "-X main.version=${VERSION}" \ + -o "dist/$bin" . + (cd dist && zip "unity-sync-${VERSION}-${label}.zip" "$bin" && rm "$bin") + done + + - name: Create GitHub release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + files: dist/*.zip diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b0ffafe --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,82 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +`unity-sync` is a Go CLI that mirrors the assets owned on the Unity Asset Store into a +local library, downloading only what changed since the last run. See `README.md` +(user-facing) and `docs/design.md` (the authoritative design doc: measured store +behaviour, identity rules, the failure model). Read `docs/design.md` before changing +enumeration, the lockfile, the cache layout, or the download guards. + +Browsing the mirrored library is a separate tool, +[quarry](https://github.com/curbol/quarry). This repo acquires files; quarry reads them. + +## Build & test + +```bash +go build -o unity-sync . # requires Go 1.26+, no cgo +go test ./... # full suite, fully offline +go test -race ./... # what CI runs +go test ./internal/syncer/ -run TestClassify -v +go vet ./... +gofmt -l . +``` + +No Makefile or task runner; use the `go` toolchain directly. The suite needs no network +and no session: everything runs against `httptest` servers and the committed, scrubbed +fixtures in `testdata/store/`. + +## Architecture + +`main.go` `run()` parses flags and dispatches `select`, `status`, `sync`, `list`, `update` +and `version`, returning an exit code alongside its error. Layered `internal/` packages, +each with a package doc comment stating its contract: + +- `model` — domain types and the identity rules. Carries `id` (the store product id) and + deliberately not `productId`, which is a different value no endpoint accepts. +- `config` — user settings by precedence: defaults → `config.toml` → env → flags. There is + no browser session default, because a browser session cannot work here. +- `session` — builds the Cookie header from a pasted curl file or a `cookies.txt`, and + asserts the `LS` cookie is present before any request. +- `retry` — backoff policy. `retry.Permanent` lets a caller stop on a body-based verdict + that the status code alone would have retried. +- `unitypackage` — reads the store descriptor from a package's gzip FEXTRA field. +- `store` — the Asset Store client and the response-level download guards. +- `cache` — the local mirror. Two-phase writes (`Store` → `Commit`/`Discard`), adopt by + scan, relocate on rename, temp sweep, root confinement. +- `lockfile` — `unity-sync.lock.json`, advertised fields kept apart from resolution fields. +- `manifest` — `unity-sync.toml`, the committed allowlist keyed by asset id. +- `syncer` — orchestration, the pure `classify`, and the semantic download guards. +- `web` — the `select` page. +- `selfupdate` — the `update` subcommand. +- `fixtures` + `cmd/scrubfixtures` — regenerate PII-free `testdata/` from raw captures. + +### Key invariants (don't break these) + +- **`LS` is the credential.** Not the NextAuth session token, which neither endpoint + consults. Its absence is reported before any request, because the store answers a + missing `LS` with an opaque 500. +- **No store client follows a redirect.** An unauthenticated download 302s to Unity's + OAuth page. `selfupdate` is the deliberate exception: it talks to GitHub, whose asset + API 302s to a signed CDN URL by design. +- **Downloads ask for `Accept-Encoding: identity`.** The endpoint honours gzip by + gzipping the already-gzipped package, and Go will not decode an encoding the caller + requested. +- **`resolvedVersionId` is the diff key**, not the advertised `version.id`. The advertised + value refreshes every run; pairing a refreshed id with an unresolved entry's file would + mark it current forever. +- **Nothing unverified reaches a real cache path.** `cache.Store` does not rename; + `Commit` does, after the syncer's guards pass. +- **A failed download fails its asset, not the run**, and a pulled asset does not make the + run exit non-zero. +- **Only `select` writes the manifest.** `status` and `sync` read it. +- **No account data in the repo.** Sessions and raw captures stay out; the + `internal/fixtures` guard test fails the build if any reaches `testdata/`. + +## Editing testdata + +Don't hand-edit `testdata/store/*.json`. They are generated by +`go run ./cmd/scrubfixtures` from git-excluded raw captures. Regenerate rather than patch, +and keep the guard test green. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6f199f7 --- /dev/null +++ b/README.md @@ -0,0 +1,167 @@ +# unity-sync + +A Go CLI that mirrors the assets you own on the Unity Asset Store into a local library, +downloading only what changed since the last run. It is the download manager the Asset +Store does not give you outside the Editor. Design: `docs/design.md`. + +## Install + +Grab the latest release into `~/.local/bin` (private repo, so it uses your `gh` login or +`GITHUB_TOKEN`): + +```bash +gh api repos/curbol/unity-sync/contents/install.sh --jq .content | base64 -d | bash +``` + +Then update in place: + +```bash +unity-sync update # latest release +unity-sync update 0.2.0 # a specific version +unity-sync version # what is installed +``` + +Releases are cut by pushing a `v*` tag; a workflow builds the cross-platform binaries and +publishes them. + +## Build from source + +```bash +go build -o unity-sync . +``` + +Requires Go 1.26+. No cgo. To stamp a version into a local build: +`go build -ldflags "-X main.version=0.2.0" -o unity-sync .` + +## One-time setup + +Two kinds of state, kept apart: + +- **User config** (session, machine defaults) lives *outside* any project, resolved as + `--config ` › `$UNITY_SYNC_CONFIG_DIR` › `$XDG_CONFIG_HOME/unity-sync` › + `~/.config/unity-sync`. + + ```bash + mkdir -p ~/.config/unity-sync + cp config.example.toml ~/.config/unity-sync/config.toml + ``` + +- **Project manifest** (`unity-sync.toml`: which assets this project draws from) lives + *in* the project that consumes the assets, committed to its repo. unity-sync finds it by + walking up from the working directory, or point at it with `--manifest`. Its lockfile + (`unity-sync.lock.json`) is written beside it. It carries no account identity. See + `unity-sync.example.toml`. + +Packages are cached at `$XDG_DATA_HOME/unity-sync` (`~/.local/share/unity-sync`) by +default; override with `library_path`, `UNITY_SYNC_LIBRARY`, or `--library`. + +## Session + +The store gates everything behind your signed-in session, and the cookie it actually +checks — `LS` — is a session cookie that lives only in your browser's memory. No browser +cookie database has it, so unity-sync cannot read your session automatically the way a +tool for some other store might. You paste one instead: + +In DevTools → Network, right-click any `assetstore.unity.com` request → Copy → Copy as +cURL, and save it: + +```bash +$EDITOR ~/.config/unity-sync/session.curl # paste, save +unity-sync status +``` + +A Netscape `cookies.txt` export works too, as long as your exporter keeps HttpOnly rows +(they are written with a `#HttpOnly_` prefix). Point at either file with `--session`, with +`session_source` in `config.toml`, or just save it as `session.curl` or `cookies.txt` in +the config dir, where unity-sync looks by default. + +If the file is missing the `LS` cookie, unity-sync says so before making any request, +because the store's own answer in that case is an HTTP 500 that reads like a server fault. + +A pasted session expires. When it does, re-copy it. + +## Commands + +```bash +unity-sync select # pick which assets to mirror (opens a local page) +unity-sync status # what a sync would change; downloads nothing, changes nothing +unity-sync sync # download the delta and update the lockfile +unity-sync list # print the current lockfile +``` + +Useful flags: `--manifest `, `--only `, `--library `, +`--concurrency `, `--verify`, `--dry-run` (makes `sync` behave like `status`), +`--config `, `--session `, `--addr ` (the `select` page's address). + +## Selecting assets + +Selection is opt-in: an asset is mirrored only once you enable it. This matters more here +than it might sound — a typical Asset Store account owns hundreds of packages and tens of +gigabytes, and individual packages reach 23 GB. + +`unity-sync select` lists every owned asset with its thumbnail and a checkbox and writes +the `[[asset]]` entries into `unity-sync.toml`: + +```toml +[[asset]] + id = "115488" + name = "Quick Outline" + enabled = true +``` + +Entries key on the asset id, so a publisher renaming their asset cannot silently deselect +it. Newly-bought assets appear disabled on the next `select`, so buying something never +downloads it behind your back. Hand-editing the file is fine. + +`select` is the only command that writes the manifest. `status` and `sync` only read it. + +## What it does + +- Lists every owned asset with its current version inline, so a run that changes nothing + costs only the enumeration: one bootstrap, a page request per 100 owned assets plus the + empty page that ends the walk, and no package bytes at all. +- Downloads only what is new, changed, or missing from the cache, into + `///.unitypackage`. +- Records everything in `unity-sync.lock.json` beside the manifest: what is owned, at what + version, what is mirrored, and its checksum. Commit it for a changelog. +- Reports assets your manifest lists that the account does not own, and assets the store + has delisted. + +## Verifying the cache + +Every run checks cached files cheaply: the file exists, its size is exactly what was +recorded, and — for packages that carry a version stamp — that stamp still matches. That +catches a truncated or replaced file without reading tens of gigabytes. A package with no +stamp is checked on size alone, so it does not re-download on every run. + +`--verify` re-hashes instead, which is the only way to catch corruption in the middle of a +file. It is opt-in for the obvious reason. + +## Cache + +The library is local and expendable: current versions are re-downloadable, and `sync` +re-fetches anything missing or failing its check. Deleting the cache and re-syncing +rebuilds it. Durability of the assets you actually ship belongs in the consuming project, +not here. + +When an asset leaves your account, unity-sync does not delete anything: its entry drops +out of the lockfile and the run tells you which file is now unreferenced, so you can +decide. A run removes a file only when it is replacing that same asset's own copy: the +superseded build after its path changed, whether the new copy was downloaded or adopted, +and a copy that failed its check when a good copy of the same package is adopted over it. + +## Browsing what you have + +Searching and previewing the mirrored library lives in a separate tool, +[quarry](https://github.com/curbol/quarry): it indexes an asset tree, reads inside +archives, and previews models. The cache layout here is three levels deep +(`publisher/asset/file`) precisely so quarry's vendor and pack filters work against it. +Point its `root` at your library path. + +## Notes + +- Test fixtures are generated: `cmd/scrubfixtures` regenerates the PII-free + `testdata/store/` from git-excluded raw captures, and a guard test fails the build if + account data reaches them. +- The tool is polite: bounded concurrent downloads (2 by default), backoff on rate limits, + and it identifies itself with a User-Agent. diff --git a/audit_test.go b/audit_test.go new file mode 100644 index 0000000..08ddb68 --- /dev/null +++ b/audit_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/curbol/unity-sync/internal/manifest" + "github.com/curbol/unity-sync/internal/model" + "github.com/curbol/unity-sync/internal/syncer" +) + +// Failure models the CLI must keep pinned: ways a command could do something other than +// what the user typed. + +// flag.Parse stops at the first positional, so an unchecked one swallows the flags after +// it: `sync foo --dry-run` would download. +func TestStrayPositionalIsRejectedAndSuggestsOnly(t *testing.T) { + isolate(t) + for _, cmd := range []string{"sync", "status", "list", "select"} { + code, err := run([]string{cmd, "some-asset", "--dry-run"}) + if code == 0 || err == nil { + t.Errorf("%s with a positional = %d, %v; want a failure", cmd, code, err) + continue + } + if !strings.Contains(err.Error(), "--only") { + t.Errorf("%s: error %q does not point at --only", cmd, err) + } + } +} + +func TestSessionWithoutTheCredentialIsNamedBeforeAnyRequest(t *testing.T) { + wd := isolate(t) + if err := os.WriteFile(filepath.Join(wd, manifest.FileName), []byte("\n"), 0o644); err != nil { + t.Fatal(err) + } + sessionPath := filepath.Join(t.TempDir(), "session.curl") + body := "curl 'https://assetstore.unity.com/' -H 'Cookie: DS=abc; _csrf=zzz'" + if err := os.WriteFile(sessionPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + code, err := run([]string{"status", "--session", sessionPath}) + if code == 0 || err == nil { + t.Fatalf("status = %d, %v; want a failure", code, err) + } + if !strings.Contains(err.Error(), "LS") { + t.Errorf("error %q does not name the missing credential cookie", err) + } +} + +// Without a session every networked command must fail with advice, not a stack trace or +// an opaque HTTP error. +func TestMissingSessionIsExplained(t *testing.T) { + wd := isolate(t) + if err := os.WriteFile(filepath.Join(wd, manifest.FileName), []byte("\n"), 0o644); err != nil { + t.Fatal(err) + } + code, err := run([]string{"status"}) + if code == 0 || err == nil { + t.Fatalf("status with no session = %d, %v; want a failure", code, err) + } + if !strings.Contains(err.Error(), "session") { + t.Errorf("error %q does not mention the session", err) + } +} + +// The class tally counts a delisted asset but cannot say which one, and which one is the +// only part the user can act on. +func TestTheSummaryNamesEachDelistedAsset(t *testing.T) { + buf := &bytes.Buffer{} + printReport(buf, syncer.Report{ + Owned: 2, + Results: []syncer.Result{ + {Asset: model.Asset{ID: "1", Name: "Still Fine", State: model.StatePublished}, Class: syncer.Unchanged}, + {Asset: model.Asset{ID: "193760", Name: "Fantasy Sounds Bundle", State: model.StateDisabled}, Class: syncer.Undownloadable}, + }, + }, false, "/lib") + out := buf.String() + for _, want := range []string{"Fantasy Sounds Bundle", "193760", "disabled"} { + if !strings.Contains(out, want) { + t.Errorf("the summary does not name %q:\n%s", want, out) + } + } + if strings.Contains(out, "Still Fine") { + t.Errorf("the summary named an asset that is not delisted:\n%s", out) + } +} diff --git a/cmd/scrubfixtures/main.go b/cmd/scrubfixtures/main.go new file mode 100644 index 0000000..33bbc19 --- /dev/null +++ b/cmd/scrubfixtures/main.go @@ -0,0 +1,53 @@ +// Command scrubfixtures regenerates testdata/store from the raw captures kept outside +// version control. Run it from the repo root; it is the only supported way to change +// those fixtures, because hand-editing them is how account data gets committed. +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/curbol/unity-sync/internal/fixtures" +) + +func main() { + from := flag.String("from", ".longrun/captures", "directory holding the raw captures") + to := flag.String("to", "testdata/store", "directory to write scrubbed fixtures into") + flag.Parse() + + if err := run(*from, *to); err != nil { + fmt.Fprintln(os.Stderr, "scrubfixtures:", err) + os.Exit(1) + } +} + +func run(from, to string) error { + matches, err := filepath.Glob(filepath.Join(from, "*.json")) + if err != nil { + return err + } + if len(matches) == 0 { + return fmt.Errorf("no captures in %s (they are git-excluded; see docs/design.md)", from) + } + if err := os.MkdirAll(to, 0o755); err != nil { + return err + } + for _, src := range matches { + raw, err := os.ReadFile(src) + if err != nil { + return err + } + clean, err := fixtures.Scrub(raw) + if err != nil { + return fmt.Errorf("%s: %w", src, err) + } + dst := filepath.Join(to, filepath.Base(src)) + if err := os.WriteFile(dst, clean, 0o644); err != nil { + return err + } + fmt.Printf("wrote %s (%d bytes)\n", dst, len(clean)) + } + return nil +} diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 0000000..7395996 --- /dev/null +++ b/config.example.toml @@ -0,0 +1,25 @@ +# Example USER config. Copy to your config dir and edit (all keys optional): +# +# mkdir -p ~/.config/unity-sync +# cp config.example.toml ~/.config/unity-sync/config.toml +# +# Config dir resolution: --config > $UNITY_SYNC_CONFIG_DIR > +# $XDG_CONFIG_HOME/unity-sync > ~/.config/unity-sync. +# +# This file is user-scoped: session and machine defaults. Which assets a project mirrors +# is project-scoped and lives in unity-sync.toml instead. + +# Where your saved session lives: a pasted-curl file or a cookies.txt export. If unset, +# unity-sync looks for session.curl or cookies.txt in this config dir. +# +# There is no browser option. The cookie the store checks is a session cookie that never +# reaches a browser's cookie database, so reading one could not authenticate. +# session_source = "~/.config/unity-sync/session.curl" + +# Where packages are cached. Default: $XDG_DATA_HOME/unity-sync +# (~/.local/share/unity-sync). Or set UNITY_SYNC_LIBRARY / pass --library. +# library_path = "/path/to/your/unity-library" + +# Max simultaneous downloads. Packages here reach 23 GB and the store is someone else's +# infrastructure, so this is deliberately small. +concurrency = 2 diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..e345c37 --- /dev/null +++ b/docs/design.md @@ -0,0 +1,252 @@ +# unity-sync design + +`unity-sync` mirrors the assets owned on the Unity Asset Store into a local library, +detecting and downloading only what changed. This document is the authoritative record of +the store's behaviour, the identity rules, and the failure model. Read it before changing +enumeration, the lockfile, the cache layout, or the download guards. + +## Goals + +- One command pulls every selected asset to its current version with no clicking. +- Detect updates without downloading: the list API carries each asset's current version. +- A committed lockfile is the record of what is owned and at what version, whose monthly + diff reads like a changelog. +- The cache is local and expendable; the assets a project actually ships are made durable + in that project, not here. +- Resilient to what actually breaks: an expired session, a delisted asset, a truncated + transfer, a renamed asset. + +## Non-goals + +- Importing or unpacking `.unitypackage` into a Unity project. +- Converting or previewing assets. Browsing the mirror is [quarry](https://github.com/curbol/quarry). +- Scripting the Unity ID login. The session is handed in. + +## What the store exposes (measured) + +| Purpose | Call | +| --- | --- | +| Owned-asset list | `POST /api/graphql/batch`, operation `SearchMyAssets` | +| Single-product re-read | the same document with `ids: [""]` | +| Package bytes | `GET /api/downloads/{productId}` | + +The GraphQL body is a batch: a JSON array of operations, answered by a positional array. + +### Authentication + +`LS` is the entire credential. Measured cookie-by-cookie: `_csrf` + `LS` alone returns the +full owned list; a junk or absent `__Secure-next-auth.session-token` changes nothing; +removing `LS` turns any user-scoped query into an HTTP 500 with an empty `GraphqlError`. + +`LS` is a session cookie, so it never reaches a browser's cookie database. That is why the +only supported session sources are a pasted curl command and a `cookies.txt` export, and +why there is no browser-reading mode: it could not work. + +The `_csrf` cookie is a double-submit token required by the GraphQL endpoint only. Not +every storefront route issues it — `/` and `/publishers/{id}` answer 200 and set nothing, +while `/packages` answers 404 and sets it. The bootstrap route is pinned to `/packages`, +treats its own 404 as normal, and is exempt from the redirect rule below. + +`x-requested-with: XMLHttpRequest` decides the *shape* of a failure: with it, a failed call +answers with the JSON error that carries the diagnosis; without it, the same call answers +302 to an HTML error page. + +### Identity + +Three ids come back per product. `id` is the store product id: the one +`/api/downloads/{id}` takes and the one stamped inside the package. `productId` is a +different 12-digit value no endpoint here accepts, and `itemId` is likewise unused. The +domain model carries only `id`. + +Assets key on `slugify(name) + "-" + id`. That key is *not* the identity — a rename changes +it — so classification looks up prior entries by product id, and a rename re-keys the entry +and moves the cached directory rather than re-downloading. + +### Two version identities + +The store advertises `currentVersion.id`; the served package carries its own `version_id`. +These usually agree but sometimes do not, steadily: product 262163 advertises 1094273 and +serves 905463; 262495 advertises 1056339 and serves 839208. Twelve of fourteen measured +packages agree. + +So the lockfile records both. `version.id` is the advertised value, refreshed every run. +`resolvedVersionId` is the advertised id the cached file was fetched against, and it is the +diff key. Diffing on the delivered id instead would make those products re-download +forever. + +### The download endpoint + +`GET /api/downloads/{id}` returns the bytes directly: 200, `application/octet-stream`, with +`Content-Disposition` usually but not always present. There is no CDN redirect, no +`Content-Length`, no `ETag`, and `Range` is ignored — so resume is impossible and nothing +pretends otherwise. + +Two behaviours matter more than they look: + +- An unauthenticated request answers **302 to Unity's OAuth authorize URL**. A client that + follows redirects writes a sign-in page into the cache under a `.unitypackage` name. +- The endpoint honours `Accept-Encoding: gzip` by **gzipping the already-gzipped package**. + Go does not transparently decode an encoding the caller asked for, so a client that sets + the header itself caches a double-gzipped blob with no readable metadata. The tool sends + `Accept-Encoding: identity` and treats any `Content-Encoding` on the response as an error. + +`downloadSize` is approximate: it runs 0-16 bytes above the bytes delivered, an artifact of +rounding up to a 16-byte boundary. It bounds a transfer; it never checksums one. + +### Packages self-describe + +A `.unitypackage` is gzip whose **FEXTRA** field carries a JSON descriptor in a subfield +with id `A$` — not the comment field, which is empty on every real package. It holds the +product id, the version id, the Unity version and the publisher. Reading it costs a header +parse, so a cached file can be identified without decompressing or hashing it. + +The reader is driven by the subfield's own length. Every sampled package ends its +descriptor by byte 338, but XLEN is a uint16: a prefix-limited reader would silently report +"no metadata" for a package that has some, downgrading the hard wrong-asset check into a +tolerated warning. + +## Run flow + +``` +1. Resolve config user config dir, library path, session source +2. Load session Cookie header; assert LS is present +3. Bootstrap CSRF GET /packages (404, but sets _csrf) +4. Enumerate page 0..n at pageSize 100; compare raw rows to `total`; dedup +5. Apply the allowlist manifest entries with enabled = true, then --only +6. Sweep stale temps walk the tree; before classification, so a partial is never adopted +7. Classify Unchanged | New | Changed | DownloadNow | CacheMissing | Adopted | Undownloadable +8. Download the delta bounded; guard against the temp file; commit; persist per asset +9. Finish final lockfile write and summary +``` + +`status` is steps 1-7 with `DryRun`, which also gates every mutating step: a dry run sweeps +nothing, moves nothing, and writes nothing. + +## Where the guards live + +The download stream crosses two packages, so ownership is fixed rather than left to +whoever writes the signature first: + +- `store` owns the response-level guards, before any bytes are kept: no redirect followed, + no `Content-Encoding` accepted, content type must be an octet-stream. It returns an open + body plus the filename it parsed. +- `cache` owns the write, in two phases. `Store` streams to a temp file beside the + destination and hashes as it goes; `Commit` renames; `Discard` removes. Nothing + unverified ever occupies a real cache path, because an interrupt in that window would + strand a rejected body where the next run's adopt scan would take it for genuine. +- `syncer` owns the semantic guards, which need both the bytes and the enumeration + metadata: gzip magic, the descriptor's product id, the size floor and its re-query. + +Retry wraps store+cache together, so every attempt necessarily opens a fresh temp file and +a fresh hasher. Appending a retried response to a partial one would survive every other +check and then be hashed and recorded as its own truth. + +## The size floor + +Truncation is normally caught by the transport, but only for a *dropped* connection: a +stream the origin ends cleanly early yields a clean EOF, and with no `Content-Length` +nothing else notices. The descriptor lives in the first ~300 bytes, so it survives +truncation too. + +So a received count below `downloadSize - min(4096, downloadSize/8)` fails that asset. The +allowance is absolute because the gap it forgives is a fixed alignment artifact, and +clamped because 4 KB is a third of the smallest owned package. A body outside the tight +±64 window but above the floor is a warning. + +The one legitimate way to fall below the floor is a republish mid-download. The +discriminator is a single re-read of that product: if its advertised version or size moved, +it was a republish. It is deliberately *not* "the delivered id differs from the advertised +id", which is a steady state for some products and would switch the floor off permanently +for exactly them. + +## Adoption + +A package can be on disk with no lockfile resolution behind it: the lockfile was deleted, +the asset was mirrored on another machine, or a rename moved the path out from under the +record. Rather than re-fetching gigabytes, a run scans the library for a file whose own +descriptor claims that product and adopts it — recording its size, digest and delivered +version id, and moving it to the path the current layout dictates before recording it, +so quarry's facets and the lockfile agree. + +Three gates keep adoption from laundering a bad file into the cache. The descriptor's +product id must match. Its version id must match what the store currently advertises, so a +stale build cannot be recorded as current. And the file must clear the same size floor a +download must clear, because that is the one route into the cache that skips the download +guards entirely. + +A file that just failed verification is excluded from the scan. A truncation or a mid-file +flip leaves the descriptor intact and a small truncation clears the floor, so without that +exclusion the damaged bytes would be re-hashed and their digest recorded as the asset's +truth — the precise outcome every other guard exists to prevent. + +A run deletes a package only when it is replacing that same asset's own copy, which +happens three ways: a download lands at a different derived path than the entry's previous +one, an adoption does the same, or an adoption replaces a recorded copy that failed its +check and is sitting on the destination. None of these is the de-owned case, which is +reported and left in place. The cache holds only current versions, so a superseded copy of +the same asset is not something to keep, but a copy the tool did not write is never +touched: every path removed here came out of the lockfile. + +## Failure model + +| Observation | Meaning | +| --- | --- | +| 400 `csrf token mismatch` | bootstrap failed | +| 500 + empty `GraphqlError` | session expired; not retried, because the status alone would say to retry | +| 3xx from any store endpoint | session expired; never followed | +| 404 on a download | the asset was pulled; permanent, so it does not fail the run | +| 429 / 408 / 5xx elsewhere | retried with backoff | +| rows collected != `total` | loud error, never a silent short walk | +| 200 with a non-empty `errors` array | loud error, never "you own nothing" | + +A failed download fails its asset, not the run: one delisted or corrupt package must not +stop a 75 GB mirror. The pool cancels early only for a run-fatal error. The exit status +separates actionable from permanent — a corrupt body exits non-zero, a pulled asset does +not. + +## Lockfile + +Committed beside the manifest as `unity-sync.lock.json`, keyed by asset slug, with +`assetId` inside each entry. Every owned asset gets an entry, whether or not it is +selected, because the file records what is *owned*. + +Each entry has two halves. The advertised half (`name`, `state`, `publisher`, `version`, +`advertisedSize`) refreshes every run. The resolution half (`tracked`, `resolvedVersionId`, +`deliveredVersionId`, `sizeBytes`, `sha256`, `cachePath`, `downloadedAt`, `storeFilename`) +is rewritten only when the run resolves that asset, and is otherwise carried forward +verbatim — along with the entry's key, so key and path cannot drift apart. + +`sizeBytes` is always the received count, never the advertised one. There is no run +timestamp: stamping one would dirty a committed file on every no-op run. + +## Cache layout + +``` +///.unitypackage +``` + +Three segments because quarry derives its vendor facet from the first path segment and its +pack facet from the second, filling the latter only when a path has at least three parts. A +flat tree would index every package with both facets empty. + +The filename is derived, not taken from `Content-Disposition`, which the store sends +inconsistently — trusting it would let one asset land under two names across runs and put a +machine-dependent path into a committed lockfile. + +Both slugs fall back when a name folds to nothing under ASCII folding: the publisher +segment becomes `publisher-` and the asset segment the bare product id. An empty +segment would collapse the layout and empty quarry's facets. + +## Testing + +The default suite is fully offline: `httptest` servers plus committed fixtures scrubbed +from real captures. The fixtures carry no account data, and a guard test fails the build if +any appears. Raw captures are never committed, and the scrubber lands before anything that +consumes fixtures, because git keeps what a later commit deletes. + +## Open questions + +- Whether Firefox's session store (`sessionstore-backups/recovery.jsonlz4`) holds `LS`. If + it does, a reader there would restore a zero-paste workflow. Unverified. +- Whether the Unity Editor recognises this cache layout if `library_path` points at its + `Asset Store-5.x` directory. Untested; the docs claim nothing. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9bcf9cd --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/curbol/unity-sync + +go 1.26 + +require github.com/BurntSushi/toml v1.6.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f74b269 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..d899311 --- /dev/null +++ b/install.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# unity-sync installer. Downloads the latest release binary for your platform into +# ~/.local/bin. The repo is private, so it authenticates with GITHUB_TOKEN, GH_TOKEN, +# or the gh CLI. +# +# Usage: +# gh api repos/curbol/unity-sync/contents/install.sh --jq .content | base64 -d | bash +set -euo pipefail + +REPO="curbol/unity-sync" +BINARY_NAME="unity-sync" +INSTALL_DIR="${HOME}/.local/bin" + +log() { printf 'INFO: %s\n' "$1"; } +err() { printf 'ERROR: %s\n' "$1" >&2; } + +TMPDIR_SELF="" +cleanup() { [[ -z "$TMPDIR_SELF" ]] || rm -rf "$TMPDIR_SELF"; } +trap cleanup EXIT + +auth_token() { + local token="${GITHUB_TOKEN:-${GH_TOKEN:-}}" + if [[ -z "$token" ]] && command -v gh >/dev/null 2>&1; then + token=$(gh auth token 2>/dev/null || true) + fi + printf '%s' "$token" +} + +# fetch GETs a URL, passing the credential through a curl config on stdin rather than as an +# argument, which would put the token in `ps` output for every local user. Callers append +# `|| true` where they want to report the failure themselves: every one of them assigns from +# a pipeline, and `set -e` plus `pipefail` would otherwise abort before the diagnostic runs. +fetch() { + local url="$1"; shift + local token; token=$(auth_token) + if [[ -n "$token" ]]; then + printf 'header = "Authorization: token %s"\n' "$token" | curl -fsSL -K - "$@" "$url" + else + curl -fsSL "$@" "$url" + fi +} + +detect_platform() { + local os arch + case "$(uname -s)" in + Darwin*) os="mac" ;; + Linux*) os="linux" ;; + *) err "unsupported OS $(uname -s); on Windows use the release zip directly"; exit 1 ;; + esac + case "$(uname -m)" in + x86_64|amd64) arch="intel" ;; + arm64|aarch64) [[ "$os" == "mac" ]] && arch="apple" || arch="arm64" ;; + *) err "unsupported arch $(uname -m)"; exit 1 ;; + esac + PLATFORM="${os}-${arch}" + log "platform: $PLATFORM" +} + +latest_version() { + VERSION=$(fetch "https://api.github.com/repos/${REPO}/releases/latest" \ + | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' || true) + VERSION=${VERSION#v} + [[ -n "$VERSION" ]] || { err "could not resolve latest version (private repo needs gh auth or GITHUB_TOKEN)"; exit 1; } + log "latest version: $VERSION" +} + +install() { + local file="${BINARY_NAME}-${VERSION}-${PLATFORM}.zip" + TMPDIR_SELF=$(mktemp -d) + local tmp="$TMPDIR_SELF" + local url + + if [[ -n "$(auth_token)" ]]; then + # Private repo: resolve the asset's API URL, then download with the token. + url=$(fetch "https://api.github.com/repos/${REPO}/releases/tags/v${VERSION}" \ + | grep -F -B3 "\"name\": \"${file}\"" | grep -F '"url"' | sed -E 's/.*"url": "([^"]+)".*/\1/' || true) + [[ -n "$url" ]] || { err "asset ${file} not found in release v${VERSION}"; exit 1; } + fetch "$url" -H "Accept: application/octet-stream" -o "${tmp}/${file}" \ + || { err "could not download ${file}"; exit 1; } + else + curl -fsSL -o "${tmp}/${file}" "https://github.com/${REPO}/releases/download/v${VERSION}/${file}" \ + || { err "could not download ${file}"; exit 1; } + fi + + command -v unzip >/dev/null 2>&1 || { err "unzip is required"; exit 1; } + unzip -q "${tmp}/${file}" -d "$tmp" + mkdir -p "$INSTALL_DIR" + mv "${tmp}/${BINARY_NAME}" "${INSTALL_DIR}/${BINARY_NAME}" + chmod +x "${INSTALL_DIR}/${BINARY_NAME}" + log "installed to ${INSTALL_DIR}/${BINARY_NAME}" +} + +check_path() { + case ":$PATH:" in + *":$INSTALL_DIR:"*) ;; + *) log "note: $INSTALL_DIR is not on your PATH; add: export PATH=\"$INSTALL_DIR:\$PATH\"" ;; + esac +} + +detect_platform +latest_version +install +check_path +"${INSTALL_DIR}/${BINARY_NAME}" version \ + || { err "installed but 'unity-sync version' failed"; exit 1; } diff --git a/internal/cache/audit_test.go b/internal/cache/audit_test.go new file mode 100644 index 0000000..27f8b43 --- /dev/null +++ b/internal/cache/audit_test.go @@ -0,0 +1,183 @@ +package cache_test + +import ( + "bytes" + "os" + "path/filepath" + "testing" + "time" + + "github.com/curbol/unity-sync/internal/cache" +) + +// Failure models this package must keep pinned. Every case is a way the cache could record +// the wrong bytes as verified. + +// The window between writing bytes and accepting them is where a rejected body would +// otherwise sit at a real cache path. +func TestStoreLeavesNothingAtTheRealPathUntilCommit(t *testing.T) { + root := t.TempDir() + body := pkg(t, "115488", "683375", 500) + p, err := cache.Store(root, "chris-nolet", "quick-outline-115488", bytes.NewReader(body)) + if err != nil { + t.Fatalf("Store: %v", err) + } + final := filepath.Join(root, filepath.FromSlash(p.RelPath)) + if _, err := os.Stat(final); !os.IsNotExist(err) { + t.Fatal("Store put bytes at the real path before they were checked") + } + if p.Size != int64(len(body)) { + t.Errorf("Size = %d, want %d", p.Size, len(body)) + } + if err := p.Commit(); err != nil { + t.Fatalf("Commit: %v", err) + } + if _, err := os.Stat(final); err != nil { + t.Fatalf("Commit did not place the file: %v", err) + } +} + +// The caller records the digest of whatever lands at the destination, so an overwrite +// here would certify the wrong bytes — with nothing else in the design watching. +func TestRelocateRefusesAnOccupiedDestination(t *testing.T) { + root := t.TempDir() + from := cache.RelPath("pub", "stray-111") + to := cache.RelPath("pub", "asset-111") + storeCommitted(t, root, "pub", "stray-111", pkg(t, "111", "9", 400)) + storeCommitted(t, root, "pub", "asset-111", pkg(t, "111", "9", 900)) + + if err := cache.Relocate(root, from, to); err == nil { + t.Fatal("Relocate silently overwrote an occupied destination") + } + fi, err := os.Stat(filepath.Join(root, filepath.FromSlash(to))) + if err != nil || fi.Size() != 900 { + t.Errorf("the destination file was disturbed: size %v, err %v", fi.Size(), err) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(from))); err != nil { + t.Error("the source file was lost to a refused move") + } +} + +func TestSweepWalksTheTreeAndSparesInFlightTemps(t *testing.T) { + root := t.TempDir() + leaf := filepath.Join(root, "pub", "asset-1") + if err := os.MkdirAll(leaf, 0o755); err != nil { + t.Fatal(err) + } + stale := filepath.Join(leaf, ".unity-sync-dl-old") + fresh := filepath.Join(leaf, ".unity-sync-dl-live") + os.WriteFile(stale, bytes.Repeat([]byte("x"), 100), 0o644) + os.WriteFile(fresh, bytes.Repeat([]byte("x"), 50), 0o644) + old := time.Now().Add(-2 * time.Hour) + os.Chtimes(stale, old, old) + + cutoff := time.Now().Add(-time.Hour) + n, bytesFreed := cache.SweepTemps(root, cutoff) + // A root-only scan would report zero here while leaving a multi-gigabyte orphan. + if n != 1 || bytesFreed != 100 { + t.Errorf("swept %d files / %d bytes, want 1 / 100", n, bytesFreed) + } + if _, err := os.Stat(fresh); err != nil { + t.Error("the sweep deleted a temp newer than the cutoff, i.e. another run's transfer") + } +} + +func TestUnsafePathsAreRefused(t *testing.T) { + root := t.TempDir() + for _, seg := range []string{"", ".", "..", "a/b", `a\b`, ".hidden", "with\x00null"} { + if _, err := cache.Store(root, seg, "asset", bytes.NewReader([]byte("x"))); err == nil { + t.Errorf("Store accepted publisher slug %q", seg) + } + if _, err := cache.Store(root, "pub", seg, bytes.NewReader([]byte("x"))); err == nil { + t.Errorf("Store accepted asset slug %q", seg) + } + } + for _, rel := range []string{"", "/etc/passwd", "../escape.unitypackage", "a/../../escape"} { + if cache.Verify(root, rel, 1, "") { + t.Errorf("Verify accepted path %q", rel) + } + if _, _, err := cache.Hash(root, rel); err == nil { + t.Errorf("Hash accepted path %q", rel) + } + } +} + +// A library that does not exist yet is the first-run case, not an error: the sweep runs +// before anything has been written. +func TestSweepingAMissingRootIsNotAnError(t *testing.T) { + n, bytes := cache.SweepTemps(filepath.Join(t.TempDir(), "never-created"), time.Now()) + if n != 0 || bytes != 0 { + t.Errorf("swept %d files / %d bytes from a missing root", n, bytes) + } +} + +// The exclude list comes from the lockfile, which is hand-editable and travels between +// machines. Comparing it as a raw string would let "./pub/a/a.unitypackage" fail to skip +// the file "pub/a/a.unitypackage" names, re-offering a file that just failed verification +// as something to adopt. +func TestLocateSkipsAnExcludedFileWrittenNonCanonically(t *testing.T) { + root := t.TempDir() + rel := cache.RelPath("pub", "asset-1") + storeCommitted(t, root, "pub", "asset-1", pkg(t, "111", "9", 400)) + + if _, ok := cache.Locate(root, "111", "", rel); ok { + t.Fatal("the canonical exclude did not skip the file") + } + for _, spelling := range []string{"./" + rel, "pub/./asset-1/asset-1.unitypackage"} { + if _, ok := cache.Locate(root, "111", "", spelling); ok { + t.Errorf("exclude %q did not skip the same file", spelling) + } + } +} + +// The library root comes from a flag or a config file, so it arrives however the user +// typed it. Comparing it raw against paths that have been cleaned makes pruning a no-op +// for the ordinary "./lib" spelling, and nothing else notices. +func TestPruningSurvivesHoweverTheRootWasSpelled(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + spellings := map[string]func(string) string{ + "absolute": func(base string) string { return base }, + "trailing slash": func(base string) string { return base + string(filepath.Separator) }, + "dot relative": func(base string) string { + rel, err := filepath.Rel(wd, base) + if err != nil { + t.Skip("temp dir is not reachable relatively from the working directory") + } + return "." + string(filepath.Separator) + rel + }, + } + for name, spell := range spellings { + t.Run(name, func(t *testing.T) { + base := t.TempDir() + root := spell(base) + storeCommitted(t, root, "pub", "old-slug-1", pkg(t, "1", "9", 400)) + if err := cache.Relocate(root, cache.RelPath("pub", "old-slug-1"), + cache.RelPath("pub", "new-slug-1")); err != nil { + t.Fatalf("Relocate: %v", err) + } + if _, err := os.Stat(filepath.Join(base, "pub", "old-slug-1")); !os.IsNotExist(err) { + t.Errorf("root spelled %q left the emptied directory behind", root) + } + }) + } +} + +// An exclusion names a file that must not be adopted. If it cannot be resolved, the scan +// cannot tell which file that is, and offering a candidate anyway would let the excluded +// one back in through the door that skips the download guards. +func TestAnUnresolvableExclusionRefusesEveryCandidate(t *testing.T) { + root := t.TempDir() + storeCommitted(t, root, "pub", "asset-1", pkg(t, "111", "9", 400)) + + if _, ok := cache.Locate(root, "111", ""); !ok { + t.Fatal("the candidate is not findable at all") + } + for _, bad := range []string{"/etc/passwd", "../outside/x.unitypackage"} { + if _, ok := cache.Locate(root, "111", "", bad); ok { + t.Errorf("exclusion %q was dropped and a candidate offered anyway", bad) + } + } +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go new file mode 100644 index 0000000..0f1e55d --- /dev/null +++ b/internal/cache/cache.go @@ -0,0 +1,388 @@ +// Package cache is the local package mirror. Its layout is three segments deep — +// //.unitypackage — because quarry derives its vendor facet +// from the first path segment and its pack facet from the second, filling the latter +// only when a path has at least three parts. A flat tree would index every package with +// both facets empty. +// +// Writes are two-phase on purpose: Store leaves the bytes in a temp file so the caller's +// semantic checks run before Commit renames anything into place. Nothing unverified ever +// occupies a real cache path, even briefly, because an interrupt in that window would +// strand a rejected body where the next run's adopt scan would take it for genuine. +package cache + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/curbol/unity-sync/internal/unitypackage" +) + +// tempPrefix marks an in-flight download. The sweep looks for it; the adopt scan +// deliberately does not consider it. +const tempPrefix = ".unity-sync-dl-" + +const packageExt = ".unitypackage" + +// RelPath is an asset's location relative to the library root, in forward slashes so the +// value is portable in a committed lockfile. +func RelPath(publisherSlug, assetSlug string) string { + return path.Join(publisherSlug, assetSlug, assetSlug+packageExt) +} + +// safeSegment rejects anything that is not a single, ordinary path element. Both slugs +// are derived from store-supplied names, so neither is trusted to be a bare word. +func safeSegment(kind, s string) error { + if s == "" || s == "." || s == ".." || strings.ContainsAny(s, `/\`) || strings.HasPrefix(s, ".") { + return fmt.Errorf("unsafe %s %q", kind, s) + } + for _, r := range s { + if r < 0x20 || r == 0x7f { + return fmt.Errorf("unsafe %s %q: contains a control character", kind, s) + } + } + return nil +} + +// resolve turns a lockfile-supplied relative path into an absolute one, refusing +// anything that would leave the library root. These values arrive from a file that is +// committed and travels between machines, so they are validated rather than trusted. +func resolve(root, rel string) (string, error) { + if rel == "" || path.IsAbs(rel) || filepath.IsAbs(rel) { + return "", fmt.Errorf("unsafe cache path %q", rel) + } + clean := filepath.Clean(filepath.FromSlash(rel)) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("unsafe cache path %q", rel) + } + return filepath.Join(root, clean), nil +} + +// Pending is a fully-written but uncommitted download. +type Pending struct { + RelPath string + SHA256 string + Size int64 + + tempPath string + final string +} + +// TempPath is where the bytes currently are, so the caller can inspect them before +// deciding to commit. +func (p *Pending) TempPath() string { return p.tempPath } + +// Store streams r into a temp file beside its eventual destination, hashing as it goes. +// It does not rename: the caller commits or discards. +func Store(root, publisherSlug, assetSlug string, r io.Reader) (*Pending, error) { + if err := safeSegment("publisher slug", publisherSlug); err != nil { + return nil, err + } + if err := safeSegment("asset slug", assetSlug); err != nil { + return nil, err + } + rel := RelPath(publisherSlug, assetSlug) + dir := filepath.Join(root, publisherSlug, assetSlug) + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + tmp, err := os.CreateTemp(dir, tempPrefix+"*") + if err != nil { + return nil, err + } + h := sha256.New() + size, err := io.Copy(io.MultiWriter(tmp, h), r) + if err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return nil, err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + os.Remove(tmp.Name()) + return nil, err + } + if err := tmp.Close(); err != nil { + os.Remove(tmp.Name()) + return nil, err + } + return &Pending{ + RelPath: rel, + SHA256: hex.EncodeToString(h.Sum(nil)), + Size: size, + tempPath: tmp.Name(), + final: filepath.Join(dir, assetSlug+packageExt), + }, nil +} + +// Commit renames the pending bytes into place. +func (p *Pending) Commit() error { + if err := os.Rename(p.tempPath, p.final); err != nil { + os.Remove(p.tempPath) + return err + } + return nil +} + +// Discard removes the pending bytes. Callers use it whenever a check fails, so a +// rejected body never reaches a real cache path. +func (p *Pending) Discard() error { + err := os.Remove(p.tempPath) + if os.IsNotExist(err) { + return nil + } + return err +} + +// Verify is the cheap check: the file exists, its size is exactly what was recorded, and +// — when a delivered version id was recorded — its own metadata still says so. Exact +// recorded size is what makes truncation detectable without hashing, since the metadata +// block sits in the leading bytes and survives a truncation. +// +// An entry with no delivered id is verified on size alone. Requiring a metadata match +// there would make a package that simply has no descriptor re-download on every run. +func Verify(root, rel string, wantSize int64, wantDeliveredID string) bool { + full, err := resolve(root, rel) + if err != nil { + return false + } + fi, err := os.Stat(full) + if err != nil || fi.Size() != wantSize { + return false + } + if wantDeliveredID == "" { + return true + } + m, err := unitypackage.ReadFile(full) + if err != nil { + return false + } + return m.VersionID == wantDeliveredID +} + +// VerifyDeep re-hashes the file. It is opt-in because the library runs to tens of +// gigabytes, and it is the only check that sees a mid-file corruption. +func VerifyDeep(root, rel, wantSHA string) bool { + full, err := resolve(root, rel) + if err != nil { + return false + } + f, err := os.Open(full) + if err != nil { + return false + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return false + } + return hex.EncodeToString(h.Sum(nil)) == wantSHA +} + +// Hash returns a cached file's digest and size, for adopting a file the tool did not +// download itself. +func Hash(root, rel string) (string, int64, error) { + full, err := resolve(root, rel) + if err != nil { + return "", 0, err + } + f, err := os.Open(full) + if err != nil { + return "", 0, err + } + defer f.Close() + h := sha256.New() + size, err := io.Copy(h, f) + if err != nil { + return "", 0, err + } + return hex.EncodeToString(h.Sum(nil)), size, nil +} + +// Candidate is a package found on disk during an adopt scan. +type Candidate struct { + RelPath string + Size int64 + Metadata unitypackage.Metadata +} + +// Locate scans the library for a package whose own metadata claims the given product id. +// It scans rather than probing the derived path because the whole point of adoption is a +// file that is not where the current layout would put it — after a rename, say. +// +// Only files ending .unitypackage and not starting with a dot are considered, so an +// abandoned download temp can never be adopted: a partial can be large enough to clear a +// size floor while still carrying an intact descriptor. +// +// When several files claim the same product, the one already at preferRel wins, so an +// adopt that is really a no-op does not turn into a relocation conflict. Paths in +// excludeRel are skipped entirely: a file that just failed verification is not a candidate +// for adoption, however intact its descriptor still looks. +func Locate(root, productID, preferRel string, excludeRel ...string) (Candidate, bool) { + // Resolved, not compared as strings: excludeRel comes from the lockfile, which is + // hand-editable and travels between machines, so "./pub/a/a.unitypackage" has to skip + // the same file "pub/a/a.unitypackage" names. Missing the match would re-offer a file + // that just failed verification as a candidate to adopt. + skip := map[string]bool{} + for _, e := range excludeRel { + if e == "" { + continue + } + full, err := resolve(root, e) + if err != nil { + // The caller is naming a file that must not be adopted and this cannot tell + // which one it is. Refusing every candidate falls back to a re-download, + // where the full set of guards applies; guessing would let the excluded file + // back in through the one door that skips them. + return Candidate{}, false + } + skip[full] = true + } + var found []Candidate + filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + name := d.Name() + if strings.HasPrefix(name, ".") || !strings.HasSuffix(name, packageExt) { + return nil + } + m, err := unitypackage.ReadFile(p) + if err != nil || m.ID != productID { + return nil + } + fi, err := d.Info() + if err != nil { + return nil + } + rel, err := filepath.Rel(root, p) + if err != nil { + return nil + } + if skip[filepath.Clean(p)] { + return nil + } + found = append(found, Candidate{RelPath: filepath.ToSlash(rel), Size: fi.Size(), Metadata: m}) + return nil + }) + if len(found) == 0 { + return Candidate{}, false + } + for _, c := range found { + if c.RelPath == preferRel { + return c, true + } + } + return found[0], true +} + +// Relocate moves a package to where the current layout puts it, creating parents and +// pruning directories the move empties. +// +// It is a no-op when the file is already there, and it refuses a destination holding a +// different file rather than renaming over it: the caller records the digest of whatever +// ends up at that path, so a silent overwrite would certify the wrong bytes. +func Relocate(root, fromRel, toRel string) error { + from, err := resolve(root, fromRel) + if err != nil { + return err + } + to, err := resolve(root, toRel) + if err != nil { + return err + } + if from == to { + return nil + } + if _, err := os.Stat(to); err == nil { + return fmt.Errorf("refusing to move %s onto %s: destination already holds a file", fromRel, toRel) + } else if !os.IsNotExist(err) { + return err + } + if err := os.MkdirAll(filepath.Dir(to), 0o755); err != nil { + return err + } + if err := os.Rename(from, to); err != nil { + return err + } + pruneEmptyParents(root, filepath.Dir(from)) + return nil +} + +// pruneEmptyParents removes directories the move emptied, walking up but never past the +// library root. +func pruneEmptyParents(root, dir string) { + // Cleaned, because every path this compares against came through resolve, which + // joins and cleans. A root of "./lib" or "lib/" would otherwise match nothing and + // silently prune nothing. + root = filepath.Clean(root) + for { + if dir == root || !strings.HasPrefix(dir, root+string(filepath.Separator)) { + return + } + entries, err := os.ReadDir(dir) + if err != nil || len(entries) > 0 { + return + } + if err := os.Remove(dir); err != nil { + return + } + dir = filepath.Dir(dir) + } +} + +// SweepTemps removes abandoned download temps anywhere in the tree, returning how many +// and how many bytes. It walks rather than scanning the root, because temps live beside +// their destinations, and it spares anything newer than the cutoff so a concurrent run's +// in-flight transfer survives. +// +// Nothing here fails: a subtree that cannot be read, or a root that does not exist yet on +// a first run, is skipped. One unreadable directory must not stop a 75 GB mirror over a +// housekeeping pass. +func SweepTemps(root string, olderThan time.Time) (int, int64) { + var count int + var bytes int64 + filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + if d.IsDir() || !strings.HasPrefix(d.Name(), tempPrefix) { + return nil + } + fi, err := d.Info() + if err != nil || !fi.ModTime().Before(olderThan) { + return nil + } + if os.Remove(p) == nil { + count++ + bytes += fi.Size() + } + return nil + }) + return count, bytes +} + +// RemoveStale deletes a package this tool mirrored and is now replacing with another copy +// of the same asset, and prunes the directories the removal empties. It is only ever +// called with a path the lockfile itself recorded, never with a file the tool did not +// write. +func RemoveStale(root, rel string) error { + full, err := resolve(root, rel) + if err != nil { + return err + } + if err := os.Remove(full); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + pruneEmptyParents(root, filepath.Dir(full)) + return nil +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go new file mode 100644 index 0000000..f31ed81 --- /dev/null +++ b/internal/cache/cache_test.go @@ -0,0 +1,199 @@ +package cache_test + +import ( + "bytes" + "compress/gzip" + "encoding/binary" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/curbol/unity-sync/internal/cache" +) + +// pkg builds a gzip stream carrying the store's descriptor, padded to size. +func pkg(t *testing.T, productID, versionID string, size int) []byte { + t.Helper() + descriptor := []byte(`{"id":"` + productID + `","version_id":"` + versionID + `"}`) + extra := []byte{'A', '$', 0, 0} + binary.LittleEndian.PutUint16(extra[2:4], uint16(len(descriptor))) + extra = append(extra, descriptor...) + + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + zw.Header.Extra = extra + zw.Write(bytes.Repeat([]byte("x"), 64)) + zw.Close() + out := buf.Bytes() + for len(out) < size { + out = append(out, 0) + } + return out +} + +func storeCommitted(t *testing.T, root, pub, asset string, body []byte) *cache.Pending { + t.Helper() + p, err := cache.Store(root, pub, asset, bytes.NewReader(body)) + if err != nil { + t.Fatalf("Store: %v", err) + } + if err := p.Commit(); err != nil { + t.Fatalf("Commit: %v", err) + } + return p +} + +func TestLayoutIsThreeSegmentsSoQuarryGetsBothFacets(t *testing.T) { + got := cache.RelPath("doublel", "quick-outline-115488") + want := "doublel/quick-outline-115488/quick-outline-115488.unitypackage" + if got != want { + t.Errorf("RelPath = %q, want %q", got, want) + } + if n := strings.Count(got, "/"); n != 2 { + t.Errorf("path has %d separators, want 2: quarry fills its pack facet only from a third segment", n+1) + } +} + +func TestDiscardRemovesTheTempAndLeavesNoFile(t *testing.T) { + root := t.TempDir() + p, err := cache.Store(root, "pub", "asset-1", bytes.NewReader(pkg(t, "1", "2", 300))) + if err != nil { + t.Fatal(err) + } + if err := p.Discard(); err != nil { + t.Fatalf("Discard: %v", err) + } + if _, err := os.Stat(p.TempPath()); !os.IsNotExist(err) { + t.Error("Discard left the temp file behind") + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(p.RelPath))); !os.IsNotExist(err) { + t.Error("Discard left a file at the real path") + } +} + +func TestVerifyUsesExactRecordedSizeAndMetadata(t *testing.T) { + root := t.TempDir() + body := pkg(t, "115488", "683375", 800) + p := storeCommitted(t, root, "pub", "quick-outline-115488", body) + + if !cache.Verify(root, p.RelPath, p.Size, "683375") { + t.Error("Verify rejected a file it just stored") + } + if cache.Verify(root, p.RelPath, p.Size+1, "683375") { + t.Error("Verify accepted a size that does not match the record") + } + if cache.Verify(root, p.RelPath, p.Size, "999999") { + t.Error("Verify accepted a file whose descriptor names another version") + } + + // Truncation is what the exact-size rule exists for: the descriptor sits in the + // leading bytes and survives it. + full := filepath.Join(root, filepath.FromSlash(p.RelPath)) + if err := os.Truncate(full, p.Size-100); err != nil { + t.Fatal(err) + } + if cache.Verify(root, p.RelPath, p.Size, "683375") { + t.Error("Verify accepted a truncated file") + } +} + +// A package with no descriptor is verified on size alone. Demanding a metadata match +// would make it re-download on every run, forever. +func TestVerifyFallsBackToSizeWhenNoDeliveredIdWasRecorded(t *testing.T) { + root := t.TempDir() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + zw.Write([]byte("no descriptor here")) + zw.Close() + p := storeCommitted(t, root, "pub", "plain-1", buf.Bytes()) + + if !cache.Verify(root, p.RelPath, p.Size, "") { + t.Error("Verify rejected a descriptor-less package that matches its recorded size") + } + if cache.Verify(root, p.RelPath, p.Size+5, "") { + t.Error("Verify accepted a size mismatch even with no delivered id") + } +} + +func TestOnlyDeepVerifySeesAMidFileFlip(t *testing.T) { + root := t.TempDir() + body := pkg(t, "1", "2", 900) + p := storeCommitted(t, root, "pub", "asset-1", body) + full := filepath.Join(root, filepath.FromSlash(p.RelPath)) + + f, err := os.OpenFile(full, os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + f.WriteAt([]byte{0xFF}, 700) // past the descriptor, size unchanged + f.Close() + + if !cache.Verify(root, p.RelPath, p.Size, "2") { + t.Error("cheap verify should not see a mid-file flip; that is what makes it cheap") + } + if cache.VerifyDeep(root, p.RelPath, p.SHA256) { + t.Error("deep verify missed a mid-file flip") + } +} + +func TestLocateFindsAPackageByItsOwnIdAndIgnoresTemps(t *testing.T) { + root := t.TempDir() + storeCommitted(t, root, "pub-a", "asset-1", pkg(t, "111", "9", 400)) + storeCommitted(t, root, "pub-b", "asset-2", pkg(t, "222", "9", 400)) + + // An abandoned partial with an intact descriptor must never be adoptable. + tempDir := filepath.Join(root, "pub-c", "asset-3") + os.MkdirAll(tempDir, 0o755) + os.WriteFile(filepath.Join(tempDir, ".unity-sync-dl-999"), pkg(t, "333", "9", 400), 0o644) + + got, ok := cache.Locate(root, "222", "") + if !ok || !strings.Contains(got.RelPath, "asset-2") { + t.Errorf("Locate(222) = %+v, %v", got, ok) + } + if _, ok := cache.Locate(root, "333", ""); ok { + t.Error("Locate adopted an abandoned download temp") + } +} + +func TestLocatePrefersTheFileAlreadyAtTheDerivedPath(t *testing.T) { + root := t.TempDir() + derived := cache.RelPath("pub", "asset-1") + storeCommitted(t, root, "pub", "asset-1", pkg(t, "111", "9", 400)) + storeCommitted(t, root, "old-pub", "old-asset", pkg(t, "111", "9", 400)) + + got, ok := cache.Locate(root, "111", derived) + if !ok || got.RelPath != derived { + t.Errorf("Locate = %q, want the copy already at %q", got.RelPath, derived) + } +} + +func TestRelocateMovesTheDirectoryAndPrunesTheOldOne(t *testing.T) { + root := t.TempDir() + from := cache.RelPath("pub", "old-slug-111") + storeCommitted(t, root, "pub", "old-slug-111", pkg(t, "111", "9", 400)) + to := cache.RelPath("pub", "new-slug-111") + + if err := cache.Relocate(root, from, to); err != nil { + t.Fatalf("Relocate: %v", err) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(to))); err != nil { + t.Fatalf("file is not at the new path: %v", err) + } + // quarry reads the pack facet from the directory, so the old directory must go. + if _, err := os.Stat(filepath.Join(root, "pub", "old-slug-111")); !os.IsNotExist(err) { + t.Error("the emptied old directory survived the move") + } +} + +func TestRelocateIsANoOpWhenAlreadyInPlace(t *testing.T) { + root := t.TempDir() + rel := cache.RelPath("pub", "asset-1") + storeCommitted(t, root, "pub", "asset-1", pkg(t, "111", "9", 400)) + if err := cache.Relocate(root, rel, rel); err != nil { + t.Errorf("Relocate onto itself = %v, want nil: this is the common adopt case", err) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(rel))); err != nil { + t.Errorf("the no-op relocation lost the file: %v", err) + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..92aeff2 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,116 @@ +// Package config resolves the user-scoped settings: where the session comes from, where +// the library lives, and how many downloads may run at once. Built-in defaults are +// overridden by config.toml in the XDG config dir, then by environment, then by flags. +// Project-scoped settings (the asset allowlist) live in the manifest, not here, and no +// machine-specific path is baked in. +package config + +import ( + "os" + "path/filepath" + "strings" + + "github.com/BurntSushi/toml" +) + +// Config is the resolved user-scoped configuration. +type Config struct { + // SessionSource is a path to a pasted-curl file or a cookies.txt. + SessionSource string + + // LibraryPath is where packages are mirrored. A user may point this at Unity's own + // Asset Store-5.x directory; whether the Editor recognises the layout is untested. + LibraryPath string + + // Concurrency bounds simultaneous downloads. Two by default: packages here reach + // 23 GB, and the store is someone else's infrastructure. + Concurrency int +} + +type fileConfig struct { + SessionSource string `toml:"session_source"` + LibraryPath string `toml:"library_path"` + Concurrency int `toml:"concurrency"` +} + +// ResolveDir picks the directory holding config.toml: an explicit flag, else +// $UNITY_SYNC_CONFIG_DIR, else $XDG_CONFIG_HOME/unity-sync, else ~/.config/unity-sync. +func ResolveDir(flag string) string { + if flag != "" { + return flag + } + if v := os.Getenv("UNITY_SYNC_CONFIG_DIR"); v != "" { + return v + } + if v := os.Getenv("XDG_CONFIG_HOME"); v != "" { + return filepath.Join(v, "unity-sync") + } + if home, err := os.UserHomeDir(); err == nil { + return filepath.Join(home, ".config", "unity-sync") + } + return "unity-sync" +} + +// defaultLibraryPath is $XDG_DATA_HOME/unity-sync, else ~/.local/share/unity-sync. App +// data rather than ~/.cache, so an OS cache cleaner cannot wipe a 75 GB mirror. +func defaultLibraryPath() string { + if v := os.Getenv("XDG_DATA_HOME"); v != "" { + return filepath.Join(v, "unity-sync") + } + if home, err := os.UserHomeDir(); err == nil { + return filepath.Join(home, ".local", "share", "unity-sync") + } + return "unity-library" +} + +func defaults() Config { + return Config{ + LibraryPath: defaultLibraryPath(), + Concurrency: 2, + } +} + +// Load merges built-in defaults, an optional config.toml in dir, then the environment +// (UNITY_SYNC_LIBRARY, UNITY_SYNC_SESSION). A missing config.toml is not an error; an +// unreadable or malformed one is. +func Load(dir string) (Config, error) { + c := defaults() + path := filepath.Join(dir, "config.toml") + if _, err := os.Stat(path); err == nil { + var fc fileConfig + if _, err := toml.DecodeFile(path, &fc); err != nil { + return Config{}, err + } + overlay(&c, fc) + } + if v := os.Getenv("UNITY_SYNC_LIBRARY"); v != "" { + c.LibraryPath = v + } + if v := os.Getenv("UNITY_SYNC_SESSION"); v != "" { + c.SessionSource = v + } + c.LibraryPath = expandHome(c.LibraryPath) + c.SessionSource = expandHome(c.SessionSource) + return c, nil +} + +func overlay(c *Config, fc fileConfig) { + if fc.SessionSource != "" { + c.SessionSource = fc.SessionSource + } + if fc.LibraryPath != "" { + c.LibraryPath = fc.LibraryPath + } + if fc.Concurrency > 0 { + c.Concurrency = fc.Concurrency + } +} + +func expandHome(p string) string { + if p == "~" || strings.HasPrefix(p, "~/") { + if home, err := os.UserHomeDir(); err == nil { + return filepath.Join(home, strings.TrimPrefix(p, "~")) + } + } + return p +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..14dfb5a --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,119 @@ +package config_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/curbol/unity-sync/internal/config" +) + +// isolate clears every variable the resolver reads, so a developer's own environment +// cannot make these pass or fail. +func isolate(t *testing.T) { + t.Helper() + for _, k := range []string{"UNITY_SYNC_CONFIG_DIR", "XDG_CONFIG_HOME", "XDG_DATA_HOME", + "UNITY_SYNC_LIBRARY", "UNITY_SYNC_SESSION"} { + t.Setenv(k, "") + os.Unsetenv(k) + } + t.Setenv("HOME", t.TempDir()) +} + +func TestResolveDirPrecedence(t *testing.T) { + isolate(t) + home := os.Getenv("HOME") + + if got, want := config.ResolveDir(""), filepath.Join(home, ".config", "unity-sync"); got != want { + t.Errorf("bare default = %q, want %q", got, want) + } + t.Setenv("XDG_CONFIG_HOME", "/xdg") + if got, want := config.ResolveDir(""), filepath.Join("/xdg", "unity-sync"); got != want { + t.Errorf("XDG_CONFIG_HOME = %q, want %q", got, want) + } + t.Setenv("UNITY_SYNC_CONFIG_DIR", "/envdir") + if got := config.ResolveDir(""); got != "/envdir" { + t.Errorf("UNITY_SYNC_CONFIG_DIR = %q, want /envdir (env beats XDG)", got) + } + if got := config.ResolveDir("/flagdir"); got != "/flagdir" { + t.Errorf("flag = %q, want /flagdir (flag beats env)", got) + } +} + +func TestLoadDefaultsWhenNoFileExists(t *testing.T) { + isolate(t) + c, err := config.Load(t.TempDir()) + if err != nil { + t.Fatalf("Load with no config.toml: %v", err) + } + if c.Concurrency != 2 { + t.Errorf("Concurrency = %d, want 2", c.Concurrency) + } + if c.SessionSource != "" { + t.Errorf("SessionSource = %q, want empty: there is no browser default", c.SessionSource) + } + want := filepath.Join(os.Getenv("HOME"), ".local", "share", "unity-sync") + if c.LibraryPath != want { + t.Errorf("LibraryPath = %q, want %q", c.LibraryPath, want) + } +} + +func TestFileThenEnvPrecedence(t *testing.T) { + isolate(t) + dir := t.TempDir() + body := "session_source = \"/from/file.curl\"\nlibrary_path = \"/from/file/lib\"\nconcurrency = 7\n" + if err := os.WriteFile(filepath.Join(dir, "config.toml"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + c, err := config.Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + if c.SessionSource != "/from/file.curl" || c.LibraryPath != "/from/file/lib" || c.Concurrency != 7 { + t.Fatalf("file values not applied: %+v", c) + } + + t.Setenv("UNITY_SYNC_LIBRARY", "/from/env/lib") + t.Setenv("UNITY_SYNC_SESSION", "/from/env.curl") + c, err = config.Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + if c.LibraryPath != "/from/env/lib" { + t.Errorf("LibraryPath = %q, want the env value to beat the file", c.LibraryPath) + } + if c.SessionSource != "/from/env.curl" { + t.Errorf("SessionSource = %q, want the env value to beat the file", c.SessionSource) + } + if c.Concurrency != 7 { + t.Errorf("Concurrency = %d, want the file value to survive (no env override exists)", c.Concurrency) + } +} + +func TestTildeExpands(t *testing.T) { + isolate(t) + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.toml"), + []byte("library_path = \"~/packages\"\n"), 0o600); err != nil { + t.Fatal(err) + } + c, err := config.Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + if want := filepath.Join(os.Getenv("HOME"), "packages"); c.LibraryPath != want { + t.Errorf("LibraryPath = %q, want %q", c.LibraryPath, want) + } +} + +func TestMalformedConfigIsAnError(t *testing.T) { + isolate(t) + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.toml"), []byte("concurrency = \"two\"\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := config.Load(dir); err == nil { + t.Error("Load accepted a malformed config.toml") + } +} diff --git a/internal/fixtures/guard_test.go b/internal/fixtures/guard_test.go new file mode 100644 index 0000000..35ac008 --- /dev/null +++ b/internal/fixtures/guard_test.go @@ -0,0 +1,71 @@ +package fixtures_test + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// The store will return several account-identifying fields if a query asks for them, +// and a capture taken with a wider query would carry them into testdata. The pinned +// query asks for none of them, so their appearance in a committed fixture means either +// the query grew or a fixture was hand-edited from another source. +var forbiddenFields = []string{ + "assignFrom", + "grantTime", + "orderId", + "organizations", + "userOverview", +} + +var forbiddenPatterns = []struct { + name string + re *regexp.Regexp +}{ + {"an email address", regexp.MustCompile(`[\w.+-]+@[\w-]+\.[\w.]+`)}, + {"a session cookie", regexp.MustCompile(`(?i)(__Secure-next-auth|_csrf\s*=|\bLS\s*=)`)}, + // Entitlement ids are 14-digit runs. Product ids are 5-6 digits and version ids 6-7, + // so this cannot collide with the catalogue data the fixtures are for. + {"a 14-digit entitlement id", regexp.MustCompile(`\b\d{14}\b`)}, +} + +// TestCommittedFixturesCarryNoAccountData fails the build rather than the review when +// account data reaches testdata. It walks whatever is committed, so it also covers +// fixtures added later by someone who never read the scrubber. +func TestCommittedFixturesCarryNoAccountData(t *testing.T) { + root := filepath.Join("..", "..", "testdata") + seen := 0 + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + seen++ + raw, err := os.ReadFile(path) + if err != nil { + return err + } + body := string(raw) + for _, field := range forbiddenFields { + if strings.Contains(body, `"`+field+`"`) { + t.Errorf("%s: contains account-identifying field %q", path, field) + } + } + for _, p := range forbiddenPatterns { + if m := p.re.FindString(body); m != "" { + t.Errorf("%s: contains %s", path, p.name) + } + } + return nil + }) + if err != nil { + t.Fatalf("walk testdata: %v", err) + } + if seen == 0 { + t.Fatal("no fixtures found; the guard would pass vacuously") + } +} diff --git a/internal/fixtures/scrub.go b/internal/fixtures/scrub.go new file mode 100644 index 0000000..0c956a8 --- /dev/null +++ b/internal/fixtures/scrub.go @@ -0,0 +1,88 @@ +// Package fixtures turns raw Asset Store captures into the PII-free JSON that the +// offline test suite runs against. Captures are never committed; the scrubbed output +// is. The account-identifying field the store returns is the per-row entitlement id, +// which nothing in unity-sync reads, so the scrub deletes the field outright rather +// than substituting a value — that also keeps a fixture shaped exactly like what the +// pinned query asks for. +package fixtures + +import ( + "bytes" + "encoding/json" + "fmt" +) + +// entitlementField is the per-row id the store returns alongside each product. It +// identifies the grant, not the asset, and no code path in this tool uses it. +const entitlementField = "id" + +// unusedImageFields are image sizes the captures carry but the pinned query does not +// request. Dropping them keeps a fixture shaped exactly like a real response to the +// query the client actually sends. +var unusedImageFields = []string{"icon", "big", "small", "facebook"} + +// Scrub rewrites one captured `searchMyAssets` batch response into fixture form, +// preserving key order-independent structure and stable indentation so the committed +// file diffs cleanly. It fails rather than guessing when the payload is not the batch +// shape the client actually parses. +func Scrub(raw []byte) ([]byte, error) { + var batch []map[string]any + if err := json.Unmarshal(raw, &batch); err != nil { + return nil, fmt.Errorf("capture is not a GraphQL batch array: %w", err) + } + if len(batch) == 0 { + return nil, fmt.Errorf("capture holds no operations") + } + for _, op := range batch { + results, err := resultRows(op) + if err != nil { + return nil, err + } + for _, row := range results { + m, ok := row.(map[string]any) + if !ok { + return nil, fmt.Errorf("result row is %T, want object", row) + } + delete(m, entitlementField) + if product, ok := m["product"].(map[string]any); ok { + if img, ok := product["mainImage"].(map[string]any); ok { + for _, f := range unusedImageFields { + delete(img, f) + } + } + } + } + } + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetIndent("", " ") + if err := enc.Encode(batch); err != nil { + return nil, err + } + return out.Bytes(), nil +} + +// resultRows walks to searchMyAssets.results, tolerating the terminator page whose +// results list is empty but not a payload missing the path entirely. +func resultRows(op map[string]any) ([]any, error) { + data, ok := op["data"].(map[string]any) + if !ok { + return nil, fmt.Errorf("operation has no data object") + } + search, ok := data["searchMyAssets"].(map[string]any) + if !ok { + return nil, fmt.Errorf("operation data has no searchMyAssets object") + } + raw, present := search["results"] + if !present { + return nil, fmt.Errorf("searchMyAssets has no results field") + } + if raw == nil { + return nil, nil + } + rows, ok := raw.([]any) + if !ok { + return nil, fmt.Errorf("searchMyAssets.results is %T, want array", raw) + } + return rows, nil +} diff --git a/internal/fixtures/scrub_test.go b/internal/fixtures/scrub_test.go new file mode 100644 index 0000000..dcddb8b --- /dev/null +++ b/internal/fixtures/scrub_test.go @@ -0,0 +1,64 @@ +package fixtures_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/curbol/unity-sync/internal/fixtures" +) + +const oneRowCapture = `[{"data":{"searchMyAssets":{"total":1,"results":[ + {"id":"20066949412942","product":{"id":"115488","name":"Quick Outline"}} +]}}}]` + +func TestScrubDropsTheEntitlementIdAndKeepsTheProduct(t *testing.T) { + out, err := fixtures.Scrub([]byte(oneRowCapture)) + if err != nil { + t.Fatalf("Scrub: %v", err) + } + if strings.Contains(string(out), "20066949412942") { + t.Error("scrubbed output still carries the entitlement id") + } + + var batch []map[string]any + if err := json.Unmarshal(out, &batch); err != nil { + t.Fatalf("scrubbed output is not valid JSON: %v", err) + } + rows := batch[0]["data"].(map[string]any)["searchMyAssets"].(map[string]any)["results"].([]any) + row := rows[0].(map[string]any) + if _, present := row["id"]; present { + t.Error("row still has an id field") + } + product, ok := row["product"].(map[string]any) + if !ok { + t.Fatal("row lost its product object") + } + if product["id"] != "115488" { + t.Errorf("product id = %v, want 115488 (the product id must survive)", product["id"]) + } +} + +// The terminator page has an empty results list. Scrubbing it must succeed, because it +// is a fixture the pagination tests need. +func TestScrubAcceptsAnEmptyResultsPage(t *testing.T) { + if _, err := fixtures.Scrub([]byte(`[{"data":{"searchMyAssets":{"total":176,"results":[]}}}]`)); err != nil { + t.Fatalf("Scrub on terminator page: %v", err) + } +} + +func TestScrubRefusesPayloadsItDoesNotUnderstand(t *testing.T) { + for name, body := range map[string]string{ + "not an array": `{"data":{}}`, + "empty batch": `[]`, + "no data": `[{"errors":[]}]`, + "no searchMyAssets": `[{"data":{"product":{}}}]`, + "no results field": `[{"data":{"searchMyAssets":{"total":0}}}]`, + } { + t.Run(name, func(t *testing.T) { + if _, err := fixtures.Scrub([]byte(body)); err == nil { + t.Error("Scrub accepted a payload it cannot have scrubbed correctly") + } + }) + } +} diff --git a/internal/lockfile/lockfile.go b/internal/lockfile/lockfile.go new file mode 100644 index 0000000..32e86cc --- /dev/null +++ b/internal/lockfile/lockfile.go @@ -0,0 +1,148 @@ +// Package lockfile reads and writes unity-sync.lock.json, the committed record of what +// an account owns and what is mirrored. It lives beside the project manifest and is +// meant to be diffed: a month's diff should read like a changelog. +// +// Each entry has two halves that are deliberately not mixed. The advertised half is what +// the store currently says and is refreshed every run. The resolution half describes the +// file on disk and is rewritten only when a run actually resolves that asset. +package lockfile + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +// Version records one build. ID is what a diff compares. +type Version struct { + ID string `json:"id"` + Name string `json:"name"` + PublishedDate string `json:"publishedDate,omitempty"` +} + +// Publisher is carried for display and for the cache's vendor directory. +type Publisher struct { + ID string `json:"id"` + Name string `json:"name"` +} + +// Entry is one owned asset. Every owned asset gets one, whether or not it is selected, +// because the file is the record of what is *owned*, not of what happens to be mirrored. +type Entry struct { + // AssetID is the store product id. It is deliberately not called productId: the API + // has a field of that exact name holding a different, unusable value, and this is + // the one place a reader compares the two documents side by side. + AssetID string `json:"assetId"` + + // Advertised half, refreshed on every run for every owned asset. + Name string `json:"name"` + State string `json:"state"` + Publisher Publisher `json:"publisher"` + Version Version `json:"version"` + AdvertisedSize int64 `json:"advertisedSize"` + + // Resolution half, rewritten only when a run resolves this asset. + // + // Tracked means "bytes have been mirrored at some point", not "selected right now": + // an asset stays tracked after it is disabled in the manifest, because the file is + // still there. + Tracked bool `json:"tracked"` + + // ResolvedVersionID is the advertised id the cached file was fetched against, and + // it — not Version.ID — is what classification compares. Keeping them apart is what + // stops a refreshed advertised id from being written next to an older file and + // making it look current forever. + ResolvedVersionID string `json:"resolvedVersionId,omitempty"` + + // DeliveredVersionID is what the stored file's own descriptor says. It can differ + // from the advertised id: some products are advertised at one version and served at + // another, steadily. + DeliveredVersionID string `json:"deliveredVersionId,omitempty"` + + SizeBytes int64 `json:"sizeBytes,omitempty"` + SHA256 string `json:"sha256,omitempty"` + CachePath string `json:"cachePath,omitempty"` + + // DownloadedAt is when bytes were last fetched. An adopted entry leaves it empty: + // the tool found the file, it did not fetch it. + DownloadedAt string `json:"downloadedAt,omitempty"` + + // StoreFilename is what the store called the package. Recorded for reference; it + // never determines a path. + StoreFilename string `json:"storeFilename,omitempty"` +} + +// Lockfile is the whole document. +type Lockfile struct { + Assets map[string]Entry `json:"assets"` +} + +// New returns an empty lockfile. +func New() Lockfile { return Lockfile{Assets: map[string]Entry{}} } + +// Load reads a lockfile, returning an empty one when the path does not exist. +func Load(path string) (Lockfile, error) { + raw, err := os.ReadFile(path) + if os.IsNotExist(err) { + return New(), nil + } + if err != nil { + return Lockfile{}, fmt.Errorf("read lockfile: %w", err) + } + var lf Lockfile + if err := json.Unmarshal(raw, &lf); err != nil { + return Lockfile{}, fmt.Errorf("parse lockfile: %w", err) + } + if lf.Assets == nil { + lf.Assets = map[string]Entry{} + } + return lf, nil +} + +// Save writes the lockfile atomically. encoding/json sorts map keys, so the output is +// stable across runs and a diff shows only what actually changed. +func Save(path string, lf Lockfile) error { + raw, err := json.MarshalIndent(lf, "", " ") + if err != nil { + return err + } + raw = append(raw, '\n') + tmp, err := os.CreateTemp(filepath.Dir(path), ".unity-sync-lock-*") + if err != nil { + return err + } + name := tmp.Name() + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + os.Remove(name) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(name) + return err + } + // CreateTemp makes the file 0600 and the rename carries that over, which would quietly + // strip group and other from a file this design wants committed and hand-edited. + mode := os.FileMode(0o644) + if fi, err := os.Stat(path); err == nil { + mode = fi.Mode().Perm() + } + if err := os.Chmod(name, mode); err != nil { + os.Remove(name) + return err + } + return os.Rename(name, path) +} + +// FindByAssetID returns the entry recorded for a product id, whatever key it sits under. +// Classification uses this rather than the key, so a renamed asset — whose key changes +// by construction — is still recognised as the same thing. +func (lf Lockfile) FindByAssetID(id string) (key string, e Entry, ok bool) { + for k, entry := range lf.Assets { + if entry.AssetID == id { + return k, entry, true + } + } + return "", Entry{}, false +} diff --git a/internal/lockfile/lockfile_test.go b/internal/lockfile/lockfile_test.go new file mode 100644 index 0000000..20e6dd9 --- /dev/null +++ b/internal/lockfile/lockfile_test.go @@ -0,0 +1,198 @@ +package lockfile_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/curbol/unity-sync/internal/lockfile" +) + +func sample() lockfile.Lockfile { + lf := lockfile.New() + lf.Assets["quick-outline-115488"] = lockfile.Entry{ + AssetID: "115488", + Name: "Quick Outline", + State: "published", + Publisher: lockfile.Publisher{ID: "37073", Name: "Chris Nolet"}, + Version: lockfile.Version{ID: "683375", Name: "1.1", PublishedDate: "2022-03-07T16:46:24Z"}, + AdvertisedSize: 33824, + Tracked: true, + ResolvedVersionID: "683375", + DeliveredVersionID: "683375", + SizeBytes: 33822, + SHA256: "abc123", + CachePath: "chris-nolet/quick-outline-115488/quick-outline-115488.unitypackage", + DownloadedAt: "2026-08-22T04:00:00Z", + StoreFilename: "Quick Outline.unitypackage", + } + lf.Assets["unowned-yet-999"] = lockfile.Entry{ + AssetID: "999", + Name: "Never Downloaded", + State: "published", + Version: lockfile.Version{ID: "1", Name: "1.0"}, + AdvertisedSize: 4096, + Tracked: false, + } + return lf +} + +func TestRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "unity-sync.lock.json") + if err := lockfile.Save(path, sample()); err != nil { + t.Fatalf("Save: %v", err) + } + got, err := lockfile.Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + want := sample() + for key, w := range want.Assets { + g, ok := got.Assets[key] + if !ok { + t.Fatalf("entry %q vanished", key) + } + if g != w { + t.Errorf("entry %q round-tripped as %+v, want %+v", key, g, w) + } + } +} + +func TestMissingFileLoadsEmpty(t *testing.T) { + lf, err := lockfile.Load(filepath.Join(t.TempDir(), "absent.json")) + if err != nil { + t.Fatalf("Load of a missing file = %v, want nil", err) + } + if len(lf.Assets) != 0 { + t.Errorf("got %d entries, want none", len(lf.Assets)) + } +} + +func TestSaveIsByteStableAcrossRuns(t *testing.T) { + dir := t.TempDir() + a := filepath.Join(dir, "a.json") + b := filepath.Join(dir, "b.json") + if err := lockfile.Save(a, sample()); err != nil { + t.Fatal(err) + } + if err := lockfile.Save(b, sample()); err != nil { + t.Fatal(err) + } + ra, _ := os.ReadFile(a) + rb, _ := os.ReadFile(b) + if string(ra) != string(rb) { + t.Error("two saves of the same content differ; the diff would be noise") + } +} + +// A run timestamp would dirty the committed file on every no-op run, which is the churn +// that buries the changelog the file exists to be. +func TestNoRunTimestampIsWritten(t *testing.T) { + path := filepath.Join(t.TempDir(), "lock.json") + if err := lockfile.Save(path, sample()); err != nil { + t.Fatal(err) + } + raw, _ := os.ReadFile(path) + for _, forbidden := range []string{"generatedAt", "updatedAt", "syncedAt"} { + if strings.Contains(string(raw), forbidden) { + t.Errorf("lockfile carries a run timestamp %q", forbidden) + } + } +} + +// sizeBytes must be what was received, never the advertised value: they differ by 0-16 +// bytes, and conflating them makes cheap verify fail forever. +func TestAdvertisedAndReceivedSizesAreSeparateFields(t *testing.T) { + path := filepath.Join(t.TempDir(), "lock.json") + if err := lockfile.Save(path, sample()); err != nil { + t.Fatal(err) + } + lf, err := lockfile.Load(path) + if err != nil { + t.Fatal(err) + } + e := lf.Assets["quick-outline-115488"] + if e.AdvertisedSize == e.SizeBytes { + t.Fatal("test fixture no longer distinguishes the two sizes") + } + if e.SizeBytes != 33822 || e.AdvertisedSize != 33824 { + t.Errorf("sizes crossed over: sizeBytes=%d advertisedSize=%d", e.SizeBytes, e.AdvertisedSize) + } +} + +func TestUntrackedEntriesOmitResolutionFields(t *testing.T) { + path := filepath.Join(t.TempDir(), "lock.json") + if err := lockfile.Save(path, sample()); err != nil { + t.Fatal(err) + } + raw, _ := os.ReadFile(path) + // The untracked entry must not carry empty resolution keys; omitempty keeps the + // committed file readable. + chunk := string(raw) + start := strings.Index(chunk, `"unowned-yet-999"`) + if start < 0 { + t.Fatal("untracked entry missing") + } + end := strings.Index(chunk[start:], "}") + block := chunk[start : start+end] + for _, field := range []string{"sha256", "cachePath", "downloadedAt", "resolvedVersionId"} { + if strings.Contains(block, field) { + t.Errorf("untracked entry carries %q", field) + } + } +} + +// A rename changes the key by construction, so lookups that must survive one go by id. +func TestFindByAssetIDIgnoresTheKey(t *testing.T) { + lf := sample() + key, e, ok := lf.FindByAssetID("115488") + if !ok { + t.Fatal("FindByAssetID missed a present entry") + } + if key != "quick-outline-115488" || e.Name != "Quick Outline" { + t.Errorf("FindByAssetID = %q, %+v", key, e) + } + if _, _, ok := lf.FindByAssetID("nope"); ok { + t.Error("FindByAssetID invented an entry") + } +} + +func TestCorruptLockfileIsAnError(t *testing.T) { + path := filepath.Join(t.TempDir(), "lock.json") + os.WriteFile(path, []byte("{not json"), 0o644) + if _, err := lockfile.Load(path); err == nil { + t.Error("Load accepted a corrupt lockfile") + } +} + +// The lockfile is meant to be committed and read by other people and tools. Writing +// through a temp file and renaming would otherwise leave it owner-only. +func TestSaveDoesNotMakeTheLockfileOwnerOnly(t *testing.T) { + path := filepath.Join(t.TempDir(), "unity-sync.lock.json") + lf := lockfile.New() + lf.Assets["a-1"] = lockfile.Entry{AssetID: "1", Name: "A"} + + if err := lockfile.Save(path, lf); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := fi.Mode().Perm(); got != 0o644 { + t.Errorf("a fresh lockfile is mode %04o, want 0644", got) + } + + // An existing file keeps whatever mode the user gave it. + if err := os.Chmod(path, 0o664); err != nil { + t.Fatal(err) + } + if err := lockfile.Save(path, lf); err != nil { + t.Fatal(err) + } + fi, _ = os.Stat(path) + if got := fi.Mode().Perm(); got != 0o664 { + t.Errorf("rewriting reset the mode to %04o, want the 0664 it had", got) + } +} diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go new file mode 100644 index 0000000..714f374 --- /dev/null +++ b/internal/manifest/manifest.go @@ -0,0 +1,194 @@ +// Package manifest is the committed project manifest, unity-sync.toml: the allowlist of +// assets a project draws from. It is discovered by walking up from the working directory +// and lives with the consuming project, not with the tool, and it carries no account +// identity. +// +// Only `select` ever writes it. +package manifest + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/BurntSushi/toml" + "github.com/curbol/unity-sync/internal/model" +) + +// FileName is what Discover looks for. +const FileName = "unity-sync.toml" + +// Entry is one asset's selection state. It keys on ID because that is the only stable +// identity: a publisher rename changes Name, and matching on the name would silently +// reset a selection to disabled. +type Entry struct { + ID string `toml:"id"` + Name string `toml:"name"` + Enabled bool `toml:"enabled"` +} + +// Manifest is the whole file. +type Manifest struct { + Assets []Entry `toml:"asset"` +} + +// Discover walks up from startDir looking for the manifest, returning its path and true +// on the first hit. +func Discover(startDir string) (string, bool) { + dir := startDir + for { + p := filepath.Join(dir, FileName) + if _, err := os.Stat(p); err == nil { + return p, true + } + parent := filepath.Dir(dir) + if parent == dir { + return "", false + } + dir = parent + } +} + +// LockPath is the lockfile that belongs beside a manifest. +func LockPath(manifestPath string) string { + return strings.TrimSuffix(manifestPath, ".toml") + ".lock.json" +} + +// Load reads the manifest, returning an empty one if the file does not exist. An unknown +// key is an error rather than a shrug: Save re-encodes from the struct, so a key that +// decodes to nothing here would be deleted from the user's file on the next write. +func Load(path string) (Manifest, error) { + var m Manifest + if _, err := os.Stat(path); os.IsNotExist(err) { + return m, nil + } + md, err := toml.DecodeFile(path, &m) + if err != nil { + return Manifest{}, err + } + if un := md.Undecoded(); len(un) > 0 { + keys := make([]string, 0, len(un)) + for _, k := range un { + keys = append(keys, k.String()) + } + return Manifest{}, fmt.Errorf("%s: unknown key(s): %s", path, strings.Join(keys, ", ")) + } + for i, e := range m.Assets { + if e.ID == "" { + return Manifest{}, fmt.Errorf("%s: [[asset]] #%d has no id", path, i+1) + } + } + return m, nil +} + +// Save writes the manifest atomically, entries sorted by id for a stable diff. +func Save(path string, m Manifest) error { + // Sort a copy: the value receiver shares the caller's backing array, so sorting in + // place would reorder their slice behind their back. + m.Assets = append([]Entry(nil), m.Assets...) + sort.Slice(m.Assets, func(i, j int) bool { return m.Assets[i].ID < m.Assets[j].ID }) + + tmp, err := os.CreateTemp(filepath.Dir(path), ".unity-sync-manifest-*") + if err != nil { + return err + } + name := tmp.Name() + if err := toml.NewEncoder(tmp).Encode(m); err != nil { + tmp.Close() + os.Remove(name) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(name) + return err + } + // CreateTemp makes the file 0600 and the rename carries that over, which would quietly + // strip group and other from a file this design wants committed and hand-edited. + mode := os.FileMode(0o644) + if fi, err := os.Stat(path); err == nil { + mode = fi.Mode().Perm() + } + if err := os.Chmod(name, mode); err != nil { + os.Remove(name) + return err + } + return os.Rename(name, path) +} + +// ErrWouldEmpty is returned by Reconcile when the owned set is empty but the manifest is +// not. An enumeration that legitimately returns nothing is indistinguishable from one +// made with the wrong org active, and this is the only file in the system a user curates +// by hand. +type ErrWouldEmpty struct{ Existing int } + +func (e *ErrWouldEmpty) Error() string { + return fmt.Sprintf("refusing to rewrite the manifest: the store reported no owned assets while "+ + "the manifest holds %d (a session with a different active org looks exactly like this)", e.Existing) +} + +// Reconcile rebuilds the allowlist against what is currently owned: existing selections +// are preserved by id, newly-owned assets are added disabled, and assets no longer owned +// drop out. It returns the entries it dropped so the caller can report them rather than +// removing them silently. +// +// Selection is opt-in: buying an asset must never cause it to download on the next run. +func (m *Manifest) Reconcile(owned []model.Asset) (dropped []Entry, err error) { + if len(owned) == 0 && len(m.Assets) > 0 { + return nil, &ErrWouldEmpty{Existing: len(m.Assets)} + } + prev := make(map[string]Entry, len(m.Assets)) + for _, e := range m.Assets { + prev[e.ID] = e + } + out := make([]Entry, 0, len(owned)) + stillOwned := make(map[string]bool, len(owned)) + for _, a := range owned { + stillOwned[a.ID] = true + out = append(out, Entry{ID: a.ID, Name: a.Name, Enabled: prev[a.ID].Enabled}) + } + for _, e := range m.Assets { + if !stillOwned[e.ID] { + dropped = append(dropped, e) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + m.Assets = out + return dropped, nil +} + +// EnabledIDs is the set a run may act on. +func (m Manifest) EnabledIDs() map[string]bool { + set := map[string]bool{} + for _, e := range m.Assets { + if e.Enabled { + set[e.ID] = true + } + } + return set +} + +// SetEnabled applies an id->enabled selection, as the select page returns it. +func (m *Manifest) SetEnabled(enabled map[string]bool) { + for i := range m.Assets { + m.Assets[i].Enabled = enabled[m.Assets[i].ID] + } +} + +// UnknownIDs returns manifest entries naming assets the account does not own. They are +// reported rather than ignored: a refunded asset, a typo, or an id invisible to the +// current session's org would otherwise produce complete silence from status and sync. +func (m Manifest) UnknownIDs(owned []model.Asset) []Entry { + ownedIDs := make(map[string]bool, len(owned)) + for _, a := range owned { + ownedIDs[a.ID] = true + } + var unknown []Entry + for _, e := range m.Assets { + if !ownedIDs[e.ID] { + unknown = append(unknown, e) + } + } + return unknown +} diff --git a/internal/manifest/manifest_test.go b/internal/manifest/manifest_test.go new file mode 100644 index 0000000..1d5fbad --- /dev/null +++ b/internal/manifest/manifest_test.go @@ -0,0 +1,199 @@ +package manifest_test + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/curbol/unity-sync/internal/manifest" + "github.com/curbol/unity-sync/internal/model" +) + +func asset(id, name string) model.Asset { return model.Asset{ID: id, Name: name} } + +func TestDiscoverWalksUpFromANestedDirectory(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "src", "game", "assets") + if err := os.MkdirAll(nested, 0o755); err != nil { + t.Fatal(err) + } + want := filepath.Join(root, manifest.FileName) + if err := os.WriteFile(want, []byte("\n"), 0o644); err != nil { + t.Fatal(err) + } + got, ok := manifest.Discover(nested) + if !ok || got != want { + t.Errorf("Discover = %q, %v; want %q", got, ok, want) + } +} + +func TestDiscoverReportsAbsence(t *testing.T) { + if _, ok := manifest.Discover(t.TempDir()); ok { + t.Error("Discover invented a manifest") + } +} + +func TestLockPathSitsBesideTheManifest(t *testing.T) { + if got := manifest.LockPath("/p/unity-sync.toml"); got != "/p/unity-sync.lock.json" { + t.Errorf("LockPath = %q", got) + } +} + +// Save re-encodes from the struct, so a key that silently decodes to nothing would be +// deleted from the user's committed file on the next write. +func TestUnknownKeysAreRejected(t *testing.T) { + path := filepath.Join(t.TempDir(), manifest.FileName) + os.WriteFile(path, []byte("variant_includes = [\"Unity_*\"]\n"), 0o644) + if _, err := manifest.Load(path); err == nil { + t.Error("Load accepted an unknown key it would later delete") + } +} + +func TestEntriesWithoutAnIdAreRejected(t *testing.T) { + path := filepath.Join(t.TempDir(), manifest.FileName) + os.WriteFile(path, []byte("[[asset]]\nname = \"No Id\"\nenabled = true\n"), 0o644) + if _, err := manifest.Load(path); err == nil { + t.Error("Load accepted an entry with no id, which nothing could match") + } +} + +func TestNewlyOwnedAssetsArriveDisabled(t *testing.T) { + var m manifest.Manifest + if _, err := m.Reconcile([]model.Asset{asset("1", "A"), asset("2", "B")}); err != nil { + t.Fatal(err) + } + for _, e := range m.Assets { + if e.Enabled { + t.Errorf("asset %s arrived enabled; buying one must never download it", e.ID) + } + } +} + +// The manifest keys on id precisely so a rename cannot silently reset a selection. +func TestReconcilePreservesSelectionAcrossARename(t *testing.T) { + m := manifest.Manifest{Assets: []manifest.Entry{ + {ID: "309177", Name: "RPG_Animations - One Hand Base", Enabled: true}, + }} + if _, err := m.Reconcile([]model.Asset{asset("309177", "Ultimate One Hand Locomotion")}); err != nil { + t.Fatal(err) + } + if len(m.Assets) != 1 { + t.Fatalf("got %d entries, want 1", len(m.Assets)) + } + if !m.Assets[0].Enabled { + t.Error("a rename reset the selection") + } + if m.Assets[0].Name != "Ultimate One Hand Locomotion" { + t.Errorf("Name = %q, want the refreshed one", m.Assets[0].Name) + } +} + +func TestReconcileReportsDropsRatherThanHidingThem(t *testing.T) { + m := manifest.Manifest{Assets: []manifest.Entry{ + {ID: "1", Name: "Kept", Enabled: true}, + {ID: "2", Name: "Refunded", Enabled: true}, + }} + dropped, err := m.Reconcile([]model.Asset{asset("1", "Kept")}) + if err != nil { + t.Fatal(err) + } + if len(dropped) != 1 || dropped[0].ID != "2" { + t.Errorf("dropped = %+v, want the refunded entry", dropped) + } +} + +// A session with a different active org returns an empty or foreign owned set, and this +// is the one file a user curates by hand. +func TestReconcileRefusesToEmptyANonEmptyManifest(t *testing.T) { + m := manifest.Manifest{Assets: []manifest.Entry{{ID: "1", Name: "A", Enabled: true}}} + _, err := m.Reconcile(nil) + var wouldEmpty *manifest.ErrWouldEmpty + if !errors.As(err, &wouldEmpty) { + t.Fatalf("Reconcile = %v, want ErrWouldEmpty", err) + } + if len(m.Assets) != 1 { + t.Error("the refused reconcile mutated the manifest anyway") + } +} + +func TestEmptyOwnedSetIsFineForAnEmptyManifest(t *testing.T) { + var m manifest.Manifest + if _, err := m.Reconcile(nil); err != nil { + t.Errorf("Reconcile on a first run with nothing owned = %v, want nil", err) + } +} + +func TestUnknownIDsAreSurfaced(t *testing.T) { + m := manifest.Manifest{Assets: []manifest.Entry{ + {ID: "1", Name: "Owned", Enabled: true}, + {ID: "404", Name: "Typo", Enabled: true}, + }} + unknown := m.UnknownIDs([]model.Asset{asset("1", "Owned")}) + if len(unknown) != 1 || unknown[0].ID != "404" { + t.Errorf("UnknownIDs = %+v, want the entry the account does not own", unknown) + } +} + +func TestSaveLoadRoundTripSortsById(t *testing.T) { + path := filepath.Join(t.TempDir(), manifest.FileName) + m := manifest.Manifest{Assets: []manifest.Entry{ + {ID: "222", Name: "B", Enabled: false}, + {ID: "111", Name: "A", Enabled: true}, + }} + if err := manifest.Save(path, m); err != nil { + t.Fatalf("Save: %v", err) + } + got, err := manifest.Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(got.Assets) != 2 || got.Assets[0].ID != "111" { + t.Fatalf("round trip = %+v, want sorted by id", got.Assets) + } + if !got.Assets[0].Enabled || got.Assets[1].Enabled { + t.Errorf("selection flags did not survive: %+v", got.Assets) + } + if ids := got.EnabledIDs(); !ids["111"] || ids["222"] { + t.Errorf("EnabledIDs = %v", ids) + } +} + +func TestSaveDoesNotReorderTheCallersSlice(t *testing.T) { + m := manifest.Manifest{Assets: []manifest.Entry{{ID: "222"}, {ID: "111"}}} + if err := manifest.Save(filepath.Join(t.TempDir(), manifest.FileName), m); err != nil { + t.Fatal(err) + } + if m.Assets[0].ID != "222" { + t.Error("Save sorted the caller's slice in place") + } +} + +// The manifest is committed and hand-edited, so a save must not quietly strip group and +// other from it. +func TestSaveDoesNotMakeTheManifestOwnerOnly(t *testing.T) { + path := filepath.Join(t.TempDir(), manifest.FileName) + m := manifest.Manifest{Assets: []manifest.Entry{{ID: "1", Name: "A", Enabled: true}}} + + if err := manifest.Save(path, m); err != nil { + t.Fatal(err) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := fi.Mode().Perm(); got != 0o644 { + t.Errorf("a fresh manifest is mode %04o, want 0644", got) + } + + if err := os.Chmod(path, 0o664); err != nil { + t.Fatal(err) + } + if err := manifest.Save(path, m); err != nil { + t.Fatal(err) + } + fi, _ = os.Stat(path) + if got := fi.Mode().Perm(); got != 0o664 { + t.Errorf("rewriting reset the mode to %04o, want the 0664 it had", got) + } +} diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 0000000..9aebe15 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,109 @@ +// Package model holds the domain types shared across unity-sync and, more importantly, +// the identity rules everything else depends on: which of the store's several ids is +// the real one, and how an asset's slug is derived from it. +package model + +import ( + "regexp" + "strings" + "unicode" +) + +// State is the store's lifecycle value for a product. Deprecated assets still download; +// disabled ones answer 404, so they are owned but not fetchable. +type State string + +const ( + StatePublished State = "published" + StateDeprecated State = "deprecated" + StateDisabled State = "disabled" +) + +// Downloadable reports whether the store will serve bytes for an asset in this state. +// An unrecognised state is treated as downloadable: the store adding a value must not +// make a whole library unmirrorable, and a wrong guess surfaces as a loud 404. +func (s State) Downloadable() bool { return s != StateDisabled } + +// Publisher is the store's publisher record. Only the id is stable; the name can change +// and is used for display and for the cache's vendor directory. +type Publisher struct { + ID string + Name string +} + +// Version is one published build of an asset. ID is what a diff compares; Name is the +// publisher's display string and is not required to be orderly or even to change. +type Version struct { + ID string + Name string + PublishedDate string +} + +// Asset is one owned Asset Store product as the enumeration reports it. +// +// ID is the *store product id* — the value /api/downloads/{id} takes and the value the +// package's own metadata carries. The API also returns a `productId` field holding a +// different 12-digit number that no endpoint here accepts; this type deliberately does +// not carry it, so it cannot be reached for by mistake. +type Asset struct { + ID string + Name string + State State + Publisher Publisher + Version Version + + // AdvertisedSize is the store's `downloadSize`. It arrives as a JSON string and is + // only approximate — measured deltas run 0-16 bytes above the bytes delivered — so + // it bounds a transfer but never checksums one. + AdvertisedSize int64 + + // ThumbnailURL is protocol-relative as the store returns it (`//host/path`). + ThumbnailURL string +} + +// Slug is the asset's key in the lockfile and its directory name in the cache. The id +// suffix keeps it unique and stable across publisher renames; the name prefix keeps +// lockfile diffs and the cache tree readable. The slug is not the identity — a rename +// changes it — so everything that must survive a rename matches on ID instead. +func (a Asset) Slug() string { + base := slugify(a.Name) + if base == "" { + return a.ID + } + return base + "-" + a.ID +} + +// PublisherSlug is the cache's vendor directory. It carries no id suffix, so the +// directory reads as a name and quarry's vendor facet stays legible. A name written +// entirely in a non-Latin script folds to nothing, and an empty path segment would +// collapse the layout to two components and empty quarry's pack facet, so it falls back +// to the id. +func (a Asset) PublisherSlug() string { + if s := slugify(a.Publisher.Name); s != "" { + return s + } + if a.Publisher.ID != "" { + return "publisher-" + a.Publisher.ID + } + return "publisher-unknown" +} + +var nonSlug = regexp.MustCompile(`[^a-z0-9]+`) + +// slugify lowercases ASCII and turns everything else into a separator, so runs of +// non-ASCII collapse to a single hyphen rather than being transliterated: "Bézier Path +// Creator" becomes "b-zier-path-creator", which is also what the store itself does. A name +// with no ASCII at all therefore slugifies to the empty string, which is why callers need +// a fallback. +func slugify(s string) string { + var b strings.Builder + for _, r := range s { + switch { + case r < unicode.MaxASCII: + b.WriteRune(unicode.ToLower(r)) + default: + b.WriteByte(' ') + } + } + return strings.Trim(nonSlug.ReplaceAllString(b.String(), "-"), "-") +} diff --git a/internal/model/model_test.go b/internal/model/model_test.go new file mode 100644 index 0000000..3a98bdd --- /dev/null +++ b/internal/model/model_test.go @@ -0,0 +1,94 @@ +package model_test + +import ( + "testing" + + "github.com/curbol/unity-sync/internal/model" +) + +func TestSlugMatchesTheStoresOwnSlugForOrdinaryNames(t *testing.T) { + for _, tc := range []struct { + name, id, want string + }{ + {"Quirky Series - Animals Mega Pack Vol.1", "137259", "quirky-series-animals-mega-pack-vol-1-137259"}, + {"Quick Outline", "115488", "quick-outline-115488"}, + {"RPG_Animations - Torch,Lantern,Enemy Attack,Hit,Magic", "309182", + "rpg-animations-torch-lantern-enemy-attack-hit-magic-309182"}, + {"UI Toolkit Bundle 1", "262163", "ui-toolkit-bundle-1-262163"}, + } { + a := model.Asset{ID: tc.id, Name: tc.name} + if got := a.Slug(); got != tc.want { + t.Errorf("Slug(%q) = %q, want %q", tc.name, got, tc.want) + } + } +} + +// A rename changes the slug by construction. The test exists to pin that the id +// survives inside it, since the id is what classification actually matches on. +func TestSlugChangesOnRenameButKeepsTheId(t *testing.T) { + before := model.Asset{ID: "309177", Name: "RPG_Animations - One Hand Base"} + after := model.Asset{ID: "309177", Name: "Ultimate One Hand Locomotion"} + if before.Slug() == after.Slug() { + t.Fatal("a rename must change the slug") + } + for _, s := range []string{before.Slug(), after.Slug()} { + if got := s[len(s)-len("309177"):]; got != "309177" { + t.Errorf("slug %q does not end in the product id", s) + } + } +} + +func TestSlugFallsBackToTheIdWhenTheNameFoldsAway(t *testing.T) { + a := model.Asset{ID: "424242", Name: "日本語アセット"} + if got := a.Slug(); got != "424242" { + t.Errorf("Slug() = %q, want the bare id; an empty slug would be an unsafe path element", got) + } +} + +func TestPublisherSlugCarriesNoIdSuffixButFallsBackToOne(t *testing.T) { + named := model.Asset{Publisher: model.Publisher{ID: "99592", Name: "DoubleL"}} + if got := named.PublisherSlug(); got != "doublel" { + t.Errorf("PublisherSlug() = %q, want %q — the vendor facet should read as a name", got, "doublel") + } + folded := model.Asset{Publisher: model.Publisher{ID: "12345", Name: "Кириллица"}} + if got := folded.PublisherSlug(); got != "publisher-12345" { + t.Errorf("PublisherSlug() = %q, want %q; an empty segment collapses the layout to two "+ + "components and empties quarry's pack facet", got, "publisher-12345") + } +} + +func TestSlugsAreUniquePerProductEvenWhenNamesCollide(t *testing.T) { + a := model.Asset{ID: "111", Name: "Forest Pack"} + b := model.Asset{ID: "222", Name: "Forest Pack!"} + if a.Slug() == b.Slug() { + t.Errorf("distinct products produced the same slug %q", a.Slug()) + } +} + +func TestOnlyDisabledIsUndownloadable(t *testing.T) { + for state, want := range map[model.State]bool{ + model.StatePublished: true, + model.StateDeprecated: true, // deprecated assets still serve bytes + model.StateDisabled: false, + model.State("retired-someday"): true, // an unknown state must not brick a library + } { + if got := state.Downloadable(); got != want { + t.Errorf("State(%q).Downloadable() = %v, want %v", state, got, want) + } + } +} + +// A partially non-ASCII name is the interesting case: the run of non-ASCII becomes one +// separator rather than disappearing, which is also what the store's own slug does. +func TestPartiallyNonASCIINamesCollapseRatherThanTransliterate(t *testing.T) { + a := model.Asset{ID: "136082", Name: "Bézier Path Creator"} + if got, want := a.Slug(), "b-zier-path-creator-136082"; got != want { + t.Errorf("Slug() = %q, want %q", got, want) + } + // Whatever the folding does, the result must stay a single safe path element. + for _, r := range a.Slug() { + if r == '/' || r == '\\' || r < 0x20 { + t.Fatalf("slug %q contains a path-unsafe rune", a.Slug()) + } + } +} diff --git a/internal/retry/retry.go b/internal/retry/retry.go new file mode 100644 index 0000000..9a70b44 --- /dev/null +++ b/internal/retry/retry.go @@ -0,0 +1,83 @@ +// Package retry provides the backoff policy for store requests: retry what a later +// attempt might fix, stop immediately on what it cannot. +package retry + +import ( + "context" + "errors" + "fmt" + "time" +) + +// Policy configures Do. A zero Sleep uses time.Sleep, which tests replace. +type Policy struct { + Attempts int + Base time.Duration + Sleep func(time.Duration) +} + +// DefaultPolicy is used for store API calls. Downloads pass their own, because +// re-transferring a multi-gigabyte body is not the same kind of cheap as re-issuing a +// 2 KB query. +func DefaultPolicy() Policy { return Policy{Attempts: 4, Base: 500 * time.Millisecond} } + +// permanent marks an error that no further attempt can fix. +type permanent struct{ err error } + +func (p permanent) Error() string { return p.err.Error() } +func (p permanent) Unwrap() error { return p.err } + +// Permanent wraps err so Do returns it without retrying. The caller uses this when the +// *body* of a response settles the question that its status code left open — an expired +// session arrives as a 500, which the status alone would say to retry. +func Permanent(err error) error { + if err == nil { + return nil + } + return permanent{err} +} + +// Retryable reports whether an HTTP status is worth another attempt: rate limiting, +// request timeout, and server errors. Every other 4xx is the caller's fault and will +// not improve. +func Retryable(status int) bool { + switch { + case status == 429, status == 408: + return true + case status >= 500: + return true + default: + return false + } +} + +// Do calls fn until it returns nil, returns a Permanent error, or the attempts run out, +// sleeping base<<(n-1) between tries. A cancelled context stops it immediately. +func Do(ctx context.Context, p Policy, fn func(attempt int) error) error { + if p.Attempts < 1 { + p.Attempts = 1 + } + sleep := p.Sleep + if sleep == nil { + sleep = time.Sleep + } + var last error + for attempt := 1; attempt <= p.Attempts; attempt++ { + if err := ctx.Err(); err != nil { + return err + } + last = fn(attempt) + if last == nil { + return nil + } + var perm permanent + if errors.As(last, &perm) { + return perm.err + } + if attempt == p.Attempts { + break + } + sleep(p.Base << (attempt - 1)) + } + return fmt.Errorf("gave up after %d attempts: %w", p.Attempts, last) +} diff --git a/internal/retry/retry_test.go b/internal/retry/retry_test.go new file mode 100644 index 0000000..dd0af33 --- /dev/null +++ b/internal/retry/retry_test.go @@ -0,0 +1,108 @@ +package retry_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/curbol/unity-sync/internal/retry" +) + +// stubClock records what Do would have slept instead of sleeping. +type stubClock struct{ slept []time.Duration } + +func (s *stubClock) sleep(d time.Duration) { s.slept = append(s.slept, d) } + +func policy(attempts int, c *stubClock) retry.Policy { + return retry.Policy{Attempts: attempts, Base: 100 * time.Millisecond, Sleep: c.sleep} +} + +func TestBackoffDoublesAndStopsAtTheAttemptLimit(t *testing.T) { + clock := &stubClock{} + calls := 0 + err := retry.Do(context.Background(), policy(4, clock), func(int) error { + calls++ + return errors.New("boom") + }) + if err == nil { + t.Fatal("Do returned nil after exhausting attempts") + } + if calls != 4 { + t.Errorf("fn called %d times, want 4", calls) + } + want := []time.Duration{100 * time.Millisecond, 200 * time.Millisecond, 400 * time.Millisecond} + if len(clock.slept) != len(want) { + t.Fatalf("slept %v, want %v (no sleep after the final attempt)", clock.slept, want) + } + for i := range want { + if clock.slept[i] != want[i] { + t.Errorf("sleep %d = %v, want %v", i, clock.slept[i], want[i]) + } + } +} + +func TestSuccessOnASecondAttemptSleepsOnce(t *testing.T) { + clock := &stubClock{} + err := retry.Do(context.Background(), policy(4, clock), func(attempt int) error { + if attempt == 1 { + return errors.New("transient") + } + return nil + }) + if err != nil { + t.Fatalf("Do = %v, want nil", err) + } + if len(clock.slept) != 1 { + t.Errorf("slept %v, want exactly one backoff", clock.slept) + } +} + +// The expired-session case: the status says 500, which Retryable would retry, but the +// body settles it. Permanent must short-circuit so the user is told immediately instead +// of after the full schedule. +func TestPermanentStopsImmediatelyAndUnwraps(t *testing.T) { + clock := &stubClock{} + sentinel := errors.New("session expired") + calls := 0 + err := retry.Do(context.Background(), policy(4, clock), func(int) error { + calls++ + return retry.Permanent(sentinel) + }) + if calls != 1 { + t.Errorf("fn called %d times, want 1", calls) + } + if len(clock.slept) != 0 { + t.Errorf("slept %v, want none", clock.slept) + } + if !errors.Is(err, sentinel) { + t.Errorf("Do = %v, want the wrapped sentinel to survive errors.Is", err) + } +} + +func TestCancelledContextStopsBeforeCalling(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + calls := 0 + err := retry.Do(ctx, policy(4, &stubClock{}), func(int) error { + calls++ + return errors.New("should not run") + }) + if calls != 0 { + t.Errorf("fn called %d times on a cancelled context, want 0", calls) + } + if !errors.Is(err, context.Canceled) { + t.Errorf("Do = %v, want context.Canceled", err) + } +} + +func TestRetryableCoversRateLimitsAndServerErrorsOnly(t *testing.T) { + for status, want := range map[int]bool{ + 200: false, 302: false, 400: false, 401: false, 403: false, 404: false, + 408: true, 429: true, 500: true, 502: true, 503: true, + } { + if got := retry.Retryable(status); got != want { + t.Errorf("Retryable(%d) = %v, want %v", status, got, want) + } + } +} diff --git a/internal/selfupdate/audit_test.go b/internal/selfupdate/audit_test.go new file mode 100644 index 0000000..547ee04 --- /dev/null +++ b/internal/selfupdate/audit_test.go @@ -0,0 +1,64 @@ +package selfupdate_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/curbol/unity-sync/internal/selfupdate" +) + +// Failure models this package must keep pinned. + +// The private-repo asset API answers with a 302 to a signed CDN URL, so the test server +// really redirects. A client that inherited the Asset Store's redirect ban fails here. +func TestDownloadFollowsTheAssetRedirect(t *testing.T) { + archive := zipWithBinary(t, "#!/bin/true\n") + var cdn *httptest.Server + cdn = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.Write(archive) + })) + defer cdn.Close() + + var api *httptest.Server + api = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/releases/latest"): + name, err := selfupdate.PlatformAsset("9.9.9") + if err != nil { + t.Fatal(err) + } + json.NewEncoder(w).Encode(map[string]any{ + "tag_name": "v9.9.9", + "assets": []map[string]string{ + {"name": "unrelated.txt", "url": api.URL + "/assets/1"}, + {"name": name, "url": api.URL + "/assets/2"}, + }, + }) + case r.URL.Path == "/assets/2": + if got := r.Header.Get("Accept"); got != "application/octet-stream" { + t.Errorf("asset request Accept = %q", got) + } + http.Redirect(w, r, cdn.URL+"/signed", http.StatusFound) + default: + http.NotFound(w, r) + } + })) + defer api.Close() + + c := selfupdate.New(api.URL, "token") + rel, err := c.Resolve("") + if err != nil { + t.Fatalf("Resolve: %v", err) + } + binary, err := c.DownloadBinary(rel) + if err != nil { + t.Fatalf("DownloadBinary: %v — a client with the store's redirect ban fails exactly here", err) + } + if string(binary) != "#!/bin/true\n" { + t.Errorf("binary = %q", binary) + } +} diff --git a/internal/selfupdate/selfupdate.go b/internal/selfupdate/selfupdate.go new file mode 100644 index 0000000..52236ca --- /dev/null +++ b/internal/selfupdate/selfupdate.go @@ -0,0 +1,258 @@ +// Package selfupdate replaces the running binary with a release build from GitHub. +// +// Its HTTP client follows redirects, unlike every client that talks to the Asset Store. +// That is deliberate: the repository is private, so the binary comes from the release +// *asset* API, which answers with a 302 to a signed CDN URL. Inheriting the store's +// redirect ban here would break updates entirely. +package selfupdate + +import ( + "archive/zip" + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" +) + +const repo = "curbol/unity-sync" + +// Client is the GitHub API surface, injectable so tests need no network. +type Client struct { + http *http.Client + apiBase string + token string +} + +// New builds a client. A caller passing an empty base uses api.github.com. +func New(apiBase, token string) *Client { + if apiBase == "" { + apiBase = "https://api.github.com" + } + return &Client{ + // This client follows redirects: the asset endpoint 302s to a signed CDN URL. + http: &http.Client{Timeout: 10 * time.Minute}, + apiBase: strings.TrimSuffix(apiBase, "/"), + token: token, + } +} + +// Token resolves a GitHub credential from the environment, falling back to the gh CLI. +// The repository is private, so an unauthenticated update cannot even list releases. +func Token() string { + for _, k := range []string{"GITHUB_TOKEN", "GH_TOKEN"} { + if v := os.Getenv(k); v != "" { + return v + } + } + out, err := exec.Command("gh", "auth", "token").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) +} + +type release struct { + TagName string `json:"tag_name"` + Assets []struct { + Name string `json:"name"` + URL string `json:"url"` + } `json:"assets"` +} + +// PlatformAsset is the release archive name for the running platform. +func PlatformAsset(version string) (string, error) { + var os_, arch string + switch runtime.GOOS { + case "darwin": + os_ = "mac" + case "linux": + os_ = "linux" + case "windows": + os_ = "win" + default: + return "", fmt.Errorf("unsupported OS %s", runtime.GOOS) + } + switch runtime.GOARCH { + case "amd64": + arch = "intel" + case "arm64": + if os_ == "mac" { + arch = "apple" + } else { + arch = "arm64" + } + default: + return "", fmt.Errorf("unsupported architecture %s", runtime.GOARCH) + } + if os_ == "win" { + return fmt.Sprintf("unity-sync-%s-win.zip", version), nil + } + return fmt.Sprintf("unity-sync-%s-%s-%s.zip", version, os_, arch), nil +} + +func (c *Client) get(url, accept string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", accept) + req.Header.Set("User-Agent", "unity-sync") + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + resp.Body.Close() + return nil, fmt.Errorf("GET %s: status %d: %s", url, resp.StatusCode, strings.TrimSpace(string(body))) + } + return resp, nil +} + +// Resolve finds a release: the latest, or a specific version when one is named. +func (c *Client) Resolve(version string) (release, error) { + url := c.apiBase + "/repos/" + repo + "/releases/latest" + if version != "" { + url = c.apiBase + "/repos/" + repo + "/releases/tags/v" + strings.TrimPrefix(version, "v") + } + resp, err := c.get(url, "application/vnd.github+json") + if err != nil { + return release{}, err + } + defer resp.Body.Close() + var rel release + if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil { + return release{}, fmt.Errorf("decode release: %w", err) + } + if rel.TagName == "" { + return release{}, fmt.Errorf("release has no tag") + } + return rel, nil +} + +// DownloadBinary fetches the platform archive for a release and returns the binary +// inside it. +func (c *Client) DownloadBinary(rel release) ([]byte, error) { + want, err := PlatformAsset(strings.TrimPrefix(rel.TagName, "v")) + if err != nil { + return nil, err + } + var assetURL string + for _, a := range rel.Assets { + if a.Name == want { + assetURL = a.URL + break + } + } + if assetURL == "" { + return nil, fmt.Errorf("release %s has no asset %s", rel.TagName, want) + } + // The asset API answers with a 302 to a signed CDN URL; this client follows it. + resp, err := c.get(assetURL, "application/octet-stream") + if err != nil { + return nil, err + } + defer resp.Body.Close() + archive, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return binaryFromZip(archive) +} + +func binaryFromZip(archive []byte) ([]byte, error) { + zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive))) + if err != nil { + return nil, fmt.Errorf("release asset is not a zip: %w", err) + } + for _, f := range zr.File { + name := filepath.Base(f.Name) + if name != "unity-sync" && name != "unity-sync.exe" { + continue + } + rc, err := f.Open() + if err != nil { + return nil, err + } + defer rc.Close() + return io.ReadAll(rc) + } + return nil, fmt.Errorf("release asset contains no unity-sync binary") +} + +// Replace swaps the running executable for the given bytes, writing beside the target so +// the rename is atomic and cannot leave a half-written binary on PATH. +func Replace(targetPath string, binary []byte) error { + dir := filepath.Dir(targetPath) + tmp, err := os.CreateTemp(dir, ".unity-sync-update-*") + if err != nil { + return err + } + name := tmp.Name() + if _, err := tmp.Write(binary); err != nil { + tmp.Close() + os.Remove(name) + return err + } + if err := tmp.Chmod(0o755); err != nil { + tmp.Close() + os.Remove(name) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(name) + return err + } + if err := os.Rename(name, targetPath); err != nil { + os.Remove(name) + return err + } + return nil +} + +// Run performs an update of the running binary. +func Run(current, version string) error { + if current == "dev" { + return fmt.Errorf("this is a dev build; install a release first") + } + token := Token() + if token == "" { + return fmt.Errorf("no GitHub credential: set GITHUB_TOKEN or run `gh auth login` (the repo is private)") + } + c := New("", token) + rel, err := c.Resolve(version) + if err != nil { + return err + } + target := strings.TrimPrefix(rel.TagName, "v") + if target == current { + fmt.Printf("already on %s\n", current) + return nil + } + binary, err := c.DownloadBinary(rel) + if err != nil { + return err + } + self, err := os.Executable() + if err != nil { + return err + } + if self, err = filepath.EvalSymlinks(self); err != nil { + return err + } + if err := Replace(self, binary); err != nil { + return err + } + fmt.Printf("updated %s -> %s\n", current, target) + return nil +} diff --git a/internal/selfupdate/selfupdate_test.go b/internal/selfupdate/selfupdate_test.go new file mode 100644 index 0000000..bbfbf18 --- /dev/null +++ b/internal/selfupdate/selfupdate_test.go @@ -0,0 +1,115 @@ +package selfupdate_test + +import ( + "archive/zip" + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/curbol/unity-sync/internal/selfupdate" +) + +func zipWithBinary(t *testing.T, body string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + w, err := zw.Create("unity-sync") + if err != nil { + t.Fatal(err) + } + w.Write([]byte(body)) + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestResolveSendsTheTokenAndCanPinAVersion(t *testing.T) { + var seenAuth, seenPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seenAuth = r.Header.Get("Authorization") + seenPath = r.URL.Path + json.NewEncoder(w).Encode(map[string]any{"tag_name": "v1.2.3"}) + })) + defer srv.Close() + + c := selfupdate.New(srv.URL, "secret-token") + if _, err := c.Resolve("1.2.3"); err != nil { + t.Fatalf("Resolve: %v", err) + } + if seenAuth != "Bearer secret-token" { + t.Errorf("Authorization = %q; the repo is private and an unauthenticated call cannot even list releases", seenAuth) + } + if !strings.HasSuffix(seenPath, "/releases/tags/v1.2.3") { + t.Errorf("path = %q, want the pinned tag", seenPath) + } +} + +func TestMissingPlatformAssetIsNamed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{ + "tag_name": "v9.9.9", + "assets": []map[string]string{{"name": "unity-sync-9.9.9-somethingelse.zip", "url": "http://x"}}, + }) + })) + defer srv.Close() + + c := selfupdate.New(srv.URL, "t") + rel, err := c.Resolve("") + if err != nil { + t.Fatal(err) + } + _, err = c.DownloadBinary(rel) + if err == nil || !strings.Contains(err.Error(), "no asset") { + t.Errorf("DownloadBinary = %v, want a complaint naming the missing asset", err) + } +} + +func TestReplaceIsAtomicAndExecutable(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "unity-sync") + if err := os.WriteFile(target, []byte("old binary"), 0o755); err != nil { + t.Fatal(err) + } + if err := selfupdate.Replace(target, []byte("new binary")); err != nil { + t.Fatalf("Replace: %v", err) + } + got, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if string(got) != "new binary" { + t.Errorf("content = %q", got) + } + fi, err := os.Stat(target) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm()&0o111 == 0 { + t.Errorf("mode = %v, want an executable bit", fi.Mode().Perm()) + } + // The swap must not leave scratch files on PATH. + entries, _ := os.ReadDir(dir) + if len(entries) != 1 { + names := []string{} + for _, e := range entries { + names = append(names, e.Name()) + } + t.Errorf("directory holds %v, want just the binary", names) + } +} + +func TestPlatformAssetNamesAreVersioned(t *testing.T) { + got, err := selfupdate.PlatformAsset("1.2.3") + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(got, "unity-sync-1.2.3-") || !strings.HasSuffix(got, ".zip") { + t.Errorf("PlatformAsset = %q", got) + } +} diff --git a/internal/session/audit_test.go b/internal/session/audit_test.go new file mode 100644 index 0000000..84a55c5 --- /dev/null +++ b/internal/session/audit_test.go @@ -0,0 +1,56 @@ +package session_test + +import ( + "errors" + "strings" + "testing" + + "github.com/curbol/unity-sync/internal/session" +) + +// Failure models this package must keep pinned. Each case exists because getting it wrong +// fails silently or misdiagnoses the user. + +// The credential is HttpOnly. A parser that treats every '#' line as a comment drops +// exactly the cookie that authenticates and then reports "no cookies found". +func TestCookiesTxtKeepsHttpOnlyRecords(t *testing.T) { + got, err := session.Resolve(write(t, "cookies.txt", cookiesTxt)) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if !strings.Contains(got, "LS=the-credential") { + t.Errorf("header %q dropped the #HttpOnly_ record", got) + } +} + +// The storefront sets some cookies host-only and others on Domain=.unity.com, and which +// scope LS uses was never established, so both must be accepted. +func TestCookiesTxtAcceptsTheWholeUnityFamilyAndNothingElse(t *testing.T) { + body := "#HttpOnly_assetstore.unity.com\tFALSE\t/\tTRUE\t0\tLS\thost-only\n" + + ".unity.com\tTRUE\t/\tFALSE\t0\twide\tyes\n" + + "evil.com\tFALSE\t/\tFALSE\t0\tleaked\tno\n" + got, err := session.Resolve(write(t, "cookies.txt", body)) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + for _, want := range []string{"LS=host-only", "wide=yes"} { + if !strings.Contains(got, want) { + t.Errorf("header %q is missing %q", got, want) + } + } + if strings.Contains(got, "leaked") { + t.Errorf("header %q carries a cookie from an unrelated site", got) + } +} + +func TestMissingCredentialIsNamedBeforeAnyRequest(t *testing.T) { + body := "assetstore.unity.com\tFALSE\t/\tTRUE\t0\tDS\tabc\n" + _, err := session.Resolve(write(t, "cookies.txt", body)) + var missing *session.ErrNoCredential + if !errors.As(err, &missing) { + t.Fatalf("Resolve = %v, want ErrNoCredential", err) + } + if !strings.Contains(err.Error(), "LS") { + t.Errorf("diagnostic %q does not name the missing cookie", err) + } +} diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 0000000..2cfc052 --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,176 @@ +// Package session turns a saved browser session into the Cookie header the Asset Store +// requires. The credential is the LS cookie: measured against the live store, _csrf plus +// LS alone returns a full owned-asset list, and the NextAuth session token is not +// consulted by either endpoint this tool uses. LS is a session cookie, so no browser +// cookie database ever holds it — which is why the only supported sources are a pasted +// curl command or a cookies.txt export. +package session + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// CredentialCookie is the one cookie the store actually checks. +const CredentialCookie = "LS" + +// cookieDomain is the family whose cookies a browser would send to the storefront. The +// storefront mixes scopes — the same response sets some cookies host-only on +// assetstore.unity.com and others with Domain=.unity.com — and which scope LS uses was +// never established, so the whole family is accepted rather than one host guessed. +const cookieDomain = "unity.com" + +// httpOnlyPrefix marks an HttpOnly record in a Netscape cookies.txt. Exporters write it +// on the line itself, so a parser that treats every '#' line as a comment drops exactly +// the credential cookies. +const httpOnlyPrefix = "#HttpOnly_" + +// ErrNoCredential means the source parsed but carries no LS cookie. It is reported +// before any request goes out, because the store answers a missing LS with an opaque +// HTTP 500 that reads like a server fault. +type ErrNoCredential struct{ Source string } + +func (e *ErrNoCredential) Error() string { + return fmt.Sprintf("session %s has no %s cookie: Unity keeps it in memory only, so a browser "+ + "cookie database never has it — re-copy the session from DevTools (Network > any "+ + "assetstore.unity.com request > Copy as cURL) while signed in", e.Source, CredentialCookie) +} + +// Resolve reads a session file and returns the Cookie header for the store. It also +// asserts the credential is present, whatever the source, so the diagnostic names the +// real problem instead of leaving it to a 500. +func Resolve(path string) (string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + pairs, err := parse(string(raw)) + if err != nil { + return "", fmt.Errorf("%s: %w", path, err) + } + if _, ok := pairs[CredentialCookie]; !ok { + return "", &ErrNoCredential{Source: path} + } + return join(pairs), nil +} + +// Discover looks for a session file in the user config dir, so a first run needs no +// flag once the file is in the obvious place. +func Discover(configDir string) (string, bool) { + for _, name := range []string{"session.curl", "cookies.txt"} { + p := filepath.Join(configDir, name) + if _, err := os.Stat(p); err == nil { + return p, true + } + } + return "", false +} + +func parse(content string) (map[string]string, error) { + if isCurlPaste(content) { + return fromCurl(content) + } + return fromCookiesTxt(content) +} + +// A cookie value can itself contain the opposite quote character, so each outer-quote +// style gets its own pattern; RE2 has no backreferences. +var ( + curlSingle = regexp.MustCompile(`(?i)(?:-H|--header)\s+'Cookie:\s*([^']*)'`) + curlDouble = regexp.MustCompile(`(?i)(?:-H|--header)\s+"Cookie:\s*([^"]*)"`) +) + +// isCurlPaste distinguishes a pasted command from a cookies.txt by structure, not by the +// word "curl" appearing somewhere: exported cookie files often carry a header comment +// mentioning curl, and treating that as a command sends it to a parser that can only +// fail. +func isCurlPaste(content string) bool { + if curlSingle.MatchString(content) || curlDouble.MatchString(content) { + return true + } + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + return strings.HasPrefix(line, "curl ") + } + return false +} + +func fromCurl(content string) (map[string]string, error) { + var header string + switch { + case curlSingle.MatchString(content): + header = curlSingle.FindStringSubmatch(content)[1] + case curlDouble.MatchString(content): + header = curlDouble.FindStringSubmatch(content)[1] + default: + return nil, fmt.Errorf("no Cookie header in the pasted curl command") + } + pairs := map[string]string{} + for _, part := range strings.Split(header, ";") { + name, value, ok := strings.Cut(strings.TrimSpace(part), "=") + if !ok || name == "" { + continue + } + pairs[name] = value + } + if len(pairs) == 0 { + return nil, fmt.Errorf("the pasted curl command's Cookie header is empty") + } + return pairs, nil +} + +func fromCookiesTxt(content string) (map[string]string, error) { + pairs := map[string]string{} + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + // "#HttpOnly_" is a record, not a comment. + line = strings.TrimPrefix(line, httpOnlyPrefix) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + f := strings.Split(line, "\t") + if len(f) < 7 { + continue + } + if !hostMatches(f[0]) { + continue + } + pairs[f[5]] = f[6] + } + if len(pairs) == 0 { + return nil, fmt.Errorf("no %s cookies found (is this a cookies.txt export for the right site?)", cookieDomain) + } + return pairs, nil +} + +func hostMatches(host string) bool { + host = strings.TrimPrefix(strings.TrimSpace(host), ".") + return host == cookieDomain || strings.HasSuffix(host, "."+cookieDomain) +} + +// join renders a deterministic header so two runs with the same session produce +// byte-identical requests. +func join(pairs map[string]string) string { + names := make([]string, 0, len(pairs)) + for n := range pairs { + names = append(names, n) + } + sort.Strings(names) + var b strings.Builder + for i, n := range names { + if i > 0 { + b.WriteString("; ") + } + b.WriteString(n) + b.WriteByte('=') + b.WriteString(pairs[n]) + } + return b.String() +} diff --git a/internal/session/session_test.go b/internal/session/session_test.go new file mode 100644 index 0000000..fa92f67 --- /dev/null +++ b/internal/session/session_test.go @@ -0,0 +1,94 @@ +package session_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/curbol/unity-sync/internal/session" +) + +// Payloads are inlined rather than kept as testdata files: the repo's .gitignore matches +// *.curl and cookies.txt at any depth, so file fixtures would be silently uncommitted +// and CI would fail on missing testdata. +const curlPaste = `curl 'https://assetstore.unity.com/api/graphql/batch' \ + -H 'accept: application/json' \ + -H 'Cookie: _csrf=stale; LS=the-credential; DS=abc; activeOrgId=123' \ + --data-raw '[]'` + +// A real export marks HttpOnly rows with a #HttpOnly_ prefix on the record itself. +const cookiesTxt = "# Netscape HTTP Cookie File\n" + + "# This file was generated by a browser extension\n" + + "#HttpOnly_.unity.com\tTRUE\t/\tTRUE\t0\tLS\tthe-credential\n" + + "assetstore.unity.com\tFALSE\t/\tTRUE\t0\tDS\tabc\n" + + "example.com\tFALSE\t/\tFALSE\t0\tunrelated\tnope\n" + +func write(t *testing.T, name, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func TestResolveReadsAPastedCurlCommand(t *testing.T) { + got, err := session.Resolve(write(t, "session.curl", curlPaste)) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if !strings.Contains(got, "LS=the-credential") { + t.Errorf("header %q lost the credential", got) + } + if !strings.Contains(got, "DS=abc") { + t.Errorf("header %q dropped an ordinary cookie", got) + } +} + +// An export whose header comment mentions curl must still parse as a cookies.txt. +func TestCurlDetectionUsesStructureNotTheWord(t *testing.T) { + body := "# Generated for use with curl\n" + + "#HttpOnly_.unity.com\tTRUE\t/\tTRUE\t0\tLS\tthe-credential\n" + if _, err := session.Resolve(write(t, "cookies.txt", body)); err != nil { + t.Fatalf("Resolve: %v", err) + } +} + +func TestHeaderIsDeterministic(t *testing.T) { + p := write(t, "session.curl", curlPaste) + first, err := session.Resolve(p) + if err != nil { + t.Fatal(err) + } + for range 5 { + again, err := session.Resolve(p) + if err != nil { + t.Fatal(err) + } + if again != first { + t.Fatalf("header varies between reads:\n%q\n%q", first, again) + } + } +} + +func TestDiscoverPrefersSessionCurl(t *testing.T) { + dir := t.TempDir() + if _, ok := session.Discover(dir); ok { + t.Fatal("Discover found a session in an empty dir") + } + if err := os.WriteFile(filepath.Join(dir, "cookies.txt"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + got, ok := session.Discover(dir) + if !ok || filepath.Base(got) != "cookies.txt" { + t.Fatalf("Discover = %q, %v; want the cookies.txt", got, ok) + } + if err := os.WriteFile(filepath.Join(dir, "session.curl"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + got, _ = session.Discover(dir) + if filepath.Base(got) != "session.curl" { + t.Errorf("Discover = %q, want session.curl to win", got) + } +} diff --git a/internal/store/audit_test.go b/internal/store/audit_test.go new file mode 100644 index 0000000..885bc54 --- /dev/null +++ b/internal/store/audit_test.go @@ -0,0 +1,380 @@ +package store_test + +import ( + "context" + "errors" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/curbol/unity-sync/internal/retry" + "github.com/curbol/unity-sync/internal/store" +) + +// Failure models this package must keep pinned: what the client sends, and how it reads a +// response that only looks successful. + +func TestErrorMapping(t *testing.T) { + cases := []struct { + name string + handler http.HandlerFunc + want error + }{ + {"csrf mismatch", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + io.WriteString(w, "csrf token mismatch") + }, store.ErrCSRF}, + {"expired session as an empty GraphqlError", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + io.WriteString(w, `[{"data":null,"errors":[{"errorCode":"GraphqlError","message":""}]}]`) + }, store.ErrExpiredSession}, + {"expired session as a redirect", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", "/errors/unexpected") + w.WriteHeader(http.StatusFound) + }, store.ErrExpiredSession}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, _ := serve(t, csrfRouter(tc.handler)) + c.Bootstrap(context.Background()) + _, err := c.Enumerate(context.Background()) + if !errors.Is(err, tc.want) { + t.Errorf("Enumerate = %v, want %v", err, tc.want) + } + }) + } +} + +// A populated errors array with a 200 must never read as "you own nothing", which on a +// first run would look like a legitimate empty library. +func TestPopulatedErrorsArrayIsNotAnEmptyLibrary(t *testing.T) { + c, _ := serve(t, csrfRouter(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `[{"data":null,"errors":[{"errorCode":"Throttled","message":"slow down"}]}]`) + })) + c.Bootstrap(context.Background()) + assets, err := c.Enumerate(context.Background()) + if err == nil { + t.Fatalf("Enumerate = %d assets, nil error; want an error", len(assets)) + } + if !strings.Contains(err.Error(), "Throttled") { + t.Errorf("error %v does not report what the store said", err) + } +} + +func TestNonJSONSuccessIsAnError(t *testing.T) { + c, _ := serve(t, csrfRouter(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "sign in") + })) + c.Bootstrap(context.Background()) + if _, err := c.Enumerate(context.Background()); err == nil { + t.Error("Enumerate parsed an HTML body as a result set") + } +} + +// Every one of these headers is invisible when missing: the request still succeeds +// against a permissive server, so only an assertion catches an omission. +func TestClientSendsTheHeadersTheStoreNeeds(t *testing.T) { + var got http.Header + var body string + c, _ := serve(t, csrfRouter(func(w http.ResponseWriter, r *http.Request) { + got = r.Header.Clone() + raw, _ := io.ReadAll(r.Body) + body = string(raw) + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `[{"data":{"searchMyAssets":{"total":0,"results":[]}}}]`) + })) + if err := c.Bootstrap(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := c.Enumerate(context.Background()); err != nil { + t.Fatal(err) + } + + for header, want := range map[string]string{ + "X-Requested-With": "XMLHttpRequest", + "Accept-Encoding": "identity", + "User-Agent": "unity-sync/test", + "X-Csrf-Token": "issued-token", + } { + if got.Get(header) != want { + t.Errorf("%s = %q, want %q", header, got.Get(header), want) + } + } + if cookie := got.Get("Cookie"); !strings.Contains(cookie, "_csrf=issued-token") { + t.Errorf("Cookie %q does not carry the token the header claims", cookie) + } + if strings.Count(got.Get("Cookie"), "_csrf=") != 1 { + t.Errorf("Cookie %q has more than one _csrf", got.Get("Cookie")) + } + // Losing this field from the document would break every classification silently. + if !strings.Contains(body, `currentVersion { id name publishedDate }`) { + t.Error("the pinned query no longer requests currentVersion.id") + } +} + +func TestFetchGuardsTheResponseBeforeAnyBytesAreKept(t *testing.T) { + pkg := "\x1f\x8b\x08\x00rest-of-a-package" + cases := []struct { + name string + handler http.HandlerFunc + wantErr error + wantSub string + }{ + {"re-encoded body", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Encoding", "gzip") + io.WriteString(w, pkg) + }, nil, "re-encoded"}, + {"wrong content type", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + io.WriteString(w, "") + }, nil, "Content-Type"}, + {"redirect to sign-in", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", "https://api.unity.com/v1/oauth2/authorize") + w.WriteHeader(http.StatusFound) + }, store.ErrExpiredSession, ""}, + {"pulled asset", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }, store.ErrNotDownloadable, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, _ := serve(t, tc.handler) + _, err := c.Fetch(context.Background(), "115488") + if err == nil { + t.Fatal("Fetch accepted a response it should have refused") + } + if tc.wantErr != nil && !errors.Is(err, tc.wantErr) { + t.Errorf("Fetch = %v, want %v", err, tc.wantErr) + } + if tc.wantSub != "" && !strings.Contains(err.Error(), tc.wantSub) { + t.Errorf("Fetch = %v, want it to mention %q", err, tc.wantSub) + } + }) + } +} + +// The whole point of a response-header timeout is that a slow *body* is legitimate — a +// 23 GB package takes a while — while a server that never answers is not. Against +// kilobyte fixtures the two policies are indistinguishable, so this pins them apart. +func TestSlowBodyIsAllowedButSlowHeadersAreNot(t *testing.T) { + slowBody, _ := serve(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + for range 4 { + time.Sleep(30 * time.Millisecond) + io.WriteString(w, "chunk") + w.(http.Flusher).Flush() + } + }, store.WithResponseHeaderTimeout(50*time.Millisecond)) + dl, err := slowBody.Fetch(context.Background(), "1") + if err != nil { + t.Fatalf("Fetch with a slow body: %v — a whole-request timeout would kill real downloads", err) + } + body, err := io.ReadAll(dl.Body) + dl.Body.Close() + if err != nil { + t.Fatalf("reading a slow body: %v", err) + } + if len(body) != 20 { + t.Errorf("read %d bytes, want 20", len(body)) + } + + slowHeaders, _ := serve(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(300 * time.Millisecond) + w.Header().Set("Content-Type", "application/octet-stream") + }, store.WithResponseHeaderTimeout(50*time.Millisecond)) + if _, err := slowHeaders.Fetch(context.Background(), "1"); err == nil { + t.Error("Fetch waited indefinitely for headers") + } +} + +// The pinned document is the tool's contract with the store, and every field in it is +// load-bearing: losing currentVersion.id would break every classification silently, and +// adding a field back would start requesting account data the tool has no use for. A +// substring check would miss both, so the whole document is golden. +func TestQueryDocumentMatchesItsGoldenCopy(t *testing.T) { + want, err := os.ReadFile(filepath.Join("testdata", "search_query.graphql")) + if err != nil { + t.Fatalf("read golden: %v", err) + } + if store.SearchDocument != string(want) { + t.Errorf("the pinned query document changed.\n--- got ---\n%s\n--- want ---\n%s\n"+ + "If this change is intended, update testdata/search_query.graphql and say why in the "+ + "commit: the field set is a privacy boundary as well as a correctness one.", + store.SearchDocument, want) + } +} + +// The identity encoding matters most on the download endpoint — that is where asking for +// gzip makes the store gzip an already-gzipped package — so the download's headers get +// their own assertion rather than riding on the GraphQL one. +func TestFetchSendsTheHeadersTheDownloadEndpointNeeds(t *testing.T) { + var got http.Header + c, _ := serve(t, func(w http.ResponseWriter, r *http.Request) { + got = r.Header.Clone() + w.Header().Set("Content-Type", "application/octet-stream") + io.WriteString(w, "\x1f\x8b\x08\x04payload") + }) + dl, err := c.Fetch(context.Background(), "115488") + if err != nil { + t.Fatalf("Fetch: %v", err) + } + dl.Body.Close() + + for header, want := range map[string]string{ + "Accept-Encoding": "identity", + "X-Requested-With": "XMLHttpRequest", + "User-Agent": "unity-sync/test", + } { + if got.Get(header) != want { + t.Errorf("%s = %q, want %q", header, got.Get(header), want) + } + } + if !strings.Contains(got.Get("Cookie"), "LS=cred") { + t.Errorf("Cookie %q does not carry the credential", got.Get("Cookie")) + } +} + +// Only the empty-message shape is the session verdict. A 5xx that says what went wrong is +// an ordinary server error, and short-circuiting it would turn a transient outage into an +// immediate failure. +func TestAServerErrorThatExplainsItselfStillRetries(t *testing.T) { + var calls int + c, _ := serve(t, csrfRouter(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.Header().Set("Content-Type", "application/json") + if calls == 1 { + w.WriteHeader(http.StatusInternalServerError) + io.WriteString(w, `[{"data":null,"errors":[{"errorCode":"Backend","message":"upstream timeout"}]}]`) + return + } + io.WriteString(w, `[{"data":{"searchMyAssets":{"total":0,"results":[]}}}]`) + })) + if err := c.Bootstrap(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := c.Enumerate(context.Background()); err != nil { + t.Fatalf("Enumerate = %v, want the second attempt to succeed", err) + } + if calls < 2 { + t.Errorf("made %d calls, want the explained 5xx to be retried", calls) + } +} + +// The token can expire between the bootstrap and the call that uses it, so one mismatch is +// worth a re-bootstrap and a second try. A server that always mismatches still reports +// ErrCSRF, which is what the other test pins; this one pins the recovery. +func TestATransientCSRFMismatchRecoversOnce(t *testing.T) { + var issued, posts int + c, _ := serve(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/packages" { + issued++ + http.SetCookie(w, &http.Cookie{Name: "_csrf", Value: "token", Path: "/"}) + w.WriteHeader(http.StatusNotFound) + return + } + posts++ + if posts == 1 { + w.WriteHeader(http.StatusBadRequest) + io.WriteString(w, "csrf token mismatch") + return + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `[{"data":{"searchMyAssets":{"total":0,"results":[]}}}]`) + }) + if err := c.Bootstrap(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := c.Enumerate(context.Background()); err != nil { + t.Fatalf("Enumerate = %v, want the retry after a re-bootstrap to succeed", err) + } + if issued < 2 { + t.Errorf("bootstrap ran %d times, want a re-bootstrap after the mismatch", issued) + } +} + +// The syncer shares one client across its download pool, and a CSRF mismatch re-bootstraps +// from inside a request, so the credential pair is written while other goroutines read it. +// Without synchronisation this reports a data race under -race; a torn Cookie value on the +// wire would present as an unreproducible mid-sync "session expired". +func TestTheClientIsSafeToShareAcrossTheDownloadPool(t *testing.T) { + c, _ := serve(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/packages": + http.SetCookie(w, &http.Cookie{Name: "_csrf", Value: "token", Path: "/"}) + w.WriteHeader(http.StatusNotFound) + case "/api/graphql/batch": + // Always a mismatch, so every call re-bootstraps mid-flight. + w.WriteHeader(http.StatusBadRequest) + io.WriteString(w, "csrf token mismatch") + default: + w.Header().Set("Content-Type", "application/octet-stream") + io.WriteString(w, "\x1f\x8b\x08\x00payload") + } + }) + if err := c.Bootstrap(context.Background()); err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + for i := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + if i%2 == 0 { + c.Lookup(context.Background(), "1") + return + } + if dl, err := c.Fetch(context.Background(), "1"); err == nil { + io.Copy(io.Discard, dl.Body) + dl.Body.Close() + } + }() + } + wg.Wait() +} + +// A 403 says the same thing on the second attempt, and the download policy's backoff is +// measured in seconds per asset, so the status has to reach the caller as permanent. Run +// through retry.Do, which is the only public way to observe that. +func TestANonRetryableDownloadStatusIsNotRetried(t *testing.T) { + var calls int + c, _ := serve(t, func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(http.StatusForbidden) + }) + policy := retry.Policy{Attempts: 3, Base: time.Millisecond, Sleep: func(time.Duration) {}} + err := retry.Do(context.Background(), policy, func(int) error { + _, err := c.Fetch(context.Background(), "1") + return err + }) + if err == nil { + t.Fatal("Fetch on a 403 = nil, want an error") + } + if calls != 1 { + t.Errorf("the store was called %d times for a 403, want 1", calls) + } + + // A 503 is the opposite case, and proves the test can tell the difference. + calls = 0 + busy, _ := serve(t, func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(http.StatusServiceUnavailable) + }) + retry.Do(context.Background(), policy, func(int) error { + _, err := busy.Fetch(context.Background(), "1") + return err + }) + if calls != 3 { + t.Errorf("a 503 was attempted %d times, want all 3", calls) + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..d1d2f11 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,501 @@ +// Package store is the Asset Store client: the batched GraphQL endpoint that lists what +// an account owns, and the endpoint that serves package bytes. It owns the +// response-level guards — refusing redirects, refusing a re-encoded body, checking the +// content type — and hands the body to the caller unbuffered, because a package here can +// be 23 GB. The checks that need the enumeration metadata belong to the syncer. +package store + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/curbol/unity-sync/internal/model" + "github.com/curbol/unity-sync/internal/retry" +) + +const ( + defaultBase = "https://assetstore.unity.com" + + // csrfRoute issues the _csrf cookie. Not every storefront path does: "/" and + // "/publishers/{id}" answer 200 and set nothing, while this one answers 404 and + // sets the token. The uncached 404 routes are the ones that issue it. + csrfRoute = "/packages" + + graphQLPath = "/api/graphql/batch" + downloadPath = "/api/downloads/" + + pageSize = 100 +) + +var ( + // ErrExpiredSession is the one failure a user can act on. The store signals it two + // different ways: an empty GraphqlError inside an HTTP 500 on the API, and a 302 to + // the OAuth authorize URL on the download endpoint. + ErrExpiredSession = errors.New("session expired or missing; re-copy it from a signed-in browser") + + // ErrCSRF means the double-submit token did not match, i.e. the bootstrap failed. + ErrCSRF = errors.New("csrf token mismatch") + + // ErrNotDownloadable means the store has no bytes for this product any more. It is + // permanent: no re-run changes it. + ErrNotDownloadable = errors.New("asset is not downloadable") +) + +// SearchDocument is pinned. Its field set is the tool's contract with the store: it asks +// for no account-identifying field, and currentVersion.id is mandatory, being the key +// every classification diffs on. +const SearchDocument = `query SearchMyAssets($page: Int, $pageSize: Int, $ids: [String!]) { + searchMyAssets(page: $page, pageSize: $pageSize, ids: $ids) { + total + results { + product { + id + name + state + downloadSize + currentVersion { id name publishedDate } + publisher { id name } + mainImage { icon75 } + } + } + } +}` + +// Client talks to one Asset Store host. +type Client struct { + http *http.Client + base string + agent string + retries retry.Policy + + // A CSRF mismatch re-bootstraps mid-flight, and the syncer's download pool shares one + // client, so these two are written while other goroutines are reading them. + mu sync.RWMutex + cookie string + csrf string +} + +// credentials reads the pair that every request carries. +func (c *Client) credentials() (cookie, csrf string) { + c.mu.RLock() + defer c.mu.RUnlock() + return c.cookie, c.csrf +} + +// adoptCSRF folds a freshly issued token into both the header and the cookie jar. +func (c *Client) adoptCSRF(token string) { + c.mu.Lock() + defer c.mu.Unlock() + c.csrf = token + c.cookie = withCSRF(c.cookie, token) +} + +// Option adjusts a Client for tests. +type Option func(*Client) + +// WithBaseURL points the client at a test server. +func WithBaseURL(u string) Option { return func(c *Client) { c.base = strings.TrimSuffix(u, "/") } } + +// WithRetryPolicy replaces the backoff policy, so tests need not sleep. +func WithRetryPolicy(p retry.Policy) Option { return func(c *Client) { c.retries = p } } + +// WithResponseHeaderTimeout shortens the header deadline so a test can prove the +// difference between bounding the headers and bounding the whole transfer. +func WithResponseHeaderTimeout(d time.Duration) Option { + return func(c *Client) { + c.http.Transport.(*http.Transport).ResponseHeaderTimeout = d + } +} + +// New builds a client for the given session Cookie header. +// +// The transport sets a response-header timeout rather than a whole-request timeout: a +// 23 GB body legitimately takes a long time, and a request deadline would kill it, while +// a server that never answers still needs bounding. +func New(cookieHeader, version string, opts ...Option) *Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ResponseHeaderTimeout = 60 * time.Second + c := &Client{ + http: &http.Client{ + Transport: transport, + // No store request may follow a redirect: an unauthenticated download 302s + // to Unity's OAuth page, and following it would write a sign-in page into + // the cache under a .unitypackage name. + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }, + base: defaultBase, + cookie: cookieHeader, + agent: "unity-sync/" + version, + retries: retry.DefaultPolicy(), + } + for _, o := range opts { + o(c) + } + return c +} + +// Bootstrap obtains the _csrf token the GraphQL endpoint requires and folds it into the +// cookie header. Its own route answers 404 by design, so a non-2xx here is normal and a +// 3xx is not the expired-session signal it is everywhere else — but a response that +// issues no token is a hard failure, since proceeding guarantees ErrCSRF. +func (c *Client) Bootstrap(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+csrfRoute, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", c.agent) + req.Header.Set("Accept", "text/html,*/*") + cookie, _ := c.credentials() + req.Header.Set("Cookie", cookie) + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("csrf bootstrap: %w", err) + } + defer drain(resp) + + for _, ck := range resp.Cookies() { + if ck.Name == "_csrf" && ck.Value != "" { + c.adoptCSRF(ck.Value) + return nil + } + } + return fmt.Errorf("csrf bootstrap: %s issued no _csrf cookie (status %d)", csrfRoute, resp.StatusCode) +} + +// Enumerate walks every page of owned assets. It compares the raw row count against the +// store's own total *before* deduplicating, so a duplicate entitlement row cannot look +// like a short page, then dedups by product id. +func (c *Client) Enumerate(ctx context.Context) ([]model.Asset, error) { + var ( + assets []model.Asset + seen = map[string]bool{} + rawRows int + total = -1 + ) + for page := 0; ; page++ { + res, err := c.search(ctx, map[string]any{"page": page, "pageSize": pageSize}) + if err != nil { + return nil, err + } + total = res.Total + if len(res.Results) == 0 { + break + } + rawRows += len(res.Results) + for _, row := range res.Results { + a, err := row.Product.asset() + if err != nil { + return nil, fmt.Errorf("page %d: %w", page, err) + } + if seen[a.ID] { + continue + } + seen[a.ID] = true + assets = append(assets, a) + } + } + if total >= 0 && rawRows != total { + return nil, fmt.Errorf("enumeration collected %d rows but the store reports %d owned; "+ + "refusing to treat a short walk as the truth", rawRows, total) + } + return assets, nil +} + +// Lookup re-reads one product through the same pinned document, using the ids filter. It +// is the discriminator for a short download: if the advertised version or size has moved +// since enumeration, the publisher republished mid-transfer. +func (c *Client) Lookup(ctx context.Context, id string) (model.Asset, bool, error) { + res, err := c.search(ctx, map[string]any{"page": 0, "pageSize": 1, "ids": []string{id}}) + if err != nil { + return model.Asset{}, false, err + } + for _, row := range res.Results { + if row.Product.ID == id { + a, err := row.Product.asset() + return a, err == nil, err + } + } + return model.Asset{}, false, nil +} + +type searchResult struct { + Total int `json:"total"` + Results []struct { + Product product `json:"product"` + } `json:"results"` +} + +type product struct { + ID string `json:"id"` + Name string `json:"name"` + State string `json:"state"` + DownloadSize string `json:"downloadSize"` + CurrentVersion struct { + ID string `json:"id"` + Name string `json:"name"` + PublishedDate string `json:"publishedDate"` + } `json:"currentVersion"` + Publisher struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"publisher"` + MainImage struct { + Icon75 string `json:"icon75"` + } `json:"mainImage"` +} + +// asset converts a decoded row, refusing anything the diff depends on being absent +// rather than defaulting it. +func (p product) asset() (model.Asset, error) { + if p.ID == "" { + return model.Asset{}, fmt.Errorf("product row has no id") + } + if p.CurrentVersion.ID == "" { + return model.Asset{}, fmt.Errorf("product %s has no currentVersion.id, which is the diff key", p.ID) + } + size, err := strconv.ParseInt(p.DownloadSize, 10, 64) + if err != nil && p.DownloadSize != "" { + return model.Asset{}, fmt.Errorf("product %s has unparseable downloadSize %q: %w", p.ID, p.DownloadSize, err) + } + return model.Asset{ + ID: p.ID, + Name: p.Name, + State: model.State(p.State), + Publisher: model.Publisher{ID: p.Publisher.ID, Name: p.Publisher.Name}, + Version: model.Version{ + ID: p.CurrentVersion.ID, + Name: p.CurrentVersion.Name, + PublishedDate: p.CurrentVersion.PublishedDate, + }, + AdvertisedSize: size, + ThumbnailURL: p.MainImage.Icon75, + }, nil +} + +type graphQLError struct { + ErrorCode string `json:"errorCode"` + Message string `json:"message"` +} + +func (c *Client) search(ctx context.Context, vars map[string]any) (searchResult, error) { + var out searchResult + var csrfRetried bool + err := retry.Do(ctx, c.retries, func(int) error { + res, err := c.searchOnce(ctx, vars) + if err == nil { + out = res + return nil + } + // A stale token is worth exactly one more go: re-bootstrap and retry, since the + // token can expire between the bootstrap and the call that uses it. A second + // mismatch is a real problem and is reported. + if errors.Is(err, ErrCSRF) && !csrfRetried { + csrfRetried = true + if boot := c.Bootstrap(ctx); boot != nil { + return retry.Permanent(err) + } + res, err = c.searchOnce(ctx, vars) + if err == nil { + out = res + return nil + } + } + return err + }) + return out, err +} + +func (c *Client) searchOnce(ctx context.Context, vars map[string]any) (searchResult, error) { + body, err := json.Marshal([]map[string]any{{ + "query": SearchDocument, + "variables": vars, + "operationName": "SearchMyAssets", + }}) + if err != nil { + return searchResult{}, retry.Permanent(err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+graphQLPath, strings.NewReader(string(body))) + if err != nil { + return searchResult{}, retry.Permanent(err) + } + req.Header.Set("User-Agent", c.agent) + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json;charset=UTF-8") + req.Header.Set("Origin", c.base) + req.Header.Set("Referer", c.base+"/") + // Without this header the store answers a failed call with a 302 to an HTML error + // page instead of the JSON error that carries the diagnosis. + req.Header.Set("X-Requested-With", "XMLHttpRequest") + req.Header.Set("X-Source", "storefront") + req.Header.Set("Operations", "SearchMyAssets") + req.Header.Set("Accept-Encoding", "identity") + cookie, csrf := c.credentials() + req.Header.Set("X-Csrf-Token", csrf) + req.Header.Set("Cookie", cookie) + + resp, err := c.http.Do(req) + if err != nil { + return searchResult{}, err + } + defer drain(resp) + + if resp.StatusCode >= 300 && resp.StatusCode < 400 { + return searchResult{}, retry.Permanent(ErrExpiredSession) + } + payload, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) + if err != nil { + return searchResult{}, err + } + if resp.StatusCode == http.StatusBadRequest && strings.Contains(string(payload), "csrf token mismatch") { + return searchResult{}, retry.Permanent(ErrCSRF) + } + + var batch []struct { + Data struct { + Search *searchResult `json:"searchMyAssets"` + } `json:"data"` + Errors []graphQLError `json:"errors"` + } + if err := json.Unmarshal(payload, &batch); err != nil { + if retry.Retryable(resp.StatusCode) { + return searchResult{}, fmt.Errorf("status %d with a non-JSON body", resp.StatusCode) + } + return searchResult{}, retry.Permanent(fmt.Errorf("status %d with a non-JSON body: %w", resp.StatusCode, err)) + } + if len(batch) == 0 { + return searchResult{}, retry.Permanent(fmt.Errorf("empty GraphQL batch response")) + } + op := batch[0] + if len(op.Errors) > 0 { + // A missing or invalid credential arrives here as a 500 whose single error has + // an empty message. Every other error shape is reported as itself, and never as + // "you own nothing". + if resp.StatusCode == http.StatusInternalServerError && op.Errors[0].Message == "" { + return searchResult{}, retry.Permanent(ErrExpiredSession) + } + err := fmt.Errorf("store returned %s: %q", op.Errors[0].ErrorCode, op.Errors[0].Message) + // Only the empty-message shape is the session verdict. A server error that + // bothered to say what went wrong is still a server error, and the ordinary + // backoff applies. + if retry.Retryable(resp.StatusCode) { + return searchResult{}, err + } + return searchResult{}, retry.Permanent(err) + } + if op.Data.Search == nil { + return searchResult{}, retry.Permanent(fmt.Errorf("response carries neither data nor errors")) + } + return *op.Data.Search, nil +} + +// Download is an open package body plus what the store called it. +type Download struct { + Body io.ReadCloser + + // Filename is parsed from Content-Disposition, which the store does not always + // send. It is recorded in the lockfile for reference and never determines a path. + Filename string +} + +// Fetch opens the package stream for one product, applying the response-level guards. +// The caller owns Body and must close it. +func (c *Client) Fetch(ctx context.Context, id string) (*Download, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+downloadPath+id, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", c.agent) + req.Header.Set("Accept", "*/*") + req.Header.Set("Referer", c.base+"/") + req.Header.Set("X-Requested-With", "XMLHttpRequest") + cookie, _ := c.credentials() + req.Header.Set("Cookie", cookie) + // Asking for identity is not hygiene: the endpoint honours Accept-Encoding: gzip by + // gzipping the already-gzipped package, and Go does not transparently decode an + // encoding the caller requested, so the cache would receive a double-gzipped blob + // with no readable metadata. + req.Header.Set("Accept-Encoding", "identity") + + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode >= 300 && resp.StatusCode < 400 { + drain(resp) + return nil, ErrExpiredSession + } + if resp.StatusCode == http.StatusNotFound { + drain(resp) + return nil, ErrNotDownloadable + } + if resp.StatusCode != http.StatusOK { + drain(resp) + err := fmt.Errorf("download %s: status %d", id, resp.StatusCode) + // A 403 or a 400 will say the same thing on the second attempt, and the download + // policy's backoff is measured in seconds per asset. + if !retry.Retryable(resp.StatusCode) { + return nil, retry.Permanent(err) + } + return nil, err + } + if enc := resp.Header.Get("Content-Encoding"); enc != "" { + drain(resp) + return nil, fmt.Errorf("download %s: server re-encoded the body as %q despite identity", id, enc) + } + if err := checkOctetStream(resp.Header.Get("Content-Type")); err != nil { + drain(resp) + return nil, fmt.Errorf("download %s: %w", id, err) + } + return &Download{Body: resp.Body, Filename: dispositionFilename(resp.Header.Get("Content-Disposition"))}, nil +} + +func checkOctetStream(header string) error { + if header == "" { + return fmt.Errorf("response has no Content-Type") + } + mt, _, err := mime.ParseMediaType(header) + if err != nil { + return fmt.Errorf("unparseable Content-Type %q: %w", header, err) + } + if mt != "application/octet-stream" { + return fmt.Errorf("Content-Type is %q, not a package body", mt) + } + return nil +} + +func dispositionFilename(header string) string { + if header == "" { + return "" + } + _, params, err := mime.ParseMediaType(header) + if err != nil { + return "" + } + return params["filename"] +} + +func withCSRF(header, token string) string { + var kept []string + for _, part := range strings.Split(header, ";") { + part = strings.TrimSpace(part) + if part == "" || strings.HasPrefix(part, "_csrf=") { + continue + } + kept = append(kept, part) + } + return strings.Join(append(kept, "_csrf="+token), "; ") +} + +func drain(resp *http.Response) { + io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20)) + resp.Body.Close() +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..e4fd7eb --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,178 @@ +package store_test + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/curbol/unity-sync/internal/retry" + "github.com/curbol/unity-sync/internal/store" +) + +const testCookie = "LS=cred; DS=abc" + +func fastRetries() store.Option { + return store.WithRetryPolicy(retry.Policy{Attempts: 2, Base: time.Millisecond, Sleep: func(time.Duration) {}}) +} + +// serve wires a handler and returns a bootstrapped client pointed at it. +func serve(t *testing.T, h http.HandlerFunc, opts ...store.Option) (*store.Client, *httptest.Server) { + t.Helper() + srv := httptest.NewServer(h) + t.Cleanup(srv.Close) + opts = append([]store.Option{store.WithBaseURL(srv.URL), fastRetries()}, opts...) + return store.New(testCookie, "test", opts...), srv +} + +// csrfRouter answers the bootstrap route the way the real store does — 404 with a +// Set-Cookie — and delegates everything else. +func csrfRouter(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/packages" { + http.SetCookie(w, &http.Cookie{Name: "_csrf", Value: "issued-token", Path: "/"}) + w.WriteHeader(http.StatusNotFound) + return + } + next(w, r) + } +} + +func fixture(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "testdata", "store", name)) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + return string(raw) +} + +func TestBootstrapAcceptsA404AndRequiresAToken(t *testing.T) { + c, _ := serve(t, csrfRouter(func(w http.ResponseWriter, r *http.Request) {})) + if err := c.Bootstrap(context.Background()); err != nil { + t.Fatalf("Bootstrap on a 404-that-sets-the-cookie: %v", err) + } + + // A response that issues no token must fail here rather than guaranteeing ErrCSRF later. + silent, _ := serve(t, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) }) + if err := silent.Bootstrap(context.Background()); err == nil { + t.Error("Bootstrap accepted a response that issued no _csrf cookie") + } +} + +// Everywhere else a 3xx means the session died. The bootstrap route is exempt: it is a +// storefront page, and a locale redirect there says nothing about the session. +func TestBootstrapIsExemptFromTheRedirectRule(t *testing.T) { + c, _ := serve(t, func(w http.ResponseWriter, r *http.Request) { + http.SetCookie(w, &http.Cookie{Name: "_csrf", Value: "issued-token", Path: "/"}) + w.Header().Set("Location", "/elsewhere") + w.WriteHeader(http.StatusFound) + }) + if err := c.Bootstrap(context.Background()); err != nil { + t.Errorf("Bootstrap on a 3xx that still issued a token: %v", err) + } +} + +func TestEnumerateWalksTheRealFixturesAndDedups(t *testing.T) { + pages := []string{ + fixture(t, "my_assets_p0.json"), + fixture(t, "my_assets_p1.json"), + fixture(t, "my_assets_p2.json"), + } + c, _ := serve(t, csrfRouter(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + page := 0 + switch { + case strings.Contains(string(body), `"page":1`): + page = 1 + case strings.Contains(string(body), `"page":2`): + page = 2 + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, pages[page]) + })) + if err := c.Bootstrap(context.Background()); err != nil { + t.Fatal(err) + } + + assets, err := c.Enumerate(context.Background()) + if err != nil { + t.Fatalf("Enumerate: %v", err) + } + if len(assets) != 176 { + t.Fatalf("got %d assets, want 176", len(assets)) + } + seen := map[string]bool{} + for _, a := range assets { + if seen[a.ID] { + t.Errorf("duplicate product id %s", a.ID) + } + seen[a.ID] = true + if a.Version.ID == "" { + t.Errorf("asset %s decoded without a version id", a.ID) + } + } + if !seen["115488"] { + t.Error("known product 115488 missing from the enumeration") + } +} + +func TestEnumerateRefusesAWalkShorterThanTheStoresOwnTotal(t *testing.T) { + c, _ := serve(t, csrfRouter(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + w.Header().Set("Content-Type", "application/json") + if strings.Contains(string(body), `"page":0`) { + io.WriteString(w, `[{"data":{"searchMyAssets":{"total":50,"results":[ + {"product":{"id":"1","name":"A","state":"published","downloadSize":"10", + "currentVersion":{"id":"9","name":"1.0"},"publisher":{"id":"p","name":"P"}}}]}}}]`) + return + } + io.WriteString(w, `[{"data":{"searchMyAssets":{"total":50,"results":[]}}}]`) + })) + c.Bootstrap(context.Background()) + if _, err := c.Enumerate(context.Background()); err == nil { + t.Error("Enumerate accepted 1 row against a reported total of 50") + } +} + +func TestStrictParsingRejectsAMissingVersionId(t *testing.T) { + c, _ := serve(t, csrfRouter(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `[{"data":{"searchMyAssets":{"total":1,"results":[ + {"product":{"id":"1","name":"A","state":"published","downloadSize":"10", + "currentVersion":{"name":"1.0"},"publisher":{"id":"p","name":"P"}}}]}}}]`) + })) + c.Bootstrap(context.Background()) + _, err := c.Enumerate(context.Background()) + if err == nil || !strings.Contains(err.Error(), "currentVersion.id") { + t.Errorf("Enumerate = %v, want a complaint about the missing diff key", err) + } +} + +func TestFetchReturnsTheBodyAndTheStoresFilename(t *testing.T) { + c, _ := serve(t, func(w http.ResponseWriter, r *http.Request) { + if enc := r.Header.Get("Accept-Encoding"); enc != "identity" { + t.Errorf("download sent Accept-Encoding %q; gzip makes the store double-gzip the package", enc) + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", `attachment; filename="Quick Outline.unitypackage"`) + io.WriteString(w, "\x1f\x8b\x08\x04payload") + }) + dl, err := c.Fetch(context.Background(), "115488") + if err != nil { + t.Fatalf("Fetch: %v", err) + } + defer dl.Body.Close() + if dl.Filename != "Quick Outline.unitypackage" { + t.Errorf("Filename = %q", dl.Filename) + } + got, _ := io.ReadAll(dl.Body) + if !strings.HasPrefix(string(got), "\x1f\x8b") { + t.Errorf("body = %q, want the package bytes untouched", got) + } +} diff --git a/internal/store/testdata/search_query.graphql b/internal/store/testdata/search_query.graphql new file mode 100644 index 0000000..6081a0d --- /dev/null +++ b/internal/store/testdata/search_query.graphql @@ -0,0 +1,16 @@ +query SearchMyAssets($page: Int, $pageSize: Int, $ids: [String!]) { + searchMyAssets(page: $page, pageSize: $pageSize, ids: $ids) { + total + results { + product { + id + name + state + downloadSize + currentVersion { id name publishedDate } + publisher { id name } + mainImage { icon75 } + } + } + } +} \ No newline at end of file diff --git a/internal/syncer/audit_test.go b/internal/syncer/audit_test.go new file mode 100644 index 0000000..f44991d --- /dev/null +++ b/internal/syncer/audit_test.go @@ -0,0 +1,771 @@ +package syncer + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/curbol/unity-sync/internal/cache" + "github.com/curbol/unity-sync/internal/lockfile" + "github.com/curbol/unity-sync/internal/model" + "github.com/curbol/unity-sync/internal/retry" + "github.com/curbol/unity-sync/internal/store" +) + +// Failure models this package must keep pinned: the guards between a bad response and a +// committed lockfile entry, and the blast radius of one bad asset. + +func TestSemanticGuardsRejectAndDiscard(t *testing.T) { + good := pkg(t, "1", "v1", 2000) + + cases := []struct { + name string + body []byte + lookup *model.Asset // when set, the re-query reports this + wantOK bool + wantWarn string + wantStore bool + }{ + {name: "not gzip at all", body: []byte("sign in")}, + {name: "descriptor names another product", body: pkg(t, "999", "v1", 2000)}, + {name: "short body, re-query unchanged", body: pkg(t, "1", "v1", 100)}, + { + // A republish is the one legitimate way to fall below the floor. It warns, + // stores nothing, and leaves the new build for the next run. + name: "short body, re-query shows a republish", + body: pkg(t, "1", "v1", 100), + lookup: &model.Asset{ID: "1", Version: model.Version{ID: "v2"}, AdvertisedSize: 4000}, + wantOK: true, + wantWarn: "republished", + wantStore: false, + }, + {name: "20 bytes short is a warning only", body: pkg(t, "1", "v1", 1980), wantOK: true, wantStore: true}, + {name: "exact", body: good, wantOK: true, wantStore: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v1", 2000) + fs := &fakeStore{owned: []model.Asset{a}, bodies: map[string][]byte{"1": tc.body}} + if tc.lookup != nil { + fs.lookups = map[string]model.Asset{"1": *tc.lookup} + } + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + res := rep.Results[0] + final := filepath.Join(root, "pub-one", "asset-1", "asset-1.unitypackage") + if tc.wantOK { + if res.Err != nil { + t.Fatalf("Run rejected an acceptable body: %v", res.Err) + } + if tc.wantWarn != "" && !strings.Contains(res.Warning, tc.wantWarn) { + t.Errorf("warning = %q, want it to mention %q", res.Warning, tc.wantWarn) + } + _, statErr := os.Stat(final) + if tc.wantStore && statErr != nil { + t.Errorf("expected the package to be committed: %v", statErr) + } + if !tc.wantStore && statErr == nil { + t.Error("a republish stored bytes; it should leave the new build for the next run") + } + if leftovers := tempsUnder(t, root); leftovers != 0 { + t.Errorf("%d temp files survived", leftovers) + } + return + } + if res.Err == nil { + t.Fatal("Run accepted a body it should have refused") + } + // Nothing may survive at a real cache path. + if _, err := os.Stat(final); !os.IsNotExist(err) { + t.Error("a rejected body was committed to the cache") + } + if leftovers := tempsUnder(t, root); leftovers != 0 { + t.Errorf("%d temp files survived a rejected download", leftovers) + } + }) + } +} + +// The template returns from Run on the first download error. Departing from that is a +// deliberate rule here, so it needs its own test. +func TestOneFailedAssetDoesNotStopTheRest(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Good one", "v1", 500) + b := asset("2", "Pulled", "v1", 500) + c := asset("3", "Also good", "v1", 500) + fs := &fakeStore{ + owned: []model.Asset{a, b, c}, + bodies: map[string][]byte{"1": pkg(t, "1", "v1", 500), "3": pkg(t, "3", "v1", 500)}, + fetchEr: map[string]error{"2": store.ErrNotDownloadable}, + } + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a, b, c))) + if err != nil { + t.Fatalf("Run: %v", err) + } + tracked := 0 + for _, e := range rep.Lockfile.Assets { + if e.Tracked { + tracked++ + } + } + if tracked != 2 { + t.Errorf("%d assets mirrored, want 2: one failure must not abort the others", tracked) + } + // A pulled asset is permanent, so it must not make every future run exit non-zero. + if rep.Permanent != 1 || rep.Retryable != 0 { + t.Errorf("Permanent=%d Retryable=%d, want 1 and 0", rep.Permanent, rep.Retryable) + } + if rep.Failed() { + t.Error("a permanently pulled asset made the run report failure") + } +} + +func TestConcurrencyCeilingIsHonoured(t *testing.T) { + root, lockPath := newRun(t) + var assets []model.Asset + bodies := map[string][]byte{} + for _, id := range []string{"1", "2", "3", "4", "5", "6"} { + a := asset(id, "Asset "+id, "v1", 500) + assets = append(assets, a) + bodies[id] = pkg(t, id, "v1", 500) + } + fs := &fakeStore{owned: assets, bodies: bodies, hold: 20 * time.Millisecond} + o := opts(root, allSelected(assets...)) + o.Concurrency = 2 + + if _, err := Run(context.Background(), fs, lockfile.New(), lockPath, o); err != nil { + t.Fatalf("Run: %v", err) + } + if got := fs.maxSeen.Load(); got > 2 { + t.Errorf("peak concurrent fetches = %d, want at most 2; the store is someone else's "+ + "infrastructure and this is the brief's one third-party requirement", got) + } + if got := fs.maxSeen.Load(); got < 2 { + t.Errorf("peak concurrent fetches = %d, want the pool actually used its budget", got) + } +} + +func TestDryRunClassifiesAndTouchesNothing(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v1", 500) + + // A stale temp and a package sitting off its derived path: a sync would sweep one + // and relocate the other. + leaf := filepath.Join(root, "pub-one", "elsewhere") + os.MkdirAll(leaf, 0o755) + os.WriteFile(filepath.Join(leaf, ".unity-sync-dl-stale"), []byte("junk"), 0o644) + os.WriteFile(filepath.Join(leaf, "elsewhere.unitypackage"), pkg(t, "1", "v1", 500), 0o644) + old := time.Unix(1600000000, 0) + os.Chtimes(filepath.Join(leaf, ".unity-sync-dl-stale"), old, old) + + before := treeSnapshot(t, root) + + fs := &fakeStore{owned: []model.Asset{a}, bodies: map[string][]byte{"1": pkg(t, "1", "v1", 500)}} + o := opts(root, allSelected(a)) + o.DryRun = true + + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, o) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(rep.Results) != 1 { + t.Fatalf("dry run produced %d results, want 1 — status must classify", len(rep.Results)) + } + if len(fs.fetched) != 0 { + t.Errorf("a dry run downloaded %v", fs.fetched) + } + if _, err := os.Stat(lockPath); !os.IsNotExist(err) { + t.Error("a dry run wrote the lockfile") + } + if after := treeSnapshot(t, root); after != before { + t.Errorf("a dry run changed the library tree:\nbefore %v\nafter %v", before, after) + } +} + +func TestEmptyEnumerationAgainstANonEmptyLockfileIsRefused(t *testing.T) { + root, lockPath := newRun(t) + prior := lockfile.New() + prior.Assets["a-1"] = lockfile.Entry{AssetID: "1"} + if err := lockfile.Save(lockPath, prior); err != nil { + t.Fatal(err) + } + before, _ := os.ReadFile(lockPath) + + fs := &fakeStore{} + _, err := Run(context.Background(), fs, prior, lockPath, opts(root, nil)) + if !errors.Is(err, ErrEmptyLibrary) { + t.Fatalf("Run = %v, want ErrEmptyLibrary", err) + } + after, _ := os.ReadFile(lockPath) + if string(before) != string(after) { + t.Error("the refused run rewrote the lockfile anyway") + } +} + +func TestPreDownloadFailureWritesNoLockfileAtAll(t *testing.T) { + root, lockPath := newRun(t) + fs := &failingEnumerate{} + if _, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, nil)); err == nil { + t.Fatal("Run succeeded despite an enumeration failure") + } + if _, err := os.Stat(lockPath); !os.IsNotExist(err) { + t.Error("a failure before any download still created a lockfile") + } +} + +// The gate the whole adopt design rests on. A file that really is this product, but a +// different build than the store now advertises, must not be adopted: a wrong adopt is +// silent and permanent, where a redundant download is loud and self-correcting. +func TestAdoptRefusesACandidateFromAnotherVersion(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v2", 500) + + // On disk: the right product, stamped with the previous build. + p, err := cache.Store(root, a.PublisherSlug(), a.Slug(), bytes.NewReader(pkg(t, "1", "v1", 500))) + if err != nil { + t.Fatal(err) + } + if err := p.Commit(); err != nil { + t.Fatal(err) + } + + fs := &fakeStore{owned: []model.Asset{a}, bodies: map[string][]byte{"1": pkg(t, "1", "v2", 500)}} + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Class == Adopted { + t.Fatal("adopted a package stamped with a different version than the store advertises") + } + if len(fs.fetched) != 1 { + t.Errorf("fetched %v, want the asset re-downloaded instead of adopted", fs.fetched) + } + _, e, _ := rep.Lockfile.FindByAssetID("1") + if e.DeliveredVersionID != "v2" { + t.Errorf("deliveredVersionId = %q, want the freshly downloaded build", e.DeliveredVersionID) + } +} + +// Adoption exists for a file that is not where the current layout would put it, so the +// relocation half needs its own coverage: live QA cannot reach it, because its subject +// already sits at the derived path where relocation is a deliberate no-op. +func TestAdoptRelocatesACandidateFoundOffTheDerivedPath(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Renamed Asset", "v1", 500) + + // The package sits under a stale slug, as it would after an upstream rename. + stale, err := cache.Store(root, a.PublisherSlug(), "old-slug-1", bytes.NewReader(pkg(t, "1", "v1", 500))) + if err != nil { + t.Fatal(err) + } + if err := stale.Commit(); err != nil { + t.Fatal(err) + } + + fs := &fakeStore{owned: []model.Asset{a}} + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Class != Adopted { + t.Fatalf("class = %v, want Adopted", rep.Results[0].Class) + } + if len(fs.fetched) != 0 { + t.Errorf("adoption downloaded %v", fs.fetched) + } + + derived := cache.RelPath(a.PublisherSlug(), a.Slug()) + _, e, _ := rep.Lockfile.FindByAssetID("1") + if e.CachePath != derived { + t.Errorf("cachePath = %q, want the derived path %q — a legacy path would keep quarry's "+ + "facets on the old name and strand the file at the next version bump", e.CachePath, derived) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(derived))); err != nil { + t.Errorf("the adopted file was not moved: %v", err) + } + if _, err := os.Stat(filepath.Join(root, a.PublisherSlug(), "old-slug-1")); !os.IsNotExist(err) { + t.Error("the emptied source directory survived the adopt") + } +} + +// The one door into the cache that skips every download guard. A truncation or a mid-file +// flip leaves the descriptor intact and can clear the size floor, so an adopt scan that +// considered the file which just failed verification would re-hash the damaged bytes and +// record them as truth. +func TestAFileThatFailedVerificationIsNotAdoptedBackIn(t *testing.T) { + for _, tc := range []struct { + name string + fullVerify bool + damage func(t *testing.T, path string, size int64) + }{ + { + name: "truncated", + damage: func(t *testing.T, path string, size int64) { + if err := os.Truncate(path, size-100); err != nil { + t.Fatal(err) + } + }, + }, + { + name: "mid-file flip under --verify", + fullVerify: true, + damage: func(t *testing.T, path string, _ int64) { + f, err := os.OpenFile(path, os.O_WRONLY, 0o644) + if err != nil { + t.Fatal(err) + } + defer f.Close() + f.WriteAt([]byte{0xFF}, 3000) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v1", 40000) + good := pkg(t, "1", "v1", 40000) + + p, err := cache.Store(root, a.PublisherSlug(), a.Slug(), bytes.NewReader(good)) + if err != nil { + t.Fatal(err) + } + if err := p.Commit(); err != nil { + t.Fatal(err) + } + prior := lockfile.New() + prior.Assets[a.Slug()] = lockfile.Entry{ + AssetID: "1", Name: a.Name, Tracked: true, + ResolvedVersionID: "v1", DeliveredVersionID: "v1", + SizeBytes: p.Size, SHA256: p.SHA256, CachePath: p.RelPath, + Version: lockfile.Version{ID: "v1"}, + } + tc.damage(t, filepath.Join(root, filepath.FromSlash(p.RelPath)), p.Size) + + fs := &fakeStore{owned: []model.Asset{a}, bodies: map[string][]byte{"1": good}} + o := opts(root, allSelected(a)) + o.FullVerify = tc.fullVerify + + rep, err := Run(context.Background(), fs, prior, lockPath, o) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Class == Adopted { + t.Fatal("a damaged file was adopted back in and its digest recorded as truth") + } + if len(fs.fetched) != 1 { + t.Errorf("fetched %v, want the damaged asset re-downloaded", fs.fetched) + } + _, e, _ := rep.Lockfile.FindByAssetID("1") + if e.SizeBytes != int64(len(good)) { + t.Errorf("sizeBytes = %d, want the freshly downloaded %d", e.SizeBytes, len(good)) + } + }) + } +} + +// A rename that also bumps the version downloads to the new derived path, so the prior +// directory would otherwise be left holding a superseded copy of the same asset. +func TestARenameWithAVersionBumpDoesNotStrandTheOldDirectory(t *testing.T) { + root, lockPath := newRun(t) + renamed := asset("1", "Brand New Name", "v2", 500) + + old, err := cache.Store(root, renamed.PublisherSlug(), "old-name-1", bytes.NewReader(pkg(t, "1", "v1", 500))) + if err != nil { + t.Fatal(err) + } + if err := old.Commit(); err != nil { + t.Fatal(err) + } + prior := lockfile.New() + prior.Assets["old-name-1"] = lockfile.Entry{ + AssetID: "1", Name: "Old Name", Tracked: true, + ResolvedVersionID: "v1", DeliveredVersionID: "v1", + SizeBytes: old.Size, SHA256: old.SHA256, CachePath: old.RelPath, + Version: lockfile.Version{ID: "v1"}, + } + + fs := &fakeStore{owned: []model.Asset{renamed}, bodies: map[string][]byte{"1": pkg(t, "1", "v2", 500)}} + rep, err := Run(context.Background(), fs, prior, lockPath, opts(root, allSelected(renamed))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Class != Changed { + t.Fatalf("class = %v, want Changed", rep.Results[0].Class) + } + if _, err := os.Stat(filepath.Join(root, renamed.PublisherSlug(), "old-name-1")); !os.IsNotExist(err) { + t.Error("the superseded copy's directory was left behind after the rename") + } + derived := cache.RelPath(renamed.PublisherSlug(), renamed.Slug()) + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(derived))); err != nil { + t.Errorf("the new build is not at its derived path: %v", err) + } +} + +// Both of these tell the user something the design says they should hear, and neither +// changes any other observable, so only an assertion catches their absence. +func TestDownloadWarnings(t *testing.T) { + t.Run("advertised and delivered versions disagree", func(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v-advertised", 500) + fs := &fakeStore{owned: []model.Asset{a}, + bodies: map[string][]byte{"1": pkg(t, "1", "v-delivered", 500)}} + + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Err != nil { + t.Fatalf("a version mismatch must not fail the asset: %v", rep.Results[0].Err) + } + if !strings.Contains(rep.Results[0].Warning, "v-delivered") { + t.Errorf("warning = %q, want it to name the build actually served", rep.Results[0].Warning) + } + _, e, _ := rep.Lockfile.FindByAssetID("1") + if e.ResolvedVersionID != "v-advertised" || e.DeliveredVersionID != "v-delivered" { + t.Errorf("both ids should be recorded, got resolved=%q delivered=%q", + e.ResolvedVersionID, e.DeliveredVersionID) + } + }) + + t.Run("package carries no descriptor", func(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v1", 500) + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + zw.Write(bytes.Repeat([]byte("x"), 400)) + zw.Close() + body := buf.Bytes() + for len(body) < 500 { + body = append(body, 0) + } + fs := &fakeStore{owned: []model.Asset{a}, bodies: map[string][]byte{"1": body}} + + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Err != nil { + t.Fatalf("a descriptor-less package must still be stored: %v", rep.Results[0].Err) + } + if !strings.Contains(rep.Results[0].Warning, "no store metadata") { + t.Errorf("warning = %q, want it to say later checks fall back to size", rep.Results[0].Warning) + } + _, e, _ := rep.Lockfile.FindByAssetID("1") + if e.DeliveredVersionID != "" { + t.Errorf("deliveredVersionId = %q, want empty", e.DeliveredVersionID) + } + }) + + t.Run("both a version mismatch and a size outside the window", func(t *testing.T) { + root, lockPath := newRun(t) + // 200 short of 4000: past the +-64 window, inside the floor's 500-byte allowance. + a := asset("1", "Asset", "v-advertised", 4000) + fs := &fakeStore{owned: []model.Asset{a}, + bodies: map[string][]byte{"1": pkg(t, "1", "v-delivered", 3800)}} + + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + got := rep.Results[0].Warning + // The size notice must not silence the version one: the lockfile now holds two + // different ids and this line is the only thing that explains why. + if !strings.Contains(got, "v-delivered") { + t.Errorf("warning = %q, want the version mismatch kept alongside the size notice", got) + } + if !strings.Contains(got, "3800") { + t.Errorf("warning = %q, want the size notice too", got) + } + }) + + t.Run("outside the advisory window but above the floor", func(t *testing.T) { + root, lockPath := newRun(t) + // 200 bytes short of 4000: past the +-64 window, inside the floor's 500-byte + // allowance (4000/8). + a := asset("1", "Asset", "v1", 4000) + fs := &fakeStore{owned: []model.Asset{a}, bodies: map[string][]byte{"1": pkg(t, "1", "v1", 3800)}} + + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Err != nil { + t.Fatalf("a body inside the floor must not fail: %v", rep.Results[0].Err) + } + if !strings.Contains(rep.Results[0].Warning, "3800") { + t.Errorf("warning = %q, want it to report the received count", rep.Results[0].Warning) + } + }) +} + +// Retrying either of these wastes a backoff on something a second attempt cannot fix, and +// the waste is invisible: the run still ends the same way, just later. +func TestPermanentDownloadFailuresAreNotRetried(t *testing.T) { + for name, sentinel := range map[string]error{ + "pulled asset": store.ErrNotDownloadable, + "expired session": store.ErrExpiredSession, + } { + t.Run(name, func(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v1", 500) + fs := &fakeStore{owned: []model.Asset{a}, fetchEr: map[string]error{"1": sentinel}} + + o := opts(root, allSelected(a)) + o.Retry = retryPolicyWithAttempts(3) + + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, o) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(fs.fetched) != 1 { + t.Errorf("fetched %d times, want exactly 1: %v cannot be fixed by trying again", + len(fs.fetched), sentinel) + } + if rep.Results[0].Err == nil { + t.Error("the failure was swallowed") + } + }) + } +} + +// retryPolicyWithAttempts gives a test a real attempt budget without real sleeping. +func retryPolicyWithAttempts(n int) retry.Policy { + return retry.Policy{Attempts: n, Base: time.Millisecond, Sleep: func(time.Duration) {}} +} + +// The probes this wraps re-hash whole packages under --verify, so asking twice does not +// merely repeat work, it doubles the cost of verifying the whole library. +func TestMemoizeRunsTheProbeOnce(t *testing.T) { + calls := 0 + probe := memoize(func() bool { + calls++ + return true + }) + for range 5 { + if !probe() { + t.Fatal("memoized probe changed its answer") + } + } + if calls != 1 { + t.Errorf("probe ran %d times, want 1", calls) + } + + calls = 0 + falsey := memoize(func() bool { + calls++ + return false + }) + for range 3 { + if falsey() { + t.Fatal("memoized probe changed its answer") + } + } + if calls != 1 { + t.Errorf("a false result was recomputed %d times, want 1", calls) + } +} + +// The damaged file sits at the derived path, so a good copy found elsewhere has nowhere to +// land unless the damaged one goes first. Getting this wrong is not a bad classification, +// it is a permanent one: nothing resolves, the prior entry carries forward, and every later +// run refuses in exactly the same way. +func TestAGoodCopyReplacesTheDamagedFileHoldingItsPath(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v1", 4000) + good := pkg(t, "1", "v1", 4000) + + derived, err := cache.Store(root, a.PublisherSlug(), a.Slug(), bytes.NewReader(good)) + if err != nil { + t.Fatal(err) + } + if err := derived.Commit(); err != nil { + t.Fatal(err) + } + stray, err := cache.Store(root, "somewhere", "else-1", bytes.NewReader(good)) + if err != nil { + t.Fatal(err) + } + if err := stray.Commit(); err != nil { + t.Fatal(err) + } + + // Truncating leaves the descriptor intact, so the scan still sees a candidate; only the + // recorded size no longer matches, which is what fails verification. + full := filepath.Join(root, filepath.FromSlash(derived.RelPath)) + if err := os.Truncate(full, derived.Size-100); err != nil { + t.Fatal(err) + } + + prior := lockfile.New() + prior.Assets[a.Slug()] = lockfile.Entry{ + AssetID: "1", Name: a.Name, Tracked: true, + ResolvedVersionID: "v1", DeliveredVersionID: "v1", + SizeBytes: derived.Size, SHA256: derived.SHA256, CachePath: derived.RelPath, + Version: lockfile.Version{ID: "v1"}, + } + + fs := &fakeStore{owned: []model.Asset{a}} + rep, err := Run(context.Background(), fs, prior, lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Err != nil { + t.Fatalf("adoption failed: %v", rep.Results[0].Err) + } + if rep.Results[0].Class != Adopted { + t.Fatalf("class = %v, want Adopted", rep.Results[0].Class) + } + if len(fs.fetched) != 0 { + t.Errorf("downloaded %v; a good copy was already on disk", fs.fetched) + } + if got := cache.VerifyDeep(root, derived.RelPath, stray.SHA256); !got { + t.Error("the derived path does not hold the good copy's bytes") + } + e, ok := rep.Lockfile.Assets[a.Slug()] + if !ok || e.CachePath != derived.RelPath || e.SHA256 != stray.SHA256 { + t.Errorf("lockfile records %+v, want the adopted copy at the derived path", e) + } +} + +// The candidate can already be at the derived path while the lockfile still points at the +// old one — a run killed between the commit and the incremental save leaves exactly that. +// Adoption then relocates nothing, so without an explicit removal the recorded copy is +// orphaned: unreferenced by the new lockfile and named by nothing in the summary. +func TestAdoptionRemovesTheEntrysOwnSupersededCopy(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "New Name", "v1", 4000) + body := pkg(t, "1", "v1", 4000) + + atDerived, err := cache.Store(root, a.PublisherSlug(), a.Slug(), bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if err := atDerived.Commit(); err != nil { + t.Fatal(err) + } + stale, err := cache.Store(root, a.PublisherSlug(), "old-name-1", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if err := stale.Commit(); err != nil { + t.Fatal(err) + } + // Truncated, so the recorded copy fails verification and adoption is what runs. + if err := os.Truncate(filepath.Join(root, filepath.FromSlash(stale.RelPath)), stale.Size-100); err != nil { + t.Fatal(err) + } + + prior := lockfile.New() + prior.Assets["old-name-1"] = lockfile.Entry{ + AssetID: "1", Name: "Old Name", Tracked: true, + ResolvedVersionID: "v1", DeliveredVersionID: "v1", + SizeBytes: stale.Size, SHA256: stale.SHA256, CachePath: stale.RelPath, + Version: lockfile.Version{ID: "v1"}, + } + + fs := &fakeStore{owned: []model.Asset{a}} + rep, err := Run(context.Background(), fs, prior, lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Class != Adopted || rep.Results[0].Err != nil { + t.Fatalf("class = %v, err = %v; want a clean Adopted", rep.Results[0].Class, rep.Results[0].Err) + } + if len(fs.fetched) != 0 { + t.Errorf("downloaded %v; the current build was already at the derived path", fs.fetched) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(stale.RelPath))); !os.IsNotExist(err) { + t.Errorf("the superseded copy at %s survived adoption", stale.RelPath) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(atDerived.RelPath))); err != nil { + t.Errorf("the adopted copy is gone: %v", err) + } +} + +// Two runs that find the same droppped assets must print them the same way; ranging a map +// does not. +func TestDroppedAssetsAreReportedInAStableOrder(t *testing.T) { + root, lockPath := newRun(t) + kept := asset("1", "Kept", "v1", 500) + + prior := lockfile.New() + for _, e := range []struct{ id, name string }{ + {"7", "Zulu"}, {"8", "Alpha"}, {"9", "Mike"}, {"10", "Bravo"}, + } { + prior.Assets[e.name] = lockfile.Entry{AssetID: e.id, Name: e.name, Tracked: true} + } + prior.Assets[kept.Slug()] = lockfile.Entry{AssetID: "1", Name: "Kept", Tracked: false} + + var first []string + for run := range 6 { + fs := &fakeStore{owned: []model.Asset{kept}} + rep, err := Run(context.Background(), fs, prior, lockPath, opts(root, allSelected(kept))) + if err != nil { + t.Fatal(err) + } + var names []string + for _, e := range rep.Removed { + names = append(names, e.Name) + } + if run == 0 { + first = names + if want := []string{"Alpha", "Bravo", "Mike", "Zulu"}; !slices.Equal(names, want) { + t.Fatalf("order = %v, want %v", names, want) + } + continue + } + if !slices.Equal(names, first) { + t.Fatalf("run %d reported %v, run 0 reported %v", run, names, first) + } + } +} + +// The dry-run test proves the sweep is gated; nothing proved it happens. Deleting the +// SweepTemps call from Run left the whole suite green. +func TestARealRunSweepsAbandonedTemps(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v1", 500) + + leaf := filepath.Join(root, "pub-one", "asset-1") + if err := os.MkdirAll(leaf, 0o755); err != nil { + t.Fatal(err) + } + stale := filepath.Join(leaf, ".unity-sync-dl-stale") + if err := os.WriteFile(stale, bytes.Repeat([]byte("x"), 100), 0o644); err != nil { + t.Fatal(err) + } + // Older than the run start, which opts() pins to 2023-11-14. + old := time.Unix(1600000000, 0) + if err := os.Chtimes(stale, old, old); err != nil { + t.Fatal(err) + } + // One the sweep must spare: a concurrent run's transfer, still in flight. + live := filepath.Join(leaf, ".unity-sync-dl-live") + if err := os.WriteFile(live, []byte("in flight"), 0o644); err != nil { + t.Fatal(err) + } + + fs := &fakeStore{owned: []model.Asset{a}, bodies: map[string][]byte{"1": pkg(t, "1", "v1", 500)}} + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Swept != 1 { + t.Errorf("Swept = %d, want 1", rep.Swept) + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Error("the abandoned temp survived the run") + } + if _, err := os.Stat(live); err != nil { + t.Errorf("the run swept a temp newer than its own start: %v", err) + } +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go new file mode 100644 index 0000000..880cfb3 --- /dev/null +++ b/internal/syncer/syncer.go @@ -0,0 +1,719 @@ +// Package syncer orchestrates a run: enumerate, classify, download the delta, and +// rewrite the lockfile. It owns the checks that need both the stored bytes and the +// enumeration metadata — the store layer sees only responses, the cache layer only bytes. +package syncer + +import ( + "context" + "errors" + "fmt" + "io" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/curbol/unity-sync/internal/cache" + "github.com/curbol/unity-sync/internal/lockfile" + "github.com/curbol/unity-sync/internal/manifest" + "github.com/curbol/unity-sync/internal/model" + "github.com/curbol/unity-sync/internal/retry" + "github.com/curbol/unity-sync/internal/store" + "github.com/curbol/unity-sync/internal/unitypackage" +) + +// ErrEmptyLibrary guards the committed record against a well-formed but wrong +// enumeration. A session whose active org differs returns a legitimately different owned +// set, and the lockfile lives in someone's project. +var ErrEmptyLibrary = errors.New("the store reported no owned assets while the lockfile holds entries; " + + "refusing to treat that as the truth (check which Unity organisation the session belongs to)") + +// Class is one asset's outcome for this run. +type Class int + +const ( + Unchanged Class = iota + New + Changed + DownloadNow // owned, previously recorded but never mirrored, now selected + CacheMissing + Adopted + Undownloadable +) + +func (c Class) String() string { + switch c { + case New: + return "new" + case Changed: + return "changed" + case DownloadNow: + return "download-now" + case CacheMissing: + return "cache-missing" + case Adopted: + return "adopted" + case Undownloadable: + return "undownloadable" + default: + return "unchanged" + } +} + +// NeedsFetch reports whether a class means bytes must come off the network. +func (c Class) NeedsFetch() bool { + switch c { + case New, Changed, DownloadNow, CacheMissing: + return true + } + return false +} + +// classify is pure. The probes are injected so it stays that way: cacheOK is the cheap +// on-disk check for the prior resolution, and adoptable reports a matching file found by +// scanning. +func classify(a model.Asset, prior lockfile.Entry, hasPrior bool, cacheOK, adoptable func() bool) Class { + resolved := hasPrior && prior.Tracked + + if !a.State.Downloadable() { + // A copy already mirrored stays usable after the store delists the asset; only + // one we do not have is a problem worth reporting. + if resolved && cacheOK() { + return Unchanged + } + return Undownloadable + } + if !resolved { + if adoptable() { + return Adopted + } + if hasPrior { + return DownloadNow + } + return New + } + if prior.ResolvedVersionID != a.Version.ID { + return Changed + } + if !cacheOK() { + if adoptable() { + return Adopted + } + return CacheMissing + } + return Unchanged +} + +// Store is the part of the Asset Store client a run needs. +type Store interface { + Enumerate(ctx context.Context) ([]model.Asset, error) + Lookup(ctx context.Context, id string) (model.Asset, bool, error) + Fetch(ctx context.Context, id string) (*store.Download, error) +} + +// Options configures a run. +type Options struct { + LibraryRoot string + Selected map[string]bool + OnlyGlob string + DryRun bool + FullVerify bool + Concurrency int + Now func() time.Time + + // Progress is called from every download goroutine as well as the main pass, so an + // implementation that does more than one write needs its own synchronisation. + Progress func(string) + + // Retry governs download attempts. Downloads get their own budget rather than the + // API's: re-transferring a multi-gigabyte body is not the same kind of cheap as + // re-issuing a 2 KB query. + Retry retry.Policy + + // Manifest is consulted for reporting only; a run never writes it. + Manifest manifest.Manifest +} + +// Result is one asset's outcome. +type Result struct { + Asset model.Asset + Class Class + Err error + Warning string +} + +// Report is what a run produced. +type Report struct { + // Owned is every asset the account holds, not only the selected ones, so a run with an + // empty allowlist can still say what there is to choose from. + Owned int + + Results []Result + Removed []lockfile.Entry + Unknown []manifest.Entry + Swept int + Lockfile lockfile.Lockfile + + // Retryable counts failures a later run might fix. A permanently gone asset is + // reported but does not make the run exit non-zero, or one dead asset would fail + // every future run forever. + Retryable int + Permanent int +} + +// Failed reports whether the run should exit non-zero. +func (r Report) Failed() bool { return r.Retryable > 0 } + +// resolution is what a successful fetch or adoption produced. +type resolution struct { + cachePath string + sha string + size int64 + resolvedVersionID string + deliveredVersionID string + downloadedAt string + storeFilename string +} + +// Run executes a sync, or a status when DryRun. It returns the report even alongside an +// error, so a caller can show what did happen. +func Run(ctx context.Context, s Store, prior lockfile.Lockfile, lockPath string, opts Options) (Report, error) { + if opts.Now == nil { + opts.Now = time.Now + } + if opts.Progress == nil { + opts.Progress = func(string) {} + } + if opts.Concurrency < 1 { + opts.Concurrency = 1 + } + if opts.Retry.Attempts < 1 { + opts.Retry = retry.Policy{Attempts: 2, Base: 2 * time.Second} + } + if opts.OnlyGlob != "" { + if _, err := filepath.Match(opts.OnlyGlob, ""); err != nil { + return Report{}, fmt.Errorf("bad --only pattern %q: %w", opts.OnlyGlob, err) + } + } + started := opts.Now() + + opts.Progress("enumerating owned assets…") + owned, err := s.Enumerate(ctx) + if err != nil { + return Report{}, err + } + if len(owned) == 0 && len(prior.Assets) > 0 { + return Report{}, ErrEmptyLibrary + } + + report := Report{Owned: len(owned), Unknown: opts.Manifest.UnknownIDs(owned)} + + // Sweeping before classification matters: an abandoned partial left in the tree is + // otherwise a candidate the adopt scan could reach. + if !opts.DryRun { + n, freed := cache.SweepTemps(opts.LibraryRoot, started) + report.Swept = n + if n > 0 { + opts.Progress(fmt.Sprintf("reclaimed %d abandoned download(s), %s", n, humanBytes(freed))) + } + } + + resolutions := map[string]resolution{} + // priorPaths remembers where each asset's bytes used to live, so a download that lands + // somewhere else can clean up after itself. + priorPaths := map[string]string{} + var mu sync.Mutex + + // Classify everything selected, then fetch what needs fetching. + var pending []Result + for _, a := range owned { + if !selected(a, opts) { + continue + } + _, prev, hasPrev := prior.FindByAssetID(a.ID) + derived := cache.RelPath(a.PublisherSlug(), a.Slug()) + + // Memoized: classify calls this, and so does the excludeRel decision below. Under + // --verify each call is a full re-hash, so letting it run twice would double the + // cost of verifying a 75 GB library. + cacheOK := memoize(func() bool { + switch { + case prev.CachePath == "": + return false + case opts.FullVerify: + return cache.VerifyDeep(opts.LibraryRoot, prev.CachePath, prev.SHA256) + default: + return cache.Verify(opts.LibraryRoot, prev.CachePath, prev.SizeBytes, prev.DeliveredVersionID) + } + }) + var found cache.Candidate + var foundOK bool + // excludeRel is set when a recorded file exists but failed verification. Adoption + // must not reach for that same file: a truncation or a mid-file flip leaves the + // descriptor intact and can clear the size floor, so the scan would re-adopt the + // damaged bytes and record them as truth — the outcome every other guard exists to + // prevent, arriving through the one door that skips them. + var excludeRel string + adoptable := func() bool { + // Decided here rather than before classify: classify short-circuits to + // Changed without ever asking, and under --verify the probe is a full + // re-hash of a file that is about to be replaced by the download. + if hasPrev && prev.Tracked && prev.CachePath != "" && !cacheOK() { + excludeRel = prev.CachePath + } + found, foundOK = cache.Locate(opts.LibraryRoot, a.ID, derived, excludeRel) + if !foundOK { + return false + } + // The same floor a download must clear. Without it, a truncated package + // left in the library enters through the one door that skips the download + // path and is then hashed and recorded as truth. + if belowFloor(found.Size, a.AdvertisedSize) { + foundOK = false + return false + } + return found.Metadata.VersionID == a.Version.ID + } + + if hasPrev && prev.Tracked { + priorPaths[a.ID] = prev.CachePath + } + class := classify(a, prev, hasPrev, cacheOK, adoptable) + res := Result{Asset: a, Class: class} + + switch class { + case Adopted: + if opts.DryRun { + break + } + r, err := adopt(opts, a, found, derived, excludeRel) + if err != nil { + res.Err = err + report.Retryable++ + } else { + // Same rule the download branch applies: the entry's own prior copy is + // superseded by what just landed at the derived path, and nothing else + // will ever mention it again — the summary names only assets that left + // the account, so an orphan here is one the tool made and never reports. + if old := prev.CachePath; old != "" && old != r.cachePath { + if rmErr := cache.RemoveStale(opts.LibraryRoot, old); rmErr != nil { + res.Warning = fmt.Sprintf( + "could not remove the superseded copy at %s: %v", old, rmErr) + } + } + resolutions[a.ID] = r + } + case Unchanged: + // Keep the prior resolution, but move it if the slug changed under it. + if !opts.DryRun && hasPrev && prev.Tracked && prev.CachePath != derived { + if err := cache.Relocate(opts.LibraryRoot, prev.CachePath, derived); err != nil { + res.Warning = err.Error() + } else { + r := fromEntry(prev) + r.cachePath = derived + resolutions[a.ID] = r + } + } + } + if class.NeedsFetch() { + pending = append(pending, res) + continue + } + report.Results = append(report.Results, res) + } + + if opts.DryRun { + report.Results = append(report.Results, pending...) + report.Lockfile = build(owned, prior, resolutions, &report) + return report, nil + } + + // Downloads run bounded, and a failure fails its asset rather than the run: one + // delisted or corrupt package must not stop a 75 GB mirror. + // The pool stops early only for a run-fatal error: an expired session makes every + // remaining download pointless, while one corrupt or delisted asset does not. + poolCtx, cancelPool := context.WithCancel(ctx) + defer cancelPool() + + var ( + wg sync.WaitGroup + sem = make(chan struct{}, opts.Concurrency) + done = make([]Result, len(pending)) + ) + for i, res := range pending { + wg.Add(1) + go func(i int, res Result) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + if poolCtx.Err() != nil { + res.Err = poolCtx.Err() + done[i] = res + return + } + opts.Progress(fmt.Sprintf("fetching %s (%s)", res.Asset.Name, humanBytes(res.Asset.AdvertisedSize))) + var ( + r resolution + warning string + resolved bool + ) + // Retry wraps the fetch and the write together, so every attempt necessarily + // opens a fresh temp file and a fresh hasher. Appending a retried response to + // a partial one would survive every guard here and then be hashed and + // recorded as its own truth. + err := retry.Do(poolCtx, opts.Retry, func(int) error { + var attemptErr error + r, warning, resolved, attemptErr = download(poolCtx, s, opts, res.Asset) + // Neither of these improves on a second attempt: the asset is gone, or + // the session is. + if errors.Is(attemptErr, store.ErrExpiredSession) || + errors.Is(attemptErr, store.ErrNotDownloadable) { + return retry.Permanent(attemptErr) + } + return attemptErr + }) + res.Warning, res.Err = warning, err + if err == nil && !resolved { + // A republish mid-download: nothing was stored, and the next run picks up + // the new build. Not a failure. + done[i] = res + return + } + if err == nil { + // A rename that also bumped the version downloads to the new derived + // path, so the prior directory would otherwise be left holding a + // superseded copy of the same asset. + if old := priorPaths[res.Asset.ID]; old != "" && old != r.cachePath { + if rmErr := cache.RemoveStale(opts.LibraryRoot, old); rmErr != nil { + res.Warning = strings.TrimSpace(res.Warning + " " + + fmt.Sprintf("(could not remove the superseded copy at %s: %v)", old, rmErr)) + } + } + mu.Lock() + resolutions[res.Asset.ID] = r + // Persisting per download is what keeps a run that dies at asset 90 of + // 100 from discarding the 89 it already fetched. The write stays inside + // the lock: released first, two goroutines can reach the rename in the + // order opposite to how they built their snapshots, and the older one + // wins — losing exactly the record this write exists to keep. + err := lockfile.Save(lockPath, build(owned, prior, resolutions, nil)) + mu.Unlock() + if err != nil { + res.Err = fmt.Errorf("persisting progress: %w", err) + } + } + if errors.Is(res.Err, store.ErrExpiredSession) { + cancelPool() + } + done[i] = res + }(i, res) + } + wg.Wait() + + for _, res := range done { + if res.Err != nil { + if errors.Is(res.Err, store.ErrNotDownloadable) { + report.Permanent++ + } else { + report.Retryable++ + } + } + report.Results = append(report.Results, res) + } + + report.Lockfile = build(owned, prior, resolutions, &report) + if err := lockfile.Save(lockPath, report.Lockfile); err != nil { + return report, err + } + return report, nil +} + +// adopt records a package already on disk, relocating it to where the layout puts it so +// the cache does not drift and quarry's facets stay right. +func adopt(opts Options, a model.Asset, found cache.Candidate, derived, damagedRel string) (resolution, error) { + // Relocate refuses an occupied destination, which is what stops it certifying bytes + // nothing checked. The one occupant that must not stop it is this asset's own recorded + // copy after it failed verification: a download would rename straight over that file, + // so a verified copy of the same package may replace it too. Without this the good copy + // can never move in, nothing is resolved, and every later run repeats the refusal. + if damagedRel != "" && damagedRel == derived { + if err := cache.RemoveStale(opts.LibraryRoot, damagedRel); err != nil { + return resolution{}, err + } + } + if err := cache.Relocate(opts.LibraryRoot, found.RelPath, derived); err != nil { + return resolution{}, err + } + sha, size, err := cache.Hash(opts.LibraryRoot, derived) + if err != nil { + return resolution{}, err + } + return resolution{ + cachePath: derived, + sha: sha, + size: size, + // The bytes are verified to be this version, so the diff key is known even + // though nothing was fetched. Leaving it empty would make every adopted asset + // classify Changed on the next run and re-download. + resolvedVersionID: a.Version.ID, + deliveredVersionID: found.Metadata.VersionID, + }, nil +} + +// download fetches one asset and runs every semantic guard against the temp file before +// committing it. The bool reports whether anything was stored: a republish discovered +// mid-transfer is a warning, not a failure, and resolves nothing this run. +func download(ctx context.Context, s Store, opts Options, a model.Asset) (resolution, string, bool, error) { + dl, err := s.Fetch(ctx, a.ID) + if err != nil { + return resolution{}, "", false, err + } + defer dl.Body.Close() + + // A 23 GB package would otherwise report nothing between "fetching" and "done", so + // progress is counted off the body as it streams rather than announced up front. + body := &progressReader{ + r: dl.Body, + total: a.AdvertisedSize, + report: func(read, total int64) { + opts.Progress(fmt.Sprintf(" %s: %s", a.Name, progressLine(read, total))) + }, + } + pending, err := cache.Store(opts.LibraryRoot, a.PublisherSlug(), a.Slug(), body) + if err != nil { + return resolution{}, "", false, err + } + + meta, metaErr := unitypackage.ReadFile(pending.TempPath()) + switch { + case metaErr != nil && !errors.Is(metaErr, unitypackage.ErrNoMetadata): + pending.Discard() + return resolution{}, "", false, retry.Permanent(fmt.Errorf("%s: %w", a.Name, metaErr)) + case metaErr == nil && meta.ID != a.ID: + pending.Discard() + return resolution{}, "", false, retry.Permanent( + fmt.Errorf("%s: the store served product %s, not %s", a.Name, meta.ID, a.ID)) + } + + var warning string + switch { + case metaErr != nil: + warning = fmt.Sprintf("%s: package carries no store metadata, so later checks fall back to size alone", a.Name) + case meta.VersionID != a.Version.ID: + // Steady state for a few products: the store advertises one build and serves + // another. Both ids are recorded; the run says so once rather than silently + // papering over the difference. + warning = fmt.Sprintf("%s: store advertises version %s but served %s; both recorded", + a.Name, a.Version.ID, meta.VersionID) + } + + if belowFloor(pending.Size, a.AdvertisedSize) { + // A short body is either a truncated transfer or a republish that moved the + // advertised size out from under us. One re-read of this product settles it. + if republished(ctx, s, a) { + pending.Discard() + return resolution{}, fmt.Sprintf( + "%s: republished mid-download; nothing stored, the next run will fetch the new build", + a.Name), false, nil + } + pending.Discard() + return resolution{}, "", false, fmt.Errorf("%s: received %d bytes against an advertised %d; body ended early", + a.Name, pending.Size, a.AdvertisedSize) + } + if a.AdvertisedSize > 0 && (pending.Size > a.AdvertisedSize || pending.Size < a.AdvertisedSize-64) { + // A package can both be served at a different version than advertised and land + // outside the window, and the version notice is the one the lockfile's two ids + // need explaining. + size := fmt.Sprintf("%s: received %d bytes, advertised %d", a.Name, pending.Size, a.AdvertisedSize) + if warning == "" { + warning = size + } else { + warning += "; " + size + } + } + + if err := pending.Commit(); err != nil { + return resolution{}, warning, false, err + } + return resolution{ + cachePath: pending.RelPath, + sha: pending.SHA256, + size: pending.Size, + resolvedVersionID: a.Version.ID, + deliveredVersionID: meta.VersionID, + downloadedAt: opts.Now().UTC().Format(time.RFC3339), + storeFilename: dl.Filename, + }, warning, true, nil +} + +// belowFloor is the hard short-body rule. The tolerance is absolute because the gap it +// forgives is a ceil-to-16 alignment artifact, and clamped so it cannot swallow a small +// package whole. +func belowFloor(received, advertised int64) bool { + if advertised <= 0 { + return false + } + allowance := int64(4096) + if eighth := advertised / 8; eighth < allowance { + allowance = eighth + } + return received < advertised-allowance +} + +// republished reports whether the store now advertises something different from what the +// enumeration saw. Keying on "delivered id differs from advertised id" instead would be +// wrong: for some products that difference is a steady state, and the floor would be +// switched off permanently for exactly those. +func republished(ctx context.Context, s Store, a model.Asset) bool { + fresh, ok, err := s.Lookup(ctx, a.ID) + if err != nil || !ok { + return false + } + return fresh.Version.ID != a.Version.ID || fresh.AdvertisedSize != a.AdvertisedSize +} + +// build produces the new lockfile: every owned asset gets an entry, advertised fields are +// refreshed, and any resolution this run did not touch is carried forward verbatim. +func build(owned []model.Asset, prior lockfile.Lockfile, resolutions map[string]resolution, + report *Report) lockfile.Lockfile { + + out := lockfile.New() + kept := map[string]bool{} + + for _, a := range owned { + _, prev, hasPrev := prior.FindByAssetID(a.ID) + if hasPrev { + kept[a.ID] = true + } + e := lockfile.Entry{ + AssetID: a.ID, + Name: a.Name, + State: string(a.State), + Publisher: lockfile.Publisher{ID: a.Publisher.ID, Name: a.Publisher.Name}, + Version: lockfile.Version{ID: a.Version.ID, Name: a.Version.Name, PublishedDate: a.Version.PublishedDate}, + AdvertisedSize: a.AdvertisedSize, + } + if r, ok := resolutions[a.ID]; ok { + e.Tracked = true + e.ResolvedVersionID = r.resolvedVersionID + e.DeliveredVersionID = r.deliveredVersionID + e.SizeBytes = r.size + e.SHA256 = r.sha + e.CachePath = r.cachePath + e.DownloadedAt = r.downloadedAt + e.StoreFilename = r.storeFilename + } else if hasPrev { + e.Tracked = prev.Tracked + e.ResolvedVersionID = prev.ResolvedVersionID + e.DeliveredVersionID = prev.DeliveredVersionID + e.SizeBytes = prev.SizeBytes + e.SHA256 = prev.SHA256 + e.CachePath = prev.CachePath + e.DownloadedAt = prev.DownloadedAt + e.StoreFilename = prev.StoreFilename + } + + // An asset the run did not resolve keeps its prior key, so the key and the + // cachePath cannot drift apart between runs. + key := a.Slug() + if _, resolvedNow := resolutions[a.ID]; !resolvedNow && hasPrev { + if prevKey, _, ok := prior.FindByAssetID(a.ID); ok { + key = prevKey + } + } + out.Assets[key] = e + } + + if report != nil { + for _, e := range prior.Assets { + if !kept[e.AssetID] { + report.Removed = append(report.Removed, e) + } + } + // Map order otherwise, which would reorder the summary between two runs that + // found the same thing. + sort.Slice(report.Removed, func(i, j int) bool { + if report.Removed[i].Name != report.Removed[j].Name { + return report.Removed[i].Name < report.Removed[j].Name + } + return report.Removed[i].AssetID < report.Removed[j].AssetID + }) + } + return out +} + +func fromEntry(e lockfile.Entry) resolution { + return resolution{ + cachePath: e.CachePath, + sha: e.SHA256, + size: e.SizeBytes, + resolvedVersionID: e.ResolvedVersionID, + deliveredVersionID: e.DeliveredVersionID, + downloadedAt: e.DownloadedAt, + storeFilename: e.StoreFilename, + } +} + +func selected(a model.Asset, opts Options) bool { + if !opts.Selected[a.ID] { + return false + } + if opts.OnlyGlob == "" { + return true + } + ok, _ := filepath.Match(opts.OnlyGlob, a.Slug()) + return ok +} + +func humanBytes(n int64) string { + const unit = 1000 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for v := n / unit; v >= unit; v /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(n)/float64(div), "kMGT"[exp]) +} + +// progressReader reports how far a transfer has got, at most once a second, so a +// multi-gigabyte download is visibly alive without flooding the terminal. +type progressReader struct { + r io.Reader + total int64 + read int64 + last time.Time + report func(read, total int64) +} + +func (p *progressReader) Read(b []byte) (int, error) { + n, err := p.r.Read(b) + p.read += int64(n) + if now := time.Now(); now.Sub(p.last) >= time.Second { + p.last = now + p.report(p.read, p.total) + } + return n, err +} + +func progressLine(read, total int64) string { + if total <= 0 { + return humanBytes(read) + } + return fmt.Sprintf("%s of %s (%d%%)", humanBytes(read), humanBytes(total), read*100/total) +} + +// memoize runs a probe at most once. The probes it wraps read or hash whole packages, so +// a caller asking twice is not merely redundant, it doubles the work. +func memoize(probe func() bool) func() bool { + var done, result bool + return func() bool { + if !done { + done, result = true, probe() + } + return result + } +} diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go new file mode 100644 index 0000000..d1162bc --- /dev/null +++ b/internal/syncer/syncer_test.go @@ -0,0 +1,454 @@ +package syncer + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/curbol/unity-sync/internal/cache" + "github.com/curbol/unity-sync/internal/lockfile" + "github.com/curbol/unity-sync/internal/manifest" + "github.com/curbol/unity-sync/internal/model" + "github.com/curbol/unity-sync/internal/store" +) + +// ---- fakes ----------------------------------------------------------------- + +type fakeStore struct { + owned []model.Asset + bodies map[string][]byte + fetchEr map[string]error + lookups map[string]model.Asset + + inFlight atomic.Int32 + maxSeen atomic.Int32 + hold time.Duration + mu sync.Mutex + fetched []string +} + +func (f *fakeStore) Enumerate(context.Context) ([]model.Asset, error) { return f.owned, nil } + +func (f *fakeStore) Lookup(_ context.Context, id string) (model.Asset, bool, error) { + a, ok := f.lookups[id] + return a, ok, nil +} + +func (f *fakeStore) Fetch(_ context.Context, id string) (*store.Download, error) { + n := f.inFlight.Add(1) + for { + max := f.maxSeen.Load() + if n <= max || f.maxSeen.CompareAndSwap(max, n) { + break + } + } + defer f.inFlight.Add(-1) + if f.hold > 0 { + time.Sleep(f.hold) + } + f.mu.Lock() + f.fetched = append(f.fetched, id) + f.mu.Unlock() + + if err := f.fetchEr[id]; err != nil { + return nil, err + } + body, ok := f.bodies[id] + if !ok { + return nil, store.ErrNotDownloadable + } + return &store.Download{Body: io.NopCloser(bytes.NewReader(body)), Filename: id + ".unitypackage"}, nil +} + +// pkg builds a package carrying a descriptor, padded to size. +func pkg(t *testing.T, productID, versionID string, size int) []byte { + t.Helper() + d := []byte(`{"id":"` + productID + `","version_id":"` + versionID + `"}`) + extra := []byte{'A', '$', 0, 0} + binary.LittleEndian.PutUint16(extra[2:4], uint16(len(d))) + extra = append(extra, d...) + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + zw.Header.Extra = extra + zw.Write(bytes.Repeat([]byte("x"), 32)) + zw.Close() + out := buf.Bytes() + for len(out) < size { + out = append(out, 0) + } + return out +} + +func asset(id, name string, versionID string, size int64) model.Asset { + return model.Asset{ + ID: id, Name: name, State: model.StatePublished, + Publisher: model.Publisher{ID: "p1", Name: "Pub One"}, + Version: model.Version{ID: versionID, Name: "1.0"}, + AdvertisedSize: size, + } +} + +func newRun(t *testing.T) (root, lockPath string) { + t.Helper() + root = t.TempDir() + return root, filepath.Join(t.TempDir(), "unity-sync.lock.json") +} + +func allSelected(assets ...model.Asset) map[string]bool { + m := map[string]bool{} + for _, a := range assets { + m[a.ID] = true + } + return m +} + +func opts(root string, sel map[string]bool) Options { + return Options{ + LibraryRoot: root, Selected: sel, Concurrency: 2, + Now: func() time.Time { return time.Unix(1700000000, 0).UTC() }, + } +} + +// ---- classify --------------------------------------------------------------- + +func TestClassifyCoversEveryClass(t *testing.T) { + yes := func() bool { return true } + no := func() bool { return false } + live := asset("1", "A", "v2", 1000) + tracked := lockfile.Entry{Tracked: true, ResolvedVersionID: "v2", CachePath: "p"} + + cases := []struct { + name string + a model.Asset + prior lockfile.Entry + hasPrior bool + cacheOK func() bool + adoptable func() bool + want Class + }{ + {"unchanged", live, tracked, true, yes, no, Unchanged}, + {"new", live, lockfile.Entry{}, false, no, no, New}, + {"changed", live, lockfile.Entry{Tracked: true, ResolvedVersionID: "v1", CachePath: "p"}, true, yes, no, Changed}, + {"download-now", live, lockfile.Entry{Tracked: false}, true, no, no, DownloadNow}, + {"cache-missing", live, tracked, true, no, no, CacheMissing}, + {"adopted with no record", live, lockfile.Entry{}, false, no, yes, Adopted}, + {"adopted when a record exists but nothing was mirrored", live, + lockfile.Entry{Tracked: false}, true, no, yes, Adopted}, + {"undownloadable", model.Asset{ID: "1", State: model.StateDisabled}, lockfile.Entry{}, false, no, no, Undownloadable}, + {"disabled but already mirrored stays usable", + model.Asset{ID: "1", State: model.StateDisabled, Version: model.Version{ID: "v2"}}, + tracked, true, yes, no, Unchanged}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := classify(tc.a, tc.prior, tc.hasPrior, tc.cacheOK, tc.adoptable); got != tc.want { + t.Errorf("classify = %v, want %v", got, tc.want) + } + }) + } +} + +// ---- lockfile coverage and carry-forward ------------------------------------- + +// The lockfile is the record of what is owned, not of what happens to be mirrored, and +// the adopt design depends on every owned asset having an entry. +func TestEveryOwnedAssetGetsAnEntryEvenWhenOnlyOneIsSelected(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Enabled", "v1", 500) + b := asset("2", "Not enabled", "v1", 500) + fs := &fakeStore{owned: []model.Asset{a, b}, bodies: map[string][]byte{"1": pkg(t, "1", "v1", 500)}} + + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(rep.Lockfile.Assets) != 2 { + t.Fatalf("lockfile has %d entries, want 2", len(rep.Lockfile.Assets)) + } + _, other, _ := rep.Lockfile.FindByAssetID("2") + if other.Tracked { + t.Error("the unselected asset was recorded as mirrored") + } + if other.Name != "Not enabled" { + t.Errorf("the unselected asset lost its advertised metadata: %+v", other) + } +} + +// The regression this guards is subtle: refreshing the advertised version beside a stale +// resolution would mark an unresolved asset as already current, forever. +func TestOnlyGlobPreservesOutOfScopeRecordsAndKeepsTheDiffKey(t *testing.T) { + root, lockPath := newRun(t) + inScope := asset("1", "In scope", "v2", 500) + outScope := asset("2", "Out of scope", "v2", 500) + + prior := lockfile.New() + prior.Assets["out-of-scope-2"] = lockfile.Entry{ + AssetID: "2", Name: "Out of scope", Tracked: true, + ResolvedVersionID: "v1", DeliveredVersionID: "v1", + SHA256: "old-sha", CachePath: "pub-one/out-of-scope-2/out-of-scope-2.unitypackage", + SizeBytes: 400, Version: lockfile.Version{ID: "v1"}, + } + + fs := &fakeStore{owned: []model.Asset{inScope, outScope}, bodies: map[string][]byte{"1": pkg(t, "1", "v2", 500)}} + o := opts(root, allSelected(inScope, outScope)) + o.OnlyGlob = "in-scope-1" + + rep, err := Run(context.Background(), fs, prior, lockPath, o) + if err != nil { + t.Fatalf("Run: %v", err) + } + _, out, ok := rep.Lockfile.FindByAssetID("2") + if !ok { + t.Fatal("the out-of-scope entry vanished") + } + if out.SHA256 != "old-sha" || out.CachePath == "" { + t.Errorf("out-of-scope resolution was not carried forward: %+v", out) + } + if out.Version.ID != "v2" { + t.Errorf("advertised version = %q, want the refreshed v2", out.Version.ID) + } + if out.ResolvedVersionID != "v1" { + t.Fatalf("resolvedVersionId = %q, want v1 carried verbatim; refreshing it would mark a "+ + "stale file as current forever", out.ResolvedVersionID) + } + // And it must still read as out of date next run. + if classify(outScope, out, true, func() bool { return true }, func() bool { return false }) != Changed { + t.Error("the carried-forward entry no longer classifies Changed") + } +} + +func TestRenamedAssetIsRecognisedByIdAndRekeyedOnce(t *testing.T) { + root, lockPath := newRun(t) + renamed := asset("1", "Brand New Name", "v1", 500) + + // Put a real file where the old slug says it is. + body := pkg(t, "1", "v1", 500) + p, err := cache.Store(root, "pub-one", "old-name-1", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if err := p.Commit(); err != nil { + t.Fatal(err) + } + + prior := lockfile.New() + prior.Assets["old-name-1"] = lockfile.Entry{ + AssetID: "1", Name: "Old Name", Tracked: true, + ResolvedVersionID: "v1", DeliveredVersionID: "v1", + SizeBytes: p.Size, SHA256: p.SHA256, CachePath: p.RelPath, + Version: lockfile.Version{ID: "v1"}, + } + + fs := &fakeStore{owned: []model.Asset{renamed}} + rep, err := Run(context.Background(), fs, prior, lockPath, opts(root, allSelected(renamed))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(fs.fetched) != 0 { + t.Errorf("a rename triggered a download of %v", fs.fetched) + } + if rep.Results[0].Class != Unchanged { + t.Errorf("class = %v, want Unchanged", rep.Results[0].Class) + } + if len(rep.Lockfile.Assets) != 1 { + t.Fatalf("lockfile has %d entries, want 1 — a rename must re-key, not duplicate", len(rep.Lockfile.Assets)) + } + if _, ok := rep.Lockfile.Assets["brand-new-name-1"]; !ok { + t.Errorf("entry was not re-keyed: %v", keys(rep.Lockfile)) + } + moved := filepath.Join(root, "pub-one", "brand-new-name-1", "brand-new-name-1.unitypackage") + if _, err := os.Stat(moved); err != nil { + t.Errorf("the cached directory did not move: %v", err) + } + if _, err := os.Stat(filepath.Join(root, "pub-one", "old-name-1")); !os.IsNotExist(err) { + t.Error("the old directory survived, so quarry's pack facet would stay stale") + } +} + +func TestOwnershipDropIsReportedNotJustRemoved(t *testing.T) { + root, lockPath := newRun(t) + kept := asset("1", "Kept", "v1", 500) + prior := lockfile.New() + prior.Assets["kept-1"] = lockfile.Entry{AssetID: "1", Name: "Kept", Version: lockfile.Version{ID: "v1"}} + prior.Assets["gone-2"] = lockfile.Entry{ + AssetID: "2", Name: "Refunded", Tracked: true, + CachePath: "pub-one/gone-2/gone-2.unitypackage", SizeBytes: 900, + } + fs := &fakeStore{owned: []model.Asset{kept}, bodies: map[string][]byte{"1": pkg(t, "1", "v1", 500)}} + + rep, err := Run(context.Background(), fs, prior, lockPath, opts(root, allSelected(kept))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(rep.Removed) != 1 || rep.Removed[0].AssetID != "2" { + t.Fatalf("Removed = %+v, want the no-longer-owned asset named", rep.Removed) + } + if _, _, ok := rep.Lockfile.FindByAssetID("2"); ok { + t.Error("the dropped asset is still in the lockfile") + } +} + +type failingEnumerate struct{ fakeStore } + +func (f *failingEnumerate) Enumerate(context.Context) ([]model.Asset, error) { + return nil, errors.New("session expired") +} + +// ---- download guards ---------------------------------------------------------- + +func tempsUnder(t *testing.T, root string) int { + t.Helper() + n := 0 + filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() && len(d.Name()) > 15 && d.Name()[:15] == ".unity-sync-dl-" { + n++ + } + return nil + }) + return n +} + +// ---- failure isolation and concurrency ---------------------------------------- + +func TestProgressSurvivesAMidRunFailure(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "First", "v1", 500) + b := asset("2", "Breaks", "v1", 500) + fs := &fakeStore{ + owned: []model.Asset{a, b}, + bodies: map[string][]byte{"1": pkg(t, "1", "v1", 500)}, + fetchEr: map[string]error{"2": errors.New("connection reset")}, + } + o := opts(root, allSelected(a, b)) + o.Concurrency = 1 + + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, o) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Retryable != 1 { + t.Errorf("Retryable = %d, want 1", rep.Retryable) + } + saved, err := lockfile.Load(lockPath) + if err != nil { + t.Fatal(err) + } + _, first, _ := saved.FindByAssetID("1") + if !first.Tracked { + t.Error("the asset fetched before the failure was not persisted") + } +} + +// ---- dry run ------------------------------------------------------------------- + +func treeSnapshot(t *testing.T, root string) string { + t.Helper() + var out []byte + filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil { + return nil + } + rel, _ := filepath.Rel(root, p) + out = append(out, rel...) + if !d.IsDir() { + if fi, err := d.Info(); err == nil { + out = append(out, []byte(fmt.Sprint(fi.Size()))...) + } + } + out = append(out, '\n') + return nil + }) + return string(out) +} + +// ---- manifest reporting --------------------------------------------------------- + +func TestUnknownManifestIdsAreReported(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Owned", "v1", 500) + o := opts(root, allSelected(a)) + o.Manifest = manifest.Manifest{Assets: []manifest.Entry{ + {ID: "1", Name: "Owned", Enabled: true}, + {ID: "404", Name: "Refunded or mistyped", Enabled: true}, + }} + fs := &fakeStore{owned: []model.Asset{a}, bodies: map[string][]byte{"1": pkg(t, "1", "v1", 500)}} + + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, o) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(rep.Unknown) != 1 || rep.Unknown[0].ID != "404" { + t.Errorf("Unknown = %+v, want the id the account does not own; silence is the failure "+ + "mode this report exists to prevent", rep.Unknown) + } +} + +// ---- adoption --------------------------------------------------------------------- + +func TestAdoptionRecordsTheDiffKeyAndLeavesDownloadedAtEmpty(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v1", 500) + body := pkg(t, "1", "v1", 500) + p, _ := cache.Store(root, a.PublisherSlug(), a.Slug(), bytes.NewReader(body)) + p.Commit() + + fs := &fakeStore{owned: []model.Asset{a}} + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Class != Adopted { + t.Fatalf("class = %v, want Adopted", rep.Results[0].Class) + } + if len(fs.fetched) != 0 { + t.Errorf("adoption downloaded %v", fs.fetched) + } + _, e, _ := rep.Lockfile.FindByAssetID("1") + if e.ResolvedVersionID != "v1" { + t.Errorf("resolvedVersionId = %q; empty would make this asset re-download every run", e.ResolvedVersionID) + } + if e.DownloadedAt != "" { + t.Errorf("downloadedAt = %q, want empty: the file was found, not fetched", e.DownloadedAt) + } + if e.SHA256 == "" { + t.Error("an adopted entry has no digest, so --verify would prove nothing about it") + } +} + +func TestATruncatedFileIsNotAdopted(t *testing.T) { + root, lockPath := newRun(t) + a := asset("1", "Asset", "v1", 4000) + // Descriptor intact, body far short of what the store advertises. + p, _ := cache.Store(root, a.PublisherSlug(), a.Slug(), bytes.NewReader(pkg(t, "1", "v1", 200))) + p.Commit() + + fs := &fakeStore{owned: []model.Asset{a}, bodies: map[string][]byte{"1": pkg(t, "1", "v1", 4000)}} + rep, err := Run(context.Background(), fs, lockfile.New(), lockPath, opts(root, allSelected(a))) + if err != nil { + t.Fatalf("Run: %v", err) + } + if rep.Results[0].Class == Adopted { + t.Fatal("a truncated package was adopted; the download floor exists to stop exactly this") + } + if len(fs.fetched) != 1 { + t.Errorf("fetched %v, want the asset re-downloaded", fs.fetched) + } +} + +func keys(lf lockfile.Lockfile) []string { + out := make([]string, 0, len(lf.Assets)) + for k := range lf.Assets { + out = append(out, k) + } + return out +} diff --git a/internal/unitypackage/unitypackage.go b/internal/unitypackage/unitypackage.go new file mode 100644 index 0000000..f9aec0f --- /dev/null +++ b/internal/unitypackage/unitypackage.go @@ -0,0 +1,86 @@ +// Package unitypackage reads the metadata the Asset Store stamps into every +// .unitypackage. A package is gzip, and the store puts a JSON descriptor in the gzip +// FEXTRA field — not the comment field, which is empty on every real package. Reading +// it costs a header parse, so a cached file can be identified without decompressing +// gigabytes or hashing them. +package unitypackage + +import ( + "compress/gzip" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "os" +) + +// subfieldID is the extra-field id the store uses for its descriptor. +var subfieldID = [2]byte{'A', '$'} + +// ErrNoMetadata means the file is a readable gzip stream that carries no store +// descriptor. Callers treat this as unverifiable-by-metadata rather than as corruption, +// because the alternative is refusing to cache a package that may be perfectly good. +var ErrNoMetadata = errors.New("no Asset Store metadata in gzip extra field") + +// Metadata is the descriptor's fields, all of which the store encodes as JSON strings. +type Metadata struct { + ID string `json:"id"` + Title string `json:"title"` + Version string `json:"version"` + VersionID string `json:"version_id"` + UploadID string `json:"upload_id"` + UnityVersion string `json:"unity_version"` + PubDate string `json:"pubdate"` +} + +// Read parses the gzip header from r and returns the store descriptor. It reads only as +// far as the header, so passing a whole multi-gigabyte package costs nothing extra. +func Read(r io.Reader) (Metadata, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return Metadata{}, fmt.Errorf("not a readable gzip stream: %w", err) + } + defer zr.Close() + return fromExtra(zr.Header.Extra) +} + +// ReadFile is Read against a path, reading only the header. +func ReadFile(path string) (Metadata, error) { + f, err := os.Open(path) + if err != nil { + return Metadata{}, err + } + defer f.Close() + return Read(f) +} + +// fromExtra walks the RFC 1952 subfield list. The walk is driven by each subfield's own +// length, never by an assumed prefix: XLEN is a uint16, so a descriptor may legitimately +// sit far past where every package sampled so far happens to end it, and a +// prefix-limited reader would silently report "no metadata" for such a package — +// downgrading the hard wrong-asset check into a warning. +func fromExtra(extra []byte) (Metadata, error) { + for i := 0; i+4 <= len(extra); { + id1, id2 := extra[i], extra[i+1] + length := int(binary.LittleEndian.Uint16(extra[i+2 : i+4])) + start := i + 4 + end := start + length + if end > len(extra) { + return Metadata{}, fmt.Errorf("extra subfield %q claims %d bytes but only %d remain", + string([]byte{id1, id2}), length, len(extra)-start) + } + if id1 == subfieldID[0] && id2 == subfieldID[1] { + var m Metadata + if err := json.Unmarshal(extra[start:end], &m); err != nil { + return Metadata{}, fmt.Errorf("metadata subfield is not valid JSON: %w", err) + } + if m.ID == "" { + return Metadata{}, fmt.Errorf("metadata subfield carries no product id") + } + return m, nil + } + i = end + } + return Metadata{}, ErrNoMetadata +} diff --git a/internal/unitypackage/unitypackage_test.go b/internal/unitypackage/unitypackage_test.go new file mode 100644 index 0000000..3f025a1 --- /dev/null +++ b/internal/unitypackage/unitypackage_test.go @@ -0,0 +1,125 @@ +package unitypackage_test + +import ( + "bytes" + "compress/gzip" + "encoding/binary" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/curbol/unity-sync/internal/unitypackage" +) + +// realHeader is the first kilobyte of an actual downloaded package, committed as a +// fixture. Everything else here is synthetic, so this is the only case proving the +// parser matches what the store really emits. +func realHeader(t *testing.T) []byte { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "testdata", "store", "package_header_sample.bin")) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + return raw +} + +func TestReadsTheDescriptorFromARealPackageHeader(t *testing.T) { + m, err := unitypackage.Read(bytes.NewReader(realHeader(t))) + if err != nil { + t.Fatalf("Read: %v", err) + } + if m.ID != "323439" { + t.Errorf("ID = %q, want 323439", m.ID) + } + if m.VersionID != "1361204" { + t.Errorf("VersionID = %q, want 1361204", m.VersionID) + } + if m.Version != "1.9" { + t.Errorf("Version = %q, want 1.9", m.Version) + } + if m.UnityVersion != "6000.0.32f1" { + t.Errorf("UnityVersion = %q, want 6000.0.32f1", m.UnityVersion) + } +} + +// gzipWithExtra builds a valid gzip stream whose extra field holds the given subfields. +func gzipWithExtra(t *testing.T, subfields ...[]byte) []byte { + t.Helper() + var extra []byte + for _, s := range subfields { + extra = append(extra, s...) + } + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + zw.Header.Extra = extra + if _, err := zw.Write([]byte("payload")); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func subfield(id string, payload []byte) []byte { + out := []byte{id[0], id[1], 0, 0} + binary.LittleEndian.PutUint16(out[2:4], uint16(len(payload))) + return append(out, payload...) +} + +// XLEN is a uint16, so a descriptor can legitimately sit tens of kilobytes in. A reader +// that only inspects a fixed prefix passes every other case here and then silently +// reports "no metadata" in production, turning the hard wrong-asset check into a +// warning. +func TestFindsADescriptorFarPastAnyObservedOffset(t *testing.T) { + filler := subfield("XX", bytes.Repeat([]byte{0}, 40000)) + real := subfield("A$", []byte(`{"id":"999","version_id":"42"}`)) + m, err := unitypackage.Read(bytes.NewReader(gzipWithExtra(t, filler, real))) + if err != nil { + t.Fatalf("Read with a 40 KB leading subfield: %v", err) + } + if m.ID != "999" || m.VersionID != "42" { + t.Errorf("got %+v, want id 999 / version_id 42", m) + } +} + +func TestForeignSubfieldsAloneMeanNoMetadata(t *testing.T) { + body := gzipWithExtra(t, subfield("QQ", []byte("something else"))) + _, err := unitypackage.Read(bytes.NewReader(body)) + if !errors.Is(err, unitypackage.ErrNoMetadata) { + t.Errorf("Read = %v, want ErrNoMetadata", err) + } +} + +func TestNoExtraFieldMeansNoMetadata(t *testing.T) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + zw.Write([]byte("payload")) + zw.Close() + _, err := unitypackage.Read(bytes.NewReader(buf.Bytes())) + if !errors.Is(err, unitypackage.ErrNoMetadata) { + t.Errorf("Read = %v, want ErrNoMetadata", err) + } +} + +func TestTruncatedHeaderIsAnError(t *testing.T) { + full := realHeader(t) + if _, err := unitypackage.Read(bytes.NewReader(full[:20])); err == nil { + t.Error("Read accepted a truncated gzip header") + } +} + +func TestNonGzipIsAnError(t *testing.T) { + if _, err := unitypackage.Read(strings.NewReader("sign in")); err == nil { + t.Error("Read accepted a body that is not gzip at all") + } +} + +func TestDescriptorWithoutAProductIdIsRejected(t *testing.T) { + body := gzipWithExtra(t, subfield("A$", []byte(`{"version_id":"42"}`))) + if _, err := unitypackage.Read(bytes.NewReader(body)); err == nil { + t.Error("Read accepted a descriptor with no product id; the id gate depends on it") + } +} diff --git a/internal/web/web.go b/internal/web/web.go new file mode 100644 index 0000000..043b461 --- /dev/null +++ b/internal/web/web.go @@ -0,0 +1,239 @@ +// Package web serves the local page behind `unity-sync select`: every owned asset with +// its thumbnail and a checkbox, returning the chosen set so the caller can persist the +// manifest. It is the only command that writes the manifest, so the page is also where +// the guards against clobbering a curated file live. +package web + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "html/template" + "net" + "net/http" + "os" + "os/exec" + "runtime" + "sort" + "strings" + + "github.com/curbol/unity-sync/internal/model" +) + +// The two refusals, as the page states them. A save that would clear every selection at +// once, and a POST from a page some earlier run served: a stale tab reopened after the +// library changed, or a mis-click on "none", should not silently wipe a curated allowlist. +const ( + msgWouldEmptySelection = "refusing a save that would deselect everything" + msgStaleTab = "this page was served by an earlier run; reload and choose again" +) + +type row struct { + ID string + Name string + Publisher string + Size string + State string + Thumb string + Enabled bool +} + +type pageData struct { + Rows []row + Token string + Count int +} + +var page = template.Must(template.New("select").Parse(` +unity-sync select + +
+ unity-sync + + {{.Count}} selected + + + +
+
+ +
+{{range .Rows}} + +{{end}} +
+
+ +`)) + +// Selection is what the page returned. +type Selection map[string]bool + +// Handler renders the page and accepts one save. It is separated from Serve so the +// behaviour can be tested without a socket. +// +// A refused save is answered and nothing more: the page stays up so the user can correct +// the mistake the refusal describes. +type Handler struct { + assets []model.Asset + enabled map[string]bool + token string + + done chan Selection +} + +// Selection delivers the one accepted save, so a caller without a socket can wait on the +// same channel Serve does. +func (h *Handler) Selection() <-chan Selection { return h.done } + +// NewHandler builds the page handler for one run. +func NewHandler(assets []model.Asset, enabled map[string]bool) *Handler { + buf := make([]byte, 16) + rand.Read(buf) + return &Handler{ + assets: assets, + enabled: enabled, + token: hex.EncodeToString(buf), + done: make(chan Selection, 1), + } +} + +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + h.save(w, r) + return + } + sorted := append([]model.Asset(nil), h.assets...) + sort.Slice(sorted, func(i, j int) bool { return strings.ToLower(sorted[i].Name) < strings.ToLower(sorted[j].Name) }) + + data := pageData{Token: h.token} + for _, a := range sorted { + if h.enabled[a.ID] { + data.Count++ + } + data.Rows = append(data.Rows, row{ + ID: a.ID, + Name: a.Name, + Publisher: a.Publisher.Name, + Size: humanBytes(a.AdvertisedSize), + State: string(a.State), + Thumb: absoluteURL(a.ThumbnailURL), + Enabled: h.enabled[a.ID], + }) + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + page.Execute(w, data) +} + +func (h *Handler) save(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + http.Error(w, "bad form", http.StatusBadRequest) + return + } + if r.PostFormValue("token") != h.token { + http.Error(w, msgStaleTab, http.StatusConflict) + return + } + chosen := Selection{} + for _, id := range r.PostForm["asset"] { + chosen[id] = true + } + if len(chosen) == 0 && anyEnabled(h.enabled) { + http.Error(w, msgWouldEmptySelection, http.StatusConflict) + return + } + fmt.Fprintf(w, "Saved %d selection(s). You can close this tab.", len(chosen)) + h.done <- chosen +} + +// Serve runs the page until it is saved once or the context ends. +func Serve(ctx context.Context, addr string, assets []model.Asset, enabled map[string]bool) (Selection, error) { + h := NewHandler(assets, enabled) + ln, err := net.Listen("tcp", addr) + if err != nil { + return nil, err + } + srv := &http.Server{Handler: h} + go srv.Serve(ln) + defer srv.Close() + + url := "http://" + ln.Addr().String() + fmt.Fprintln(os.Stderr, "select assets at", url) + openBrowser(url) + + select { + case sel := <-h.done: + return sel, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// absoluteURL fixes the store's protocol-relative image URLs, which would otherwise +// resolve to http:// on a page served from localhost. +func absoluteURL(u string) string { + if strings.HasPrefix(u, "//") { + return "https:" + u + } + return u +} + +func anyEnabled(m map[string]bool) bool { + for _, v := range m { + if v { + return true + } + } + return false +} + +func openBrowser(url string) { + var cmd string + switch runtime.GOOS { + case "darwin": + cmd = "open" + case "windows": + cmd = "explorer" + default: + cmd = "xdg-open" + } + exec.Command(cmd, url).Start() +} + +func humanBytes(n int64) string { + const unit = 1000 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for v := n / unit; v >= unit; v /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(n)/float64(div), "kMGT"[exp]) +} diff --git a/internal/web/web_test.go b/internal/web/web_test.go new file mode 100644 index 0000000..686ba3d --- /dev/null +++ b/internal/web/web_test.go @@ -0,0 +1,185 @@ +package web_test + +import ( + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "strings" + "testing" + "time" + + "github.com/curbol/unity-sync/internal/model" + "github.com/curbol/unity-sync/internal/web" +) + +func assets() []model.Asset { + return []model.Asset{ + {ID: "115488", Name: "Quick Outline", State: model.StatePublished, + Publisher: model.Publisher{Name: "Chris Nolet"}, AdvertisedSize: 33824, + ThumbnailURL: "//assetstorev1-prd-cdn.unity3d.com/key-image/abc.png"}, + {ID: "193760", Name: "Fantasy Sounds Bundle", State: model.StateDisabled, + Publisher: model.Publisher{Name: "Cafofo"}, AdvertisedSize: 1000}, + } +} + +func render(t *testing.T, h http.Handler) string { + t.Helper() + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("GET = %d", rec.Code) + } + return rec.Body.String() +} + +var tokenRe = regexp.MustCompile(`name=token value="([0-9a-f]+)"`) + +func tokenFrom(t *testing.T, body string) string { + t.Helper() + m := tokenRe.FindStringSubmatch(body) + if m == nil { + t.Fatal("page carries no token") + } + return m[1] +} + +func TestPageRendersEveryAssetWithItsState(t *testing.T) { + h := web.NewHandler(assets(), map[string]bool{"115488": true}) + body := render(t, h) + + for _, want := range []string{"Quick Outline", "Chris Nolet", "Fantasy Sounds Bundle", "disabled"} { + if !strings.Contains(body, want) { + t.Errorf("page is missing %q", want) + } + } + if !strings.Contains(body, `value="115488" checked`) { + t.Error("the already-enabled asset is not checked") + } +} + +// The store returns protocol-relative image URLs; on a page served over http://localhost +// they would resolve to http:// and be blocked or broken. +func TestThumbnailsAreMadeAbsolute(t *testing.T) { + body := render(t, web.NewHandler(assets(), nil)) + if !strings.Contains(body, `src="https://assetstorev1-prd-cdn.unity3d.com/key-image/abc.png"`) { + t.Error("thumbnail URL was not normalised to https") + } + if strings.Contains(body, `src="//assetstore`) { + t.Error("page still carries a protocol-relative image URL") + } +} + +func TestSaveReturnsTheChosenSet(t *testing.T) { + h := web.NewHandler(assets(), map[string]bool{"115488": true}) + body := render(t, h) + + form := url.Values{"token": {tokenFrom(t, body)}, "asset": {"115488", "193760"}} + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + done := make(chan struct{}) + go func() { h.ServeHTTP(rec, req); close(done) }() + <-done + + if rec.Code != http.StatusOK { + t.Fatalf("POST = %d: %s", rec.Code, rec.Body) + } +} + +// A page served by an earlier run still has a Save button. Honouring it would apply a +// selection made against a different library. +func TestStaleTabIsRefused(t *testing.T) { + h := web.NewHandler(assets(), map[string]bool{"115488": true}) + render(t, h) + + form := url.Values{"token": {"from-an-older-run"}, "asset": {"115488"}} + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusConflict { + t.Errorf("POST from a stale tab = %d, want %d", rec.Code, http.StatusConflict) + } + if !strings.Contains(rec.Body.String(), "earlier run") { + t.Errorf("response %q does not explain the refusal", rec.Body) + } +} + +// select is the only command that writes the manifest, so clearing every selection at +// once has to be deliberate rather than a mis-click or a reloaded old tab. +func TestSaveThatWouldDeselectEverythingIsRefused(t *testing.T) { + h := web.NewHandler(assets(), map[string]bool{"115488": true}) + body := render(t, h) + + form := url.Values{"token": {tokenFrom(t, body)}} + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusConflict { + t.Errorf("empty save = %d, want %d", rec.Code, http.StatusConflict) + } + if !strings.Contains(rec.Body.String(), "deselect everything") { + t.Errorf("response %q does not explain the refusal", rec.Body) + } +} + +// An empty save is legitimate when nothing was selected to begin with: that is a first +// run where the user decided not to pick anything yet. +func TestEmptySaveIsFineWhenNothingWasSelected(t *testing.T) { + h := web.NewHandler(assets(), map[string]bool{}) + body := render(t, h) + + form := url.Values{"token": {tokenFrom(t, body)}} + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + done := make(chan struct{}) + go func() { h.ServeHTTP(rec, req); close(done) }() + <-done + + if rec.Code != http.StatusOK { + t.Errorf("empty save on a fresh manifest = %d, want 200", rec.Code) + } +} + +// The 409 body tells the user to reload and choose again, so the page has to still be +// there when they do. The same property is what stops any page the user has open from +// ending a selection with one cross-origin POST. +func TestARefusedSaveLeavesThePageServing(t *testing.T) { + h := web.NewHandler(assets(), map[string]bool{"115488": true}) + body := render(t, h) + + stale := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader( + url.Values{"token": {"from-an-older-run"}}.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + h.ServeHTTP(stale, req) + if stale.Code != http.StatusConflict { + t.Fatalf("stale POST = %d, want %d", stale.Code, http.StatusConflict) + } + + // The page is still up, with the same token, and a correct save still lands. + if got := render(t, h); tokenFrom(t, got) != tokenFrom(t, body) { + t.Error("the refusal changed the page token") + } + saved := httptest.NewRecorder() + ok := httptest.NewRequest(http.MethodPost, "/", strings.NewReader( + url.Values{"token": {tokenFrom(t, body)}, "asset": {"115488"}}.Encode())) + ok.Header.Set("Content-Type", "application/x-www-form-urlencoded") + go h.ServeHTTP(saved, ok) + + select { + case sel := <-h.Selection(): + if !sel["115488"] { + t.Errorf("selection = %v, want asset 115488", sel) + } + case <-time.After(2 * time.Second): + t.Fatal("the save after a refusal never arrived") + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..201135d --- /dev/null +++ b/main.go @@ -0,0 +1,360 @@ +// Command unity-sync mirrors the assets owned on the Unity Asset Store into a local +// library, downloading only what changed since the last run. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "sort" + "strings" + "syscall" + + "github.com/curbol/unity-sync/internal/config" + "github.com/curbol/unity-sync/internal/lockfile" + "github.com/curbol/unity-sync/internal/manifest" + "github.com/curbol/unity-sync/internal/model" + "github.com/curbol/unity-sync/internal/selfupdate" + "github.com/curbol/unity-sync/internal/session" + "github.com/curbol/unity-sync/internal/store" + "github.com/curbol/unity-sync/internal/syncer" + "github.com/curbol/unity-sync/internal/web" +) + +// version is stamped at release time. +var version = "dev" + +const defaultSelectAddr = "127.0.0.1:8788" + +// stdout is where the tool's own output goes, so tests can capture it. Progress and +// diagnostics stay on stderr. +var stdout io.Writer = os.Stdout + +func main() { + code, err := run(os.Args[1:]) + if err != nil { + fmt.Fprintln(os.Stderr, "unity-sync:", err) + if code == 0 { + code = 1 + } + } + os.Exit(code) +} + +func run(args []string) (int, error) { + if len(args) == 0 { + usage() + return 1, errors.New("a subcommand is required") + } + cmd, rest := args[0], args[1:] + + fs := flag.NewFlagSet(cmd, flag.ContinueOnError) + fs.SetOutput(io.Discard) + cfgDir := fs.String("config", "", "user config dir (default $XDG_CONFIG_HOME/unity-sync)") + manifestFlag := fs.String("manifest", "", "project manifest path (default: nearest unity-sync.toml walking up)") + sessionFlag := fs.String("session", "", "session file: a pasted-curl or cookies.txt export") + library := fs.String("library", "", "library directory (overrides config / UNITY_SYNC_LIBRARY)") + only := fs.String("only", "", "limit to assets whose slug matches this glob") + concurrency := fs.Int("concurrency", 0, "max simultaneous downloads (overrides config)") + verify := fs.Bool("verify", false, "re-hash cached files instead of the cheap size+metadata check") + dryRun := fs.Bool("dry-run", false, "on sync, classify and report only") + addr := fs.String("addr", defaultSelectAddr, "on select, the address to serve the page at") + + switch cmd { + case "select", "status", "sync", "list", "update", "version", + "-h", "--help", "help", "-v", "--version": + default: + usage() + return 1, fmt.Errorf("unknown subcommand %q", cmd) + } + switch cmd { + case "-h", "--help", "help", "version", "-v", "--version": + // These return before flag parsing, so their positionals are checked here or not + // at all: `unity-sync version foo` should not quietly succeed. + if len(rest) > 0 { + return 1, fmt.Errorf("%s takes no arguments (got %q)", cmd, rest[0]) + } + if cmd == "version" || cmd == "-v" || cmd == "--version" { + fmt.Fprintln(stdout, "unity-sync", version) + } else { + usage() + } + return 0, nil + } + if err := fs.Parse(rest); err != nil { + if errors.Is(err, flag.ErrHelp) { + usage() + return 0, nil + } + return 1, err + } + + // flag.Parse stops at the first non-flag argument, so an unchecked positional would + // silently swallow every flag after it: `sync foo --dry-run` would download. + if cmd == "update" { + if fs.NArg() > 1 { + return 1, fmt.Errorf("update takes at most one version, got %d arguments", fs.NArg()) + } + return 0, selfupdate.Run(version, fs.Arg(0)) + } + if fs.NArg() > 0 { + return 1, fmt.Errorf("%s takes no positional arguments (got %q); to limit assets use --only %s", + cmd, fs.Arg(0), fs.Arg(0)) + } + + configDir := config.ResolveDir(*cfgDir) + cfg, err := config.Load(configDir) + if err != nil { + return 1, err + } + if *library != "" { + cfg.LibraryPath = *library + } + if *concurrency > 0 { + cfg.Concurrency = *concurrency + } + if *sessionFlag != "" { + cfg.SessionSource = *sessionFlag + } + + manifestPath, err := resolveManifest(*manifestFlag, cmd) + if err != nil { + return 1, err + } + lockPath := manifest.LockPath(manifestPath) + + if cmd == "list" { + return 0, list(stdout, lockPath) + } + + cookie, err := resolveSession(cfg, configDir) + if err != nil { + return 1, err + } + client := store.New(cookie, version) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if err := client.Bootstrap(ctx); err != nil { + return 1, err + } + + if cmd == "select" { + return 0, selectAssets(ctx, client, manifestPath, *addr) + } + return syncOrStatus(ctx, client, cfg, manifestPath, lockPath, *only, *verify, cmd == "status" || *dryRun) +} + +// resolveManifest finds the project manifest. Every command needs one except select, +// which creates it in the working directory when no ancestor has one. +func resolveManifest(flagValue, cmd string) (string, error) { + if flagValue != "" { + return flagValue, nil + } + wd, err := os.Getwd() + if err != nil { + return "", err + } + if p, ok := manifest.Discover(wd); ok { + return p, nil + } + if cmd == "select" { + return filepath.Join(wd, manifest.FileName), nil + } + return "", fmt.Errorf("no %s found in this directory or its parents; run `unity-sync select` to create one", + manifest.FileName) +} + +func resolveSession(cfg config.Config, configDir string) (string, error) { + src := cfg.SessionSource + if src == "" { + if found, ok := session.Discover(configDir); ok { + src = found + } + } + if src == "" { + return "", fmt.Errorf("no session configured: save a pasted-curl file as %s, "+ + "set session_source in config.toml, or pass --session", + filepath.Join(configDir, "session.curl")) + } + return session.Resolve(src) +} + +// enumerator is the slice of the store client that `select` needs. +type enumerator interface { + Enumerate(ctx context.Context) ([]model.Asset, error) +} + +func selectAssets(ctx context.Context, client enumerator, manifestPath, addr string) error { + m, err := manifest.Load(manifestPath) + if err != nil { + return err + } + owned, err := client.Enumerate(ctx) + if err != nil { + return err + } + dropped, err := m.Reconcile(owned) + if err != nil { + return err + } + for _, e := range dropped { + fmt.Fprintf(os.Stderr, "no longer owned, dropping from the manifest: %s (%s)\n", e.Name, e.ID) + } + chosen, err := web.Serve(ctx, addr, owned, m.EnabledIDs()) + if err != nil { + return err + } + m.SetEnabled(chosen) + if err := manifest.Save(manifestPath, m); err != nil { + return err + } + fmt.Fprintf(stdout, "saved %d selected asset(s) to %s\n", len(chosen), manifestPath) + return nil +} + +func syncOrStatus(ctx context.Context, client syncer.Store, cfg config.Config, + manifestPath, lockPath, only string, verify, dry bool) (int, error) { + + m, err := manifest.Load(manifestPath) + if err != nil { + return 1, err + } + prior, err := lockfile.Load(lockPath) + if err != nil { + return 1, err + } + selected := m.EnabledIDs() + if len(selected) == 0 { + fmt.Fprintln(os.Stderr, "nothing is enabled in", manifestPath, "— run `unity-sync select` to choose assets") + } + + rep, err := syncer.Run(ctx, client, prior, lockPath, syncer.Options{ + LibraryRoot: cfg.LibraryPath, + Selected: selected, + OnlyGlob: only, + DryRun: dry, + FullVerify: verify, + Concurrency: cfg.Concurrency, + Manifest: m, + Progress: func(s string) { fmt.Fprintln(os.Stderr, s) }, + }) + if err != nil { + // Run hands back the report alongside an error, and a late failure — the final + // lockfile write, say — comes after a full download pass whose outcome the user + // still needs. An error raised before any of that has nothing to show. + if len(rep.Results) > 0 { + printReport(stdout, rep, dry, cfg.LibraryPath) + } + return 1, err + } + printReport(stdout, rep, dry, cfg.LibraryPath) + if rep.Failed() { + return 1, nil + } + return 0, nil +} + +func printReport(w io.Writer, rep syncer.Report, dry bool, libraryPath string) { + counts := map[string]int{} + for _, r := range rep.Results { + counts[r.Class.String()]++ + } + verb := "sync" + if dry { + verb = "status (no changes made)" + } + fmt.Fprintf(w, "%s: %d owned, %d selected\n", verb, rep.Owned, len(rep.Results)) + for _, class := range []string{"new", "changed", "download-now", "cache-missing", "adopted", "unchanged", "undownloadable"} { + if n := counts[class]; n > 0 { + fmt.Fprintf(w, " %-15s %d\n", class, n) + } + } + if rep.Swept > 0 { + fmt.Fprintf(w, " reclaimed %d abandoned download(s)\n", rep.Swept) + } + fmt.Fprintf(w, "library: %s\n", libraryPath) + + for _, r := range rep.Results { + if r.Warning != "" { + fmt.Fprintf(w, "warning: %s\n", r.Warning) + } + } + for _, r := range rep.Results { + if r.Err != nil { + fmt.Fprintf(w, "failed: %s: %v\n", r.Asset.Name, r.Err) + } + } + // A tally of one among hundreds of owned assets does not tell the user which package + // the store stopped serving, and that is the only thing they can act on. + for _, r := range rep.Results { + if r.Class == syncer.Undownloadable { + fmt.Fprintf(w, "delisted, cannot be downloaded: %s (%s), state %s\n", + r.Asset.Name, r.Asset.ID, r.Asset.State) + } + } + // A dropped asset leaves its bytes on disk; the summary names them so the user can + // decide. Losing ownership of an asset never deletes its package: a run only ever + // removes a copy it is replacing with a newer one of that same asset. + for _, e := range rep.Removed { + if e.CachePath != "" { + fmt.Fprintf(w, "no longer owned: %s — %s (%d bytes) left in place\n", e.Name, e.CachePath, e.SizeBytes) + } else { + fmt.Fprintf(w, "no longer owned: %s\n", e.Name) + } + } + for _, e := range rep.Unknown { + fmt.Fprintf(w, "manifest lists asset %s (%s), which this account does not own\n", e.ID, e.Name) + } +} + +func list(w io.Writer, lockPath string) error { + lf, err := lockfile.Load(lockPath) + if err != nil { + return err + } + if len(lf.Assets) == 0 { + fmt.Fprintln(w, "no lockfile yet at", lockPath) + return nil + } + keys := make([]string, 0, len(lf.Assets)) + for k := range lf.Assets { + keys = append(keys, k) + } + sort.Strings(keys) + + mirrored := 0 + for _, k := range keys { + e := lf.Assets[k] + state := "owned" + if e.Tracked { + state = "mirrored" + mirrored++ + } + fmt.Fprintf(w, "%-10s %-10s %-8s %s\n", state, e.Version.Name, e.AssetID, e.Name) + } + fmt.Fprintf(w, "\n%d owned, %d mirrored\n", len(keys), mirrored) + return nil +} + +func usage() { + fmt.Fprint(os.Stderr, strings.TrimLeft(` +unity-sync mirrors the assets you own on the Unity Asset Store. + + unity-sync select pick which assets to mirror (opens a local page) + unity-sync status what a sync would change; downloads nothing + unity-sync sync download the delta and update the lockfile + unity-sync list print the current lockfile + unity-sync update replace this binary with the latest release + unity-sync version print the installed version + +Flags: --config --manifest --session --library --only --concurrency --verify + --dry-run --addr +`, "\n")) +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..9d33a9c --- /dev/null +++ b/main_test.go @@ -0,0 +1,266 @@ +package main + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/binary" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/curbol/unity-sync/internal/config" + "github.com/curbol/unity-sync/internal/lockfile" + "github.com/curbol/unity-sync/internal/manifest" + "github.com/curbol/unity-sync/internal/model" + "github.com/curbol/unity-sync/internal/store" +) + +func capture(t *testing.T) *bytes.Buffer { + t.Helper() + buf := &bytes.Buffer{} + old := stdout + stdout = buf + t.Cleanup(func() { stdout = old }) + return buf +} + +// isolate keeps a developer's real config, session and library out of the tests. +func isolate(t *testing.T) string { + t.Helper() + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "config")) + t.Setenv("XDG_DATA_HOME", filepath.Join(home, "data")) + for _, k := range []string{"UNITY_SYNC_CONFIG_DIR", "UNITY_SYNC_LIBRARY", "UNITY_SYNC_SESSION"} { + os.Unsetenv(k) + } + wd := t.TempDir() + prev, _ := os.Getwd() + if err := os.Chdir(wd); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chdir(prev) }) + return wd +} + +func TestVersionAndHelpSucceed(t *testing.T) { + isolate(t) + out := capture(t) + code, err := run([]string{"version"}) + if code != 0 || err != nil { + t.Fatalf("version = %d, %v", code, err) + } + if !strings.Contains(out.String(), "unity-sync") { + t.Errorf("version printed %q", out) + } + if code, err := run([]string{"help"}); code != 0 || err != nil { + t.Errorf("help = %d, %v", code, err) + } +} + +func TestUnknownSubcommandFails(t *testing.T) { + isolate(t) + code, err := run([]string{"frobnicate"}) + if code == 0 || err == nil { + t.Fatalf("unknown subcommand = %d, %v; want a failure", code, err) + } + if !strings.Contains(err.Error(), "frobnicate") { + t.Errorf("error %q does not name the subcommand", err) + } +} + +// update is the one subcommand that takes a positional. +func TestUpdateAcceptsOneVersionAndRejectsTwo(t *testing.T) { + isolate(t) + code, err := run([]string{"update", "1.2.3", "4.5.6"}) + if code == 0 || err == nil || !strings.Contains(err.Error(), "at most one") { + t.Errorf("update with two versions = %d, %v", code, err) + } + // One argument gets past dispatch and fails later, on the dev-build guard. + _, err = run([]string{"update", "1.2.3"}) + if err == nil || !strings.Contains(err.Error(), "dev build") { + t.Errorf("update on a dev build = %v, want the dev-build guard", err) + } +} + +func TestCommandsThatNeedAManifestSayHowToMakeOne(t *testing.T) { + isolate(t) + for _, cmd := range []string{"status", "sync"} { + code, err := run([]string{cmd}) + if code == 0 || err == nil { + t.Fatalf("%s with no manifest = %d, %v; want a failure", cmd, code, err) + } + if !strings.Contains(err.Error(), "select") { + t.Errorf("%s: error %q does not tell the user how to create one", cmd, err) + } + } +} + +// list reads only the lockfile, so it must work with no session and no network. +func TestListNeedsNoSession(t *testing.T) { + wd := isolate(t) + out := capture(t) + + manifestPath := filepath.Join(wd, manifest.FileName) + if err := os.WriteFile(manifestPath, []byte("\n"), 0o644); err != nil { + t.Fatal(err) + } + lf := lockfile.New() + lf.Assets["quick-outline-115488"] = lockfile.Entry{ + AssetID: "115488", Name: "Quick Outline", Tracked: true, + Version: lockfile.Version{ID: "683375", Name: "1.1"}, + } + lf.Assets["owned-only-999"] = lockfile.Entry{ + AssetID: "999", Name: "Owned But Not Mirrored", + Version: lockfile.Version{ID: "1", Name: "1.0"}, + } + if err := lockfile.Save(manifest.LockPath(manifestPath), lf); err != nil { + t.Fatal(err) + } + + code, err := run([]string{"list"}) + if code != 0 || err != nil { + t.Fatalf("list = %d, %v", code, err) + } + body := out.String() + for _, want := range []string{"Quick Outline", "Owned But Not Mirrored", "2 owned, 1 mirrored"} { + if !strings.Contains(body, want) { + t.Errorf("list output is missing %q:\n%s", want, body) + } + } +} + +// fakeStore drives the commands that otherwise need a live session, so their happy paths +// are covered rather than only their error paths. +type fakeStore struct { + owned []model.Asset + bodies map[string][]byte +} + +func (f *fakeStore) Enumerate(context.Context) ([]model.Asset, error) { return f.owned, nil } + +func (f *fakeStore) Lookup(_ context.Context, id string) (model.Asset, bool, error) { + for _, a := range f.owned { + if a.ID == id { + return a, true, nil + } + } + return model.Asset{}, false, nil +} + +func (f *fakeStore) Fetch(_ context.Context, id string) (*store.Download, error) { + body, ok := f.bodies[id] + if !ok { + return nil, store.ErrNotDownloadable + } + return &store.Download{Body: io.NopCloser(bytes.NewReader(body)), Filename: id + ".unitypackage"}, nil +} + +func testPackage(t *testing.T, productID, versionID string, size int) []byte { + t.Helper() + d := []byte(`{"id":"` + productID + `","version_id":"` + versionID + `"}`) + extra := []byte{'A', '$', 0, 0} + binary.LittleEndian.PutUint16(extra[2:4], uint16(len(d))) + extra = append(extra, d...) + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + zw.Header.Extra = extra + zw.Write(bytes.Repeat([]byte("x"), 32)) + zw.Close() + out := buf.Bytes() + for len(out) < size { + out = append(out, 0) + } + return out +} + +func ownedAsset(id, name, versionID string, size int64) model.Asset { + return model.Asset{ + ID: id, Name: name, State: model.StatePublished, + Publisher: model.Publisher{ID: "p1", Name: "Pub One"}, + Version: model.Version{ID: versionID, Name: "1.0"}, + AdvertisedSize: size, + } +} + +func TestStatusThenSyncThroughTheCommandLayer(t *testing.T) { + wd := isolate(t) + out := capture(t) + + a := ownedAsset("115488", "Quick Outline", "683375", 500) + manifestPath := filepath.Join(wd, manifest.FileName) + if err := manifest.Save(manifestPath, manifest.Manifest{ + Assets: []manifest.Entry{{ID: a.ID, Name: a.Name, Enabled: true}}, + }); err != nil { + t.Fatal(err) + } + fake := &fakeStore{owned: []model.Asset{a}, bodies: map[string][]byte{a.ID: testPackage(t, a.ID, "683375", 500)}} + cfg := config.Config{LibraryPath: filepath.Join(wd, "library"), Concurrency: 1} + lockPath := manifest.LockPath(manifestPath) + + // status classifies without writing anything. + code, err := syncOrStatus(context.Background(), fake, cfg, manifestPath, lockPath, "", false, true) + if code != 0 || err != nil { + t.Fatalf("status = %d, %v", code, err) + } + if !strings.Contains(out.String(), "new") { + t.Errorf("status did not classify the asset as new:\n%s", out) + } + if _, err := os.Stat(lockPath); !os.IsNotExist(err) { + t.Error("status wrote a lockfile") + } + + // sync downloads it and records it. + out.Reset() + code, err = syncOrStatus(context.Background(), fake, cfg, manifestPath, lockPath, "", false, false) + if code != 0 || err != nil { + t.Fatalf("sync = %d, %v", code, err) + } + lf, err := lockfile.Load(lockPath) + if err != nil { + t.Fatal(err) + } + _, e, ok := lf.FindByAssetID(a.ID) + if !ok || !e.Tracked { + t.Fatalf("sync did not record the asset: %+v", e) + } + pkgPath := filepath.Join(cfg.LibraryPath, "pub-one", "quick-outline-115488", "quick-outline-115488.unitypackage") + if _, err := os.Stat(pkgPath); err != nil { + t.Errorf("package not at its derived path: %v", err) + } + + // and a second status is a no-op. + out.Reset() + code, err = syncOrStatus(context.Background(), fake, cfg, manifestPath, lockPath, "", false, true) + if code != 0 || err != nil { + t.Fatalf("second status = %d, %v", code, err) + } + if !strings.Contains(out.String(), "unchanged") { + t.Errorf("second status did not report unchanged:\n%s", out) + } +} + +// select is the only command that writes the manifest, so its refusal to rewrite a +// curated file against an empty enumeration is checked at the command layer. +func TestSelectRefusesToEmptyACuratedManifest(t *testing.T) { + wd := isolate(t) + manifestPath := filepath.Join(wd, manifest.FileName) + if err := manifest.Save(manifestPath, manifest.Manifest{ + Assets: []manifest.Entry{{ID: "115488", Name: "Quick Outline", Enabled: true}}, + }); err != nil { + t.Fatal(err) + } + before, _ := os.ReadFile(manifestPath) + + err := selectAssets(context.Background(), &fakeStore{}, manifestPath, "127.0.0.1:0") + if err == nil { + t.Fatal("select rewrote the manifest against an empty enumeration") + } + after, _ := os.ReadFile(manifestPath) + if string(before) != string(after) { + t.Error("the refused select changed the manifest anyway") + } +} diff --git a/testdata/store/my_assets_p0.json b/testdata/store/my_assets_p0.json new file mode 100644 index 0000000..1e651c0 --- /dev/null +++ b/testdata/store/my_assets_p0.json @@ -0,0 +1,2011 @@ +[ + { + "data": { + "searchMyAssets": { + "results": [ + { + "product": { + "currentVersion": { + "id": "1344228", + "name": "1.10.4", + "publishedDate": "2026-08-18T02:50:14Z" + }, + "downloadSize": "410768352", + "id": "234255", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/881019ea-8af0-4d97-910f-08f50e4e22c2.png" + }, + "name": "POLYGON - Meadow Forest - Nature Biomes - 3D Environment Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1450672", + "name": "3.6.22", + "publishedDate": "2026-08-17T14:49:20Z" + }, + "downloadSize": "381108720", + "id": "271742", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/c32b83ff-43d3-40fe-b0e2-f47f4b886bf0.png" + }, + "name": "COZY: Stylized Weather 3", + "publisher": { + "id": "40676", + "name": "Distant Lands" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1451476", + "name": "1.9.2", + "publishedDate": "2026-08-18T10:34:11Z" + }, + "downloadSize": "124330544", + "id": "137126", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/e895e0c0-5864-4419-b003-7dd31d90f8dc.png" + }, + "name": "POLYGON - Prototype Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1455206", + "name": "1.03", + "publishedDate": "2026-08-19T20:54:18Z" + }, + "downloadSize": "62329520", + "id": "258357", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/540cf39e-7c51-4116-9cd2-e0a200ba2aa6.png" + }, + "name": "STYLIZED Fantasy Forge \u0026 Armory - Low Poly 3D Art", + "publisher": { + "id": "82433", + "name": "Daniel Mistage" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1455240", + "name": "1.22", + "publishedDate": "2026-08-15T11:11:09Z" + }, + "downloadSize": "45844352", + "id": "249203", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/d89ebf71-4ec1-45ef-a2fd-d23e607432d8.png" + }, + "name": "STYLIZED Fantasy Armory - Low Poly 3D Art", + "publisher": { + "id": "82433", + "name": "Daniel Mistage" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1451246", + "name": "1.4.1", + "publishedDate": "2026-08-18T03:19:10Z" + }, + "downloadSize": "118370384", + "id": "168372", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/842d003c-538f-4cb5-8457-4e260a04c181.png" + }, + "name": "POLYGON - Particle FX Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1454706", + "name": "1.0", + "publishedDate": "2022-01-16T08:28:12Z" + }, + "downloadSize": "1384007920", + "id": "211348", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/f85b17c7-b4e2-4864-b6b9-7e7d6a91ae02.png" + }, + "name": "Wyrms", + "publisher": { + "id": "10181", + "name": "Dmitriy Dryzhak" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1451100", + "name": "1.2.1", + "publishedDate": "2026-08-17T23:49:09Z" + }, + "downloadSize": "170034608", + "id": "120152", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/9fa2e075-eb09-4b6b-a28f-d0faa6253007.png" + }, + "name": "POLYGON - Nature Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1455186", + "name": "1.04", + "publishedDate": "2026-08-19T21:09:09Z" + }, + "downloadSize": "49821984", + "id": "260824", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/a337b2bd-7fd5-4382-b5dc-9a6759305fd1.png" + }, + "name": "STYLIZED Fantasy Market - Low Poly 3D Art", + "publisher": { + "id": "82433", + "name": "Daniel Mistage" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1353434", + "name": "1.6.2", + "publishedDate": "2026-08-18T04:04:10Z" + }, + "downloadSize": "148676576", + "id": "92579", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/497ec2f3-4f11-4809-a8c8-ad0690e62cfd.png" + }, + "name": "POLYGON - Pirate Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1451602", + "name": "1.2.1", + "publishedDate": "2026-08-18T10:34:12Z" + }, + "downloadSize": "114664160", + "id": "156819", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/6e243fac-4f94-471a-aab5-a129fe57f0f6.png" + }, + "name": "POLYGON - Starter Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1450236", + "name": "1.4.1", + "publishedDate": "2026-08-17T07:33:10Z" + }, + "downloadSize": "164470832", + "id": "143468", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/629f4436-4f44-4921-aca5-814597e682ad.png" + }, + "name": "POLYGON - Modular Fantasy Hero Characters Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1343060", + "name": "1.10.4", + "publishedDate": "2026-08-18T01:51:09Z" + }, + "downloadSize": "261514240", + "id": "234254", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/1ad28ece-190c-4ef3-989a-f8d784f61c5a.png" + }, + "name": "POLYGON - Swamp Marshland - Nature Biomes - 3D Environment Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1450152", + "name": "1.2.2", + "publishedDate": "2026-08-17T04:03:11Z" + }, + "downloadSize": "127717968", + "id": "202117", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/e44eb0c6-cc18-40f4-a984-b451aa3b2847.png" + }, + "name": "POLYGON - Icons Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1451122", + "name": "1.10.5", + "publishedDate": "2026-08-18T02:06:10Z" + }, + "downloadSize": "280778352", + "id": "234253", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/17df8081-ecbe-4ae9-b6d7-a9d625ab67e2.png" + }, + "name": "POLYGON - Tropical Jungle - Nature Biomes - 3D Environment Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1451572", + "name": "1.7.1", + "publishedDate": "2026-08-18T15:49:18Z" + }, + "downloadSize": "128628848", + "id": "89551", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/fa945e09-1bed-4103-9d8a-5bdd652e34cc.png" + }, + "name": "POLYGON - Samurai Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1450188", + "name": "1.6.2", + "publishedDate": "2026-08-17T05:18:12Z" + }, + "downloadSize": "138812272", + "id": "83694", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/29ce21ee-922b-4328-b2f7-3f78ed670230.png" + }, + "name": "POLYGON - Knights Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1443860", + "name": "3.8.0", + "publishedDate": "2026-08-18T11:04:24Z" + }, + "downloadSize": "15232192", + "id": "104017", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/380870e7-418c-439e-9631-ff31faf61238.png" + }, + "name": "Polyverse Skies | Low Poly Skybox Shaders", + "publisher": { + "id": "20529", + "name": "BOXOPHOBIC" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1442968", + "name": "4.9.12", + "publishedDate": "2026-08-09T21:36:10Z" + }, + "downloadSize": "161756256", + "id": "143368", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/c516c43b-9b15-4e82-bf16-e70d22fdf7cb.png" + }, + "name": "Flat Kit: Toon Shading and Water", + "publisher": { + "id": "16150", + "name": "Dustyroom" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1346974", + "name": "1.6.1", + "publishedDate": "2026-08-17T00:18:11Z" + }, + "downloadSize": "114824416", + "id": "97186", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/0b7ee5a6-1ed2-4cd5-b368-f446affa6775.png" + }, + "name": "POLYGON - Fantasy Characters Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1450234", + "name": "1.7.1", + "publishedDate": "2026-08-17T06:18:12Z" + }, + "downloadSize": "121372240", + "id": "122084", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/6e1805c7-e18a-4ccf-9107-46f6d03e709e.png" + }, + "name": "POLYGON MINI - Fantasy Character Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1450036", + "name": "1.4.2", + "publishedDate": "2026-08-17T01:02:10Z" + }, + "downloadSize": "119597840", + "id": "118399", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/07bdc399-3221-4fb3-b017-06a67fe031c3.png" + }, + "name": "POLYGON - Fantasy Rivals Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1344268", + "name": "1.4.1", + "publishedDate": "2026-08-17T06:33:10Z" + }, + "downloadSize": "119818704", + "id": "96800", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/857b27ab-7c0a-4f05-b1bc-7b9a35fa054a.png" + }, + "name": "POLYGON MINI - Fantasy Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1343046", + "name": "1.5.1", + "publishedDate": "2026-08-18T04:34:10Z" + }, + "downloadSize": "132651904", + "id": "85664", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/053ac792-3f6d-4ff4-ac05-d34d08ba7aa4.png" + }, + "name": "POLYGON - Vikings Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1446094", + "name": "1.6.1", + "publishedDate": "2026-08-13T07:00:46Z" + }, + "downloadSize": "232818896", + "id": "224020", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/ac5c83c0-716c-4b74-b9d7-f52fad39984b.png" + }, + "name": "POLYGON - Ancient Empire Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1445746", + "name": "6.0.5", + "publishedDate": "2026-07-13T17:47:09Z" + }, + "downloadSize": "94958240", + "id": "133704", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/6dd90315-d360-4f76-8069-4571aa3a9b61.png" + }, + "name": "RPG VFX Bundle", + "publisher": { + "id": "28391", + "name": "Hovl Studio" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1450038", + "name": "1.12.5", + "publishedDate": "2026-08-17T03:33:10Z" + }, + "downloadSize": "278496384", + "id": "164532", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/7e90c4ff-c0ef-482d-9400-4b2573657da9.png" + }, + "name": "POLYGON - Fantasy Kingdom Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1442862", + "name": "2.17", + "publishedDate": "2026-08-09T19:21:16Z" + }, + "downloadSize": "47344", + "id": "282348", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/683d7f6c-b78e-4d92-80dd-2f6731547265.png" + }, + "name": "RPG Animations Pack(Bundle)", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1449170", + "name": "1.3.4", + "publishedDate": "2026-08-16T21:47:09Z" + }, + "downloadSize": "10914544", + "id": "182328", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/44005ad2-d1ab-4c3c-ba1d-57d6c25a4fde.png" + }, + "name": "Grabbit - Editor Physics Transforms", + "publisher": { + "id": "36143", + "name": "Jungle" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1447634", + "name": "1.10.2", + "publishedDate": "2026-08-14T05:52:10Z" + }, + "downloadSize": "163896224", + "id": "102677", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/1564c479-107d-4789-83fb-8ea4689cc39e.png" + }, + "name": "POLYGON - Dungeons Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1442808", + "name": "1.10", + "publishedDate": "2026-08-09T19:21:10Z" + }, + "downloadSize": "297530528", + "id": "309179", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/4b877d35-7e95-4009-9a0c-6cf7b3c8e8e4.png" + }, + "name": "RPG_Animations - Two Hand Base", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1447640", + "name": "1.3.2", + "publishedDate": "2026-08-14T05:22:11Z" + }, + "downloadSize": "114301584", + "id": "143026", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/e21f218a-49b9-4a02-9d36-6227610d3a65.png" + }, + "name": "POLYGON - Fantasy Dungeon Map - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1447646", + "name": "1.4.5", + "publishedDate": "2026-08-14T07:52:31Z" + }, + "downloadSize": "186585328", + "id": "189093", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/0d96a9df-dcf7-4ac4-8062-a85cb18483f7.png" + }, + "name": "POLYGON - Dungeon Realms Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1442810", + "name": "1.6", + "publishedDate": "2026-08-09T19:36:11Z" + }, + "downloadSize": "266475040", + "id": "309180", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/785bee30-ea83-4981-8b66-26c9c0b22c54.png" + }, + "name": "RPG_Animations - Two Hand Up", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1344704", + "name": "6.0", + "publishedDate": "2026-07-19T14:48:11Z" + }, + "downloadSize": "316587600", + "id": "183370", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/0f44cacc-0406-4a7b-bb8a-63859b223e54.png" + }, + "name": "Feel", + "publisher": { + "id": "10305", + "name": "More Mountains" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1422362", + "name": "4.0.2.3", + "publishedDate": "2026-07-22T13:55:09Z" + }, + "downloadSize": "6290368", + "id": "89041", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/6497b8ed-fee7-4580-9507-54690d70fec4.png" + }, + "name": "Odin Inspector and Serializer", + "publisher": { + "id": "3727", + "name": "Sirenix" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1419322", + "name": "3.5.27", + "publishedDate": "2026-07-18T09:35:09Z" + }, + "downloadSize": "256992", + "id": "768", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3f104c5c-f7eb-470b-8201-168e80f305bf.png" + }, + "name": "Easy Save - The Complete Save Game \u0026 Data Serializer System", + "publisher": { + "id": "486", + "name": "Moodkie" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1423508", + "name": "2.6.0", + "publishedDate": "2026-07-21T13:47:10Z" + }, + "downloadSize": "219234432", + "id": "154574", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/cfa4737f-10de-4074-a1b3-8b73e9041528.png" + }, + "name": "Ultimate Clean GUI Pack", + "publisher": { + "id": "18519", + "name": "gamevanilla" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1413752", + "name": "1.8.2", + "publishedDate": "2026-07-13T07:47:21Z" + }, + "downloadSize": "124846384", + "id": "80585", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/952cc307-b8a3-4822-9f37-217dec345be2.png" + }, + "name": "POLYGON - Adventure Pack - Art by Synty", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1393792", + "name": "1.0.430", + "publishedDate": "2026-06-23T11:24:13Z" + }, + "downloadSize": "603360", + "id": "32416", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/cf534981-836c-4f21-a66a-f9e7e504bbf7.png" + }, + "name": "DOTween Pro", + "publisher": { + "id": "1341", + "name": "Demigiant" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1407502", + "name": "1.31.1", + "publishedDate": "2026-06-13T19:07:09Z" + }, + "downloadSize": "23164235200", + "id": "89126", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/fa39240f-65fb-43aa-bd99-35023d0c7112.png" + }, + "name": "Total Music Collection", + "publisher": { + "id": "4078", + "name": "Andrew Sitkov Music" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1393790", + "name": "1.3.030", + "publishedDate": "2026-06-23T10:54:10Z" + }, + "downloadSize": "243120", + "id": "27676", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/778803bc-a2db-49ff-92e2-609bbf31ac9f.png" + }, + "name": "DOTween (HOTween v2)", + "publisher": { + "id": "1341", + "name": "Demigiant" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1401782", + "name": "1.5", + "publishedDate": "2026-07-01T18:58:11Z" + }, + "downloadSize": "79283824", + "id": "309183", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/ba395a76-e85e-4d0c-b0da-07ca9d3d9f38.png" + }, + "name": "RPG_Animations - Action \u0026 Dead Pose", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1389838", + "name": "3.0.5", + "publishedDate": "2026-03-20T11:41:12Z" + }, + "downloadSize": "2836958224", + "id": "132195", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/f64a8b5f-8e7f-4728-8006-706a0515e050.png" + }, + "name": "Meadow Environment - Dynamic Nature", + "publisher": { + "id": "6887", + "name": "NatureManufacture" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1403586", + "name": "2.11", + "publishedDate": "2026-07-03T05:10:23Z" + }, + "downloadSize": "127113344", + "id": "15567", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/4137b417-21a8-4971-bb31-4430c32cca00.png" + }, + "name": "BIRDS PACK", + "publisher": { + "id": "265", + "name": "PROTOFACTOR, INC" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1393638", + "name": "2.6.4", + "publishedDate": "2026-06-23T07:09:09Z" + }, + "downloadSize": "135255472", + "id": "203178", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/247103b7-9b85-48d3-a99d-ad5fc9203d9b.png" + }, + "name": "Quibli: Anime Shaders and Tools", + "publisher": { + "id": "16150", + "name": "Dustyroom" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1384352", + "name": "1.6", + "publishedDate": "2026-06-19T21:47:09Z" + }, + "downloadSize": "179761360", + "id": "217205", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/ea71498b-df62-4322-ba53-9981a0b4b72b.png" + }, + "name": "Ivy Studio - Procedural vine generation", + "publisher": { + "id": "6503", + "name": "ARTnGAME" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1394542", + "name": "1.3.2", + "publishedDate": "2026-06-24T03:42:17Z" + }, + "downloadSize": "1792112", + "id": "159992", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/54fa1ee6-a54d-4a77-832b-588f0d5f9479.png" + }, + "name": "Jupiter - Procedural Sky Shader \u0026 Day Night Cycle", + "publisher": { + "id": "17305", + "name": "Pinwheel Studio" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1389544", + "name": "32.2.2", + "publishedDate": "2026-06-18T21:36:11Z" + }, + "downloadSize": "34980432", + "id": "225934", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/95be70a7-172e-4405-ba62-d873e18ab64d.png" + }, + "name": "Radiant Global Illumination", + "publisher": { + "id": "15018", + "name": "Kronnect" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1381398", + "name": "6.23", + "publishedDate": "2026-06-11T20:21:09Z" + }, + "downloadSize": "23453600", + "id": "158988", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/87d9a20e-2f07-4887-9f2f-619b6c1134b3.png" + }, + "name": "Sprite Shaders Ultimate", + "publisher": { + "id": "45819", + "name": "Ekincan Tas" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1387780", + "name": "1.2", + "publishedDate": "2026-06-17T14:26:13Z" + }, + "downloadSize": "68256368", + "id": "309181", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/9b294624-2a69-4f1c-9579-46ddac78f6de.png" + }, + "name": "RPG_Animations - NPC Actions", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1383192", + "name": "1.6", + "publishedDate": "2026-06-13T10:28:11Z" + }, + "downloadSize": "256785136", + "id": "309182", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/adbc82f3-cb1d-4c3a-86cb-8cf7d16df4b5.png" + }, + "name": "RPG_Animations - Torch,Lantern,Enemy Attack,Hit,Magic", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1388338", + "name": "4.4.2", + "publishedDate": "2026-06-18T00:56:22Z" + }, + "downloadSize": "14146000", + "id": "230509", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/99964151-b71a-450b-9a26-60569b0bcac0.png" + }, + "name": "Flexalon Pro: 3D \u0026 UI Layouts", + "publisher": { + "id": "72095", + "name": "Virtual Maker" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1349310", + "name": "1.6", + "publishedDate": "2026-05-28T12:12:10Z" + }, + "downloadSize": "3503953968", + "id": "229460", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/8f75c6cd-a955-487c-9466-922b4ea7b737.png" + }, + "name": "6200 Fantasy RPG Icons Pack", + "publisher": { + "id": "41787", + "name": "CraftPix" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1378814", + "name": "1.0.2", + "publishedDate": "2026-06-09T14:17:10Z" + }, + "downloadSize": "5485024", + "id": "148408", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/8834c4b6-04b5-4223-868f-d74058652051.png" + }, + "name": "IK Helper Tool", + "publisher": { + "id": "36307", + "name": "Kevin Iglesias" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1373424", + "name": "2.02", + "publishedDate": "2026-06-04T04:06:11Z" + }, + "downloadSize": "10904544", + "id": "194727", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/d8e0529c-9387-485f-9ae8-cc36be902091.png" + }, + "name": "Pixelate - Pixel Art Converter", + "publisher": { + "id": "42030", + "name": "Tom Black" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1361204", + "name": "1.9", + "publishedDate": "2026-05-23T07:57:09Z" + }, + "downloadSize": "199251504", + "id": "323439", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/cd07a668-193e-4f3f-8b67-09d36cde71a5.png" + }, + "name": "Base Move Animation", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1351146", + "name": "4.2.0", + "publishedDate": "2026-03-05T14:18:14Z" + }, + "downloadSize": "55913936", + "id": "50282", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/c311904a-5d51-4ee7-aa53-89ee2b9e7022.png" + }, + "name": "Turn Based Strategy Framework", + "publisher": { + "id": "17793", + "name": "Crooked Head" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1343516", + "name": "2.0", + "publishedDate": "2026-05-06T12:18:19Z" + }, + "downloadSize": "210158944", + "id": "162341", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/e25b18b8-9afe-45b5-b0b3-48da849d20a4.png" + }, + "name": "Human Mega Animations Pack", + "publisher": { + "id": "36307", + "name": "Kevin Iglesias" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1337694", + "name": "2.5.1", + "publishedDate": "2026-04-30T09:21:11Z" + }, + "downloadSize": "53836656", + "id": "157744", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/00a7471d-99ae-4203-8316-4aefd00e3a84.png" + }, + "name": "Human Basic Motions", + "publisher": { + "id": "36307", + "name": "Kevin Iglesias" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1353276", + "name": "1.2.48", + "publishedDate": "2026-05-15T00:08:11Z" + }, + "downloadSize": "31302416", + "id": "135594", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/4559f61c-3507-4eec-ab7e-8f3ce317c698.png" + }, + "name": "Boing Kit: Dynamic Bouncy Bones, Grass, and More", + "publisher": { + "id": "33759", + "name": "Long Bunny Labs" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1342458", + "name": "1.10", + "publishedDate": "2026-05-05T14:03:10Z" + }, + "downloadSize": "327960496", + "id": "309177", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/cb4ebbe5-0725-4126-94ba-961a4eeaeb87.png" + }, + "name": "RPG_Animations - One Hand Base", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1341510", + "name": "2.4.0", + "publishedDate": "2024-12-02T15:29:14Z" + }, + "downloadSize": "21483104", + "id": "108753", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/fbefb7dc-1b57-449e-9532-39e94c8a51e4.png" + }, + "name": "SC Post Effects Pack (for Unity 2020-2023)", + "publisher": { + "id": "15580", + "name": "Staggart Creations" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1342510", + "name": "1.10", + "publishedDate": "2026-05-06T15:51:22Z" + }, + "downloadSize": "302579376", + "id": "309178", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3f5aeb12-ea15-4191-8ecc-a540e22bdb0d.png" + }, + "name": "RPG_Animations - One Hand Up", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1322324", + "name": "10.0", + "publishedDate": "2025-10-15T23:40:18Z" + }, + "downloadSize": "258868928", + "id": "54733", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/f27170ce-59eb-419a-8ccc-78d7cbf232bb.png" + }, + "name": "Low Poly Ultimate Pack", + "publisher": { + "id": "19123", + "name": "polyperfect" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1327366", + "name": "3.7", + "publishedDate": "2026-04-19T11:32:10Z" + }, + "downloadSize": "38446560", + "id": "157187", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/e51e38cb-3dc6-4355-8a3b-94db4fae5c76.png" + }, + "name": "Easy Performant Outline 2D | 3D (URP / HDRP)", + "publisher": { + "id": "21614", + "name": "Pirate Parrot" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1341518", + "name": "1.10", + "publishedDate": "2026-05-04T13:38:11Z" + }, + "downloadSize": "245836528", + "id": "309184", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/2f9cbca5-c5c2-4d57-896a-76436eb458b3.png" + }, + "name": "RPG_Animations - Bow", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1322240", + "name": "4.1.1", + "publishedDate": "2025-08-14T00:01:12Z" + }, + "downloadSize": "130309120", + "id": "93089", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/65c27d72-90da-4717-adef-bb8fab369cd2.png" + }, + "name": "Low Poly Animated Animals", + "publisher": { + "id": "19123", + "name": "polyperfect" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1321388", + "name": "3.02", + "publishedDate": "2025-11-30T17:12:33Z" + }, + "downloadSize": "69129536", + "id": "156748", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/b4777594-7859-4ffc-ac18-19cbffad6930.png" + }, + "name": "Low Poly Animated People", + "publisher": { + "id": "19123", + "name": "polyperfect" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1317166", + "name": "1.32.1", + "publishedDate": "2026-04-10T11:54:07Z" + }, + "downloadSize": "38718320", + "id": "177023", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/12904059-22ab-4ec2-a022-14795e18fc7f.png" + }, + "name": "Map Graph", + "publisher": { + "id": "39394", + "name": "Insane Scatterbrain" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1326074", + "name": "2.5.18", + "publishedDate": "2026-04-18T08:47:09Z" + }, + "downloadSize": "1986960", + "id": "205336", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/4afcc942-a01a-44b6-9d3f-c01dc145ee03.png" + }, + "name": "SensorToolkit 2", + "publisher": { + "id": "22315", + "name": "Micosmo" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1312902", + "name": "2.6.2", + "publishedDate": "2026-04-07T14:22:27Z" + }, + "downloadSize": "96192", + "id": "112837", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/63934f4d-23d7-43ca-9941-fd29beaa0d15.png" + }, + "name": "Asset Usage Detector", + "publisher": { + "id": "22792", + "name": "yasirkula" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1292966", + "name": "1.5", + "publishedDate": "2026-03-14T11:24:29Z" + }, + "downloadSize": "31130656", + "id": "137259", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/a90f91f4-e27c-449f-be64-aa8067273389.png" + }, + "name": "Quirky Series - Animals Mega Pack Vol 1", + "publisher": { + "id": "19311", + "name": "Omabuarts Studio" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1301960", + "name": "1.5", + "publishedDate": "2026-03-23T09:56:14Z" + }, + "downloadSize": "29964416", + "id": "183280", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/dd9b99c6-4b76-48cb-b1a4-aa4bbff662f3.png" + }, + "name": "Quirky Series - Animals Mega Pack Vol 2", + "publisher": { + "id": "19311", + "name": "Omabuarts Studio" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1258142", + "name": "1.0", + "publishedDate": "2026-02-12T12:50:23Z" + }, + "downloadSize": "1216250880", + "id": "152412", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/a9f8c83e-0c18-4a35-8b23-25ec6d27db7c.png" + }, + "name": "Demonic UI 8k + Icons", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1234028", + "name": "7.2.2", + "publishedDate": "2026-01-20T06:43:16Z" + }, + "downloadSize": "154211504", + "id": "63772", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/dbc04e94-67cb-42e7-87c6-819146a48967.png" + }, + "name": "RPG Character Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1258132", + "name": "1.01", + "publishedDate": "2026-02-12T22:05:13Z" + }, + "downloadSize": "610242160", + "id": "193841", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3c71011f-adfe-4190-8f27-e4daa7bc3f74.png" + }, + "name": "Classic Fantasy RPG - UI Kit", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1267174", + "name": "1.6", + "publishedDate": "2026-02-04T10:01:06Z" + }, + "downloadSize": "25414870", + "id": "183075", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/b2af1f9b-9e5d-41cd-8120-91af030eeb72.png" + }, + "name": "Unity Learn | Foundations of Real-Time Audio | URP", + "publisher": { + "id": "1", + "name": "Unity Technologies" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1257540", + "name": "1.03", + "publishedDate": "2026-02-12T01:37:10Z" + }, + "downloadSize": "2807614464", + "id": "180688", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/c2c3f43d-c445-42d6-957a-d572e631f64c.png" + }, + "name": "Medieval Kingdom UI", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1225076", + "name": "1.99", + "publishedDate": "2026-01-12T13:09:39Z" + }, + "downloadSize": "29543760", + "id": "111398", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/ab225426-93f1-4771-bc6e-61c07443e66f.png" + }, + "name": "Character Creator 2D", + "publisher": { + "id": "34299", + "name": "Simpleton" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1214888", + "name": "1.3", + "publishedDate": "2026-01-02T16:14:16Z" + }, + "downloadSize": "57083264", + "id": "135882", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/7969defd-9f66-4bfe-971d-94ead7d16503.png" + }, + "name": "Gothic UI", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1216840", + "name": "1.2", + "publishedDate": "2026-01-05T09:29:12Z" + }, + "downloadSize": "50304", + "id": "319643", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/432f1fb1-c37d-4c20-b81e-4c3686b931ca.png" + }, + "name": "Easy FBX Anim Modifier", + "publisher": { + "id": "99592", + "name": "DoubleL" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1218204", + "name": "1.0", + "publishedDate": "2018-08-02T22:51:19Z" + }, + "downloadSize": "404090272", + "id": "123499", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/788cfe1b-dc4c-434f-8ae0-dda3674802c6.png" + }, + "name": "Citadel RPG GUI", + "publisher": { + "id": "5183", + "name": "Kodiak Graphics" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1215720", + "name": "22.1.3", + "publishedDate": "2025-08-05T15:08:15Z" + }, + "downloadSize": "19373744", + "id": "134149", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/5e77cd21-c32b-437c-95a2-ff69748fd193.png" + }, + "name": "Highlight Plus - All in One Outline \u0026 Selection Effects", + "publisher": { + "id": "15018", + "name": "Kronnect" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "1156730", + "name": "1.0", + "publishedDate": "2019-12-18T08:05:10Z" + }, + "downloadSize": "7021840", + "id": "159068", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/2deaf83f-722d-4284-bf41-841a42321cfd.png" + }, + "name": "GUI Parts", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1158758", + "name": "2.1", + "publishedDate": "2025-11-05T00:02:14Z" + }, + "downloadSize": "383190928", + "id": "112002", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3bf08591-4a60-4838-acb0-f8225b9ec6aa.png" + }, + "name": "Sci-Fi Turret Constructor", + "publisher": { + "id": "12193", + "name": "Slava Z." + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1129684", + "name": "2.1.1", + "publishedDate": "2025-05-07T04:35:46Z" + }, + "downloadSize": "6056048", + "id": "47302", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/89ceeeb0-f8f2-468f-8a5a-f59cc5ffb1d6.png" + }, + "name": "Crossbow Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1106040", + "name": "1.0.9", + "publishedDate": "2025-08-29T18:56:15Z" + }, + "downloadSize": "1040271056", + "id": "215197", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3e3599d8-d9de-4152-92e5-46bdbfaa4706.png" + }, + "name": "Toon Fantasy Nature", + "publisher": { + "id": "18116", + "name": "SICS Games" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1146608", + "name": "1.0.1", + "publishedDate": "2025-05-03T08:36:12Z" + }, + "downloadSize": "52608", + "id": "190683", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/0443bc51-6608-4849-8156-e65928e46f82.png" + }, + "name": "Warrior Pack Super Bundle", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1129476", + "name": "2.6.1", + "publishedDate": "2025-05-07T05:05:13Z" + }, + "downloadSize": "6669808", + "id": "35101", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/aa08f459-41e1-4ade-8e21-c09531e17655.png" + }, + "name": "Karate Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1127882", + "name": "2.0.1", + "publishedDate": "2025-05-07T04:35:34Z" + }, + "downloadSize": "6311296", + "id": "42286", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/e9b295a8-585f-4c4c-a045-2fa73d436c79.png" + }, + "name": "2-Handed Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1129470", + "name": "2.4.1", + "publishedDate": "2025-05-07T04:35:40Z" + }, + "downloadSize": "5948256", + "id": "35577", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3fc9921f-3766-4cb4-92c8-c3d371826851.png" + }, + "name": "Brute Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1129670", + "name": "2.8.1", + "publishedDate": "2025-05-07T10:35:17Z" + }, + "downloadSize": "8427008", + "id": "35814", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3b435b1d-6ddc-4691-b011-d4ea8450c163.png" + }, + "name": "Sorceress Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1129674", + "name": "2.3.1", + "publishedDate": "2025-05-07T04:51:34Z" + }, + "downloadSize": "6106288", + "id": "43153", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/56dc3ae6-85b5-4aae-a4f0-c44578682b85.png" + }, + "name": "Swordsman Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1130712", + "name": "3.2.1", + "publishedDate": "2025-05-07T04:51:27Z" + }, + "downloadSize": "10606272", + "id": "35307", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/027c15f6-ce99-4eb5-85c2-46162b04d6e8.png" + }, + "name": "Ninja Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1127880", + "name": "2.2.1", + "publishedDate": "2025-05-07T04:35:27Z" + }, + "downloadSize": "6177360", + "id": "41714", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/b942e066-73b9-4774-a1ad-ae92e1714e58.png" + }, + "name": "Archer Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1129478", + "name": "2.7.1", + "publishedDate": "2025-05-07T04:51:11Z" + }, + "downloadSize": "7166592", + "id": "38814", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/7d486bbe-371d-42c5-87fc-5586af73925f.png" + }, + "name": "Knight Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1129472", + "name": "2.1.1", + "publishedDate": "2025-05-07T04:35:15Z" + }, + "downloadSize": "6568624", + "id": "46860", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/512458a2-9782-40dc-a2e6-5b7895b88975.png" + }, + "name": "Hammer Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1129480", + "name": "2.4.1", + "publishedDate": "2025-05-07T04:51:19Z" + }, + "downloadSize": "6424448", + "id": "39519", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/acd458c4-960d-456e-a250-626bb4e0cec1.png" + }, + "name": "Mage Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1129672", + "name": "2.2.1", + "publishedDate": "2025-05-07T04:51:29Z" + }, + "downloadSize": "6182048", + "id": "46399", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/d2ac582e-b6d1-4997-bda3-ca1368e4516a.png" + }, + "name": "Spearman Warrior Mecanim Animation Pack", + "publisher": { + "id": "3415", + "name": "Explosive" + }, + "state": "published" + } + } + ], + "total": 176 + } + } + } +] diff --git a/testdata/store/my_assets_p1.json b/testdata/store/my_assets_p1.json new file mode 100644 index 0000000..48c0817 --- /dev/null +++ b/testdata/store/my_assets_p1.json @@ -0,0 +1,1531 @@ +[ + { + "data": { + "searchMyAssets": { + "results": [ + { + "product": { + "currentVersion": { + "id": "1095832", + "name": "4.0", + "publishedDate": "2025-08-06T16:53:13Z" + }, + "downloadSize": "211058768", + "id": "64248", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/e4bf5216-4c2f-4720-940a-0f21979d149e.png" + }, + "name": "Little Dragons: Tiger", + "publisher": { + "id": "16163", + "name": "MalberS Animations" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1113190", + "name": "3.5.2", + "publishedDate": "2025-09-10T11:12:15Z" + }, + "downloadSize": "63142032", + "id": "266535", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/d680075a-394b-4ec6-b757-377a24ced8b8.png" + }, + "name": "Body Proportions", + "publisher": { + "id": "59523", + "name": "OnlyNew Studio" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1080711", + "name": "1.0.1", + "publishedDate": "2025-06-23T08:52:15Z" + }, + "downloadSize": "2903760", + "id": "148410", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/276b62dd-b4dd-4b81-a261-967e651e90fd.png" + }, + "name": "RPG Poly Pack - Lite", + "publisher": { + "id": "42095", + "name": "Gigel3d" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1094273", + "name": "1.0.0", + "publishedDate": "2024-03-20T09:34:05Z" + }, + "downloadSize": "12640", + "id": "262163", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/6e2d5d2a-4487-4080-a729-0938710fbe10.png" + }, + "name": "UI Toolkit Bundle 1", + "publisher": { + "id": "37829", + "name": "KAMGAM" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1069514", + "name": "1.0", + "publishedDate": "2025-06-10T09:44:12Z" + }, + "downloadSize": "159429792", + "id": "179083", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/31f22bf7-395a-4cbd-a20d-f52776445348.png" + }, + "name": "Monsters Ultimate Pack 02 Cute Series", + "publisher": { + "id": "3867", + "name": "Meshtint Studio" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "637666", + "name": "1.6", + "publishedDate": "2021-08-26T15:57:14Z" + }, + "downloadSize": "87409280", + "id": "61157", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/2e3bc354-323c-459d-baca-ac98b9e4e530.png" + }, + "name": "Underwater FX", + "publisher": { + "id": "2575", + "name": "Rivermill Studios" + }, + "state": "disabled" + } + }, + { + "product": { + "currentVersion": { + "id": "643695", + "name": "1.0", + "publishedDate": "2021-05-17T09:41:09Z" + }, + "downloadSize": "121657712", + "id": "193760", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/1a0a6a99-02ac-488d-a385-5c41b67cfe50.png" + }, + "name": "Fantasy Sounds Bundle", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "disabled" + } + }, + { + "product": { + "currentVersion": { + "id": "1078634", + "name": "1.0", + "publishedDate": "2018-10-01T13:07:06Z" + }, + "downloadSize": "625728", + "id": "127775", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/27d23e8e-6446-4e05-8277-2a8d7b64d32f.png" + }, + "name": "Low Poly Fantasy Warrior", + "publisher": { + "id": "38620", + "name": "asoliddev" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1058207", + "name": "3.42", + "publishedDate": "2025-05-01T21:20:15Z" + }, + "downloadSize": "399312", + "id": "96925", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/809f22b5-3b8a-466c-ace1-13d9ae3e545f.png" + }, + "name": "Smooth Sync", + "publisher": { + "id": "20301", + "name": "Noble Whale Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1063362", + "name": "2.0.0", + "publishedDate": "2024-04-15T22:16:13Z" + }, + "downloadSize": "548052304", + "id": "163280", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/9d0f079a-ff91-4e84-ad85-d7e947c0f9d5.png" + }, + "name": "5000 Fantasy Icons", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1017477", + "name": "1.2.1", + "publishedDate": "2025-02-03T18:45:11Z" + }, + "downloadSize": "5410304", + "id": "205222", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/9bd43fdc-bee1-4832-9f07-4e461ae6b1f2.png" + }, + "name": "Pixel Art Full GUI / UI Kit + 151 icons!", + "publisher": { + "id": "5245", + "name": "HONETi" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1016949", + "name": "1.0.3", + "publishedDate": "2025-01-29T17:02:22Z" + }, + "downloadSize": "3040904275", + "id": "213197", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/0f944911-416f-474a-9073-fc872015a707.png" + }, + "name": "Unity Terrain - URP Demo Scene", + "publisher": { + "id": "1", + "name": "Unity Technologies" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1056339", + "name": "0.1.0", + "publishedDate": "2023-09-14T14:37:11Z" + }, + "downloadSize": "48544", + "id": "262495", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/b747be4c-fca1-40b5-8f90-ad34df0f8846.png" + }, + "name": "Utility USS", + "publisher": { + "id": "89401", + "name": "Robby K" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1030373", + "name": "1.12.13", + "publishedDate": "2024-01-22T01:19:13Z" + }, + "downloadSize": "24521904", + "id": "160144", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/54ff8831-e6df-4f10-8508-1c07ebe3ef84.png" + }, + "name": "Magica Cloth", + "publisher": { + "id": "36991", + "name": "Magica Soft" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "1047025", + "name": "1.5", + "publishedDate": "2025-04-06T09:59:11Z" + }, + "downloadSize": "50476272", + "id": "152053", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/e30cc1e2-eedb-4d96-b369-139bbf132115.png" + }, + "name": "RPG \u0026 MMO UI X", + "publisher": { + "id": "3533", + "name": "Evil" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "979796", + "name": "1.8.2", + "publishedDate": "2024-10-21T07:08:12Z" + }, + "downloadSize": "1883744", + "id": "177877", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/ecb49962-d7c2-4063-990f-7e5d3e32fec0.png" + }, + "name": "ProPixelizer - Realtime 3D Pixel Art", + "publisher": { + "id": "49852", + "name": "ElliotB256" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "998405", + "name": "1.692", + "publishedDate": "2024-12-09T13:43:00Z" + }, + "downloadSize": "272226032", + "id": "231178", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/c7c08c5e-f345-46cf-b7b3-9eb01cdf12d1.png" + }, + "name": "Dragon Crashers - UI Toolkit Sample project", + "publisher": { + "id": "1", + "name": "Unity Technologies" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "1008063", + "name": "4.0", + "publishedDate": "2025-01-07T17:11:32Z" + }, + "downloadSize": "238291072", + "id": "186580", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/6b3bdab9-8714-4a93-9680-2b5737649c18.png" + }, + "name": "Easy AR : Make Awesome AR Apps Without Coding", + "publisher": { + "id": "47462", + "name": "Render Island" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "947306", + "name": "1.2.0", + "publishedDate": "2024-07-20T20:55:10Z" + }, + "downloadSize": "335387776", + "id": "146014", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/820684d8-011d-4ec4-967a-76253d48f6d2.png" + }, + "name": "AllSky Free - 10 Sky / Skybox Set", + "publisher": { + "id": "3830", + "name": "rpgwhitelock" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "918644", + "name": "1.1", + "publishedDate": "2021-07-19T10:34:35Z" + }, + "downloadSize": "91462928", + "id": "160253", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/4a9c5254-9c3e-4161-b496-409ffefdcd2c.png" + }, + "name": "Classic RPG GUI", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "965924", + "name": "1.4.1", + "publishedDate": "2024-09-13T17:03:09Z" + }, + "downloadSize": "44284368", + "id": "251843", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/648debc1-9e39-4ed4-8f95-d704bc6e46d2.png" + }, + "name": "URP+ 2022 - Improved Universal Render Pipeline", + "publisher": { + "id": "46006", + "name": "ShadowShardTools" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "895314", + "name": "1.4.0", + "publishedDate": "2023-01-03T08:29:59Z" + }, + "downloadSize": "21649024", + "id": "221125", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/fcda904d-5100-470c-99a1-629ce2105f0b.png" + }, + "name": "Skill \u0026 Attack Indicators", + "publisher": { + "id": "35054", + "name": "DTT" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "928824", + "name": "1.0", + "publishedDate": "2022-10-14T11:24:16Z" + }, + "downloadSize": "928667344", + "id": "234071", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/5ffa31e9-ad6c-4732-ba5b-ba71a4fb27ac.png" + }, + "name": "Fantasy RPG Music Collection - Fallen Kingdom", + "publisher": { + "id": "55342", + "name": "Marcin Szmuc" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "875570", + "name": "1.1.2", + "publishedDate": "2023-11-28T16:07:12Z" + }, + "downloadSize": "1118524096", + "id": "89624", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3d1c2570-4f74-47f5-9e34-cc80bb0889fd.png" + }, + "name": "SciFi Space Base", + "publisher": { + "id": "3021", + "name": "Daelonik Artworks" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "905707", + "name": "1.0.2", + "publishedDate": "2023-11-15T23:00:18Z" + }, + "downloadSize": "2184784", + "id": "81692", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/80d8b213-bfc7-4fad-8559-60bb18cb95c9.png" + }, + "name": "Simple Forest Animals - Cartoon Assets", + "publisher": { + "id": "5217", + "name": "Synty Studios" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "898246", + "name": "1.4", + "publishedDate": "2020-01-17T09:57:12Z" + }, + "downloadSize": "191259600", + "id": "136564", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/50b90022-5d84-4028-a2be-1f4784fd0baa.png" + }, + "name": "500 Skill Icons", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "858190", + "name": "1.3", + "publishedDate": "2023-06-27T10:15:28Z" + }, + "downloadSize": "95393792", + "id": "221389", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/d2ebdca7-60b8-4529-a2b3-5abb933f3436.png" + }, + "name": "Brute Force - Snow \u0026 Ice Shader", + "publisher": { + "id": "52742", + "name": "BRUTE FORCE STUDIO" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "872804", + "name": "1.2", + "publishedDate": "2023-06-16T21:47:12Z" + }, + "downloadSize": "32205840", + "id": "207966", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/273a2497-efe9-43c4-ae58-fd028269798b.png" + }, + "name": "Shiny Items VFX for URP", + "publisher": { + "id": "57633", + "name": "Paul Jewell" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "856476", + "name": "1.9.5", + "publishedDate": "2023-08-01T08:40:16Z" + }, + "downloadSize": "2299743520", + "id": "115747", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/2297589a-1306-4ed0-a48b-ef19207cef5c.png" + }, + "name": "Unity Learn | 3D Game Kit", + "publisher": { + "id": "1", + "name": "Unity Technologies" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "847756", + "name": "1.1", + "publishedDate": "2023-09-02T19:30:26Z" + }, + "downloadSize": "607672928", + "id": "213593", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/dec49d89-714f-4aa2-9c1a-b10b0f874e0e.png" + }, + "name": "Fantasy Map Creator", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "888774", + "name": "1.0", + "publishedDate": "2019-03-27T13:25:08Z" + }, + "downloadSize": "1338836112", + "id": "142554", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/44dcbfa1-2a21-452f-a85e-9a1e5feab231.png" + }, + "name": "Forest animals", + "publisher": { + "id": "12623", + "name": "Red Deer" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "878127", + "name": "1.34", + "publishedDate": "2023-12-15T13:55:11Z" + }, + "downloadSize": "356912", + "id": "235062", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/592efcd1-bca5-4efd-a6f3-77b506a1948d.png" + }, + "name": "Exporter for Unreal to Unity 2023", + "publisher": { + "id": "26243", + "name": "Ciprian Stanciu" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "895993", + "name": "1.6", + "publishedDate": "2024-02-20T15:31:44Z" + }, + "downloadSize": "461868736", + "id": "191447", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/dd972b1b-c6d2-4518-abab-83610fc05aba.png" + }, + "name": "5592 Fantasy RPG Icons", + "publisher": { + "id": "41787", + "name": "CraftPix" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "799231", + "name": "1.0", + "publishedDate": "2023-04-03T13:03:13Z" + }, + "downloadSize": "82775552", + "id": "198411", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3ff9f3e5-5f90-4bd3-a9f5-8de61174630e.png" + }, + "name": "Party Monster Rumble PBR", + "publisher": { + "id": "23554", + "name": "Dungeon Mason" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "830668", + "name": "2.0", + "publishedDate": "2023-07-10T14:08:52Z" + }, + "downloadSize": "326481040", + "id": "64768", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/4125a11c-e529-4fe1-980a-75a5cb842fdb.png" + }, + "name": "Skill And Ability Icons", + "publisher": { + "id": "21903", + "name": "Yusuf Artun" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "848851", + "name": "1.1.1", + "publishedDate": "2023-09-06T16:07:11Z" + }, + "downloadSize": "8264192", + "id": "161366", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/19ae8c50-d16c-4b3c-b276-88f4fdc591f1.png" + }, + "name": "Seamless Texture Generator", + "publisher": { + "id": "37262", + "name": "FImpossible Creations" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "743936", + "name": "1.07", + "publishedDate": "2023-07-10T14:28:13Z" + }, + "downloadSize": "28805888", + "id": "154032", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/1411cc37-cc9b-420d-95ab-9f899d84b867.png" + }, + "name": "Powerslide Kart Physics", + "publisher": { + "id": "4634", + "name": "JustInvoke" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "822628", + "name": "1.7.6a", + "publishedDate": "2023-06-14T03:10:18Z" + }, + "downloadSize": "23661984", + "id": "44361", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/cfe8a505-1adf-460d-bef4-27cb2d41c9ea.png" + }, + "name": "ProTips - Tooltip System", + "publisher": { + "id": "7702", + "name": "ModelShark Studio" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "773259", + "name": "2.1", + "publishedDate": "2023-01-13T22:52:11Z" + }, + "downloadSize": "6291278720", + "id": "151756", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/b3772630-55e9-4a53-ab7c-73e3d7ddf618.png" + }, + "name": "Ultimate Sound FX Bundle", + "publisher": { + "id": "39089", + "name": "Sidearm Studios" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "773585", + "name": "1.0", + "publishedDate": "2023-01-14T22:41:11Z" + }, + "downloadSize": "130365232", + "id": "176744", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/a0ee1a91-3926-4b05-b955-eed7630ad2b0.png" + }, + "name": "Monster Sounds Pack", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "659998", + "name": "1.0.6", + "publishedDate": "2023-02-14T11:13:13Z" + }, + "downloadSize": "972558697", + "id": "208063", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/53239341-b683-41d4-83d8-3b270a763d31.png" + }, + "name": "Interactive Guide: Cross-Platform UI", + "publisher": { + "id": "1", + "name": "Unity Technologies" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "683375", + "name": "1.1", + "publishedDate": "2022-03-07T16:46:24Z" + }, + "downloadSize": "33824", + "id": "115488", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3c58effc-acb7-48f1-9da3-e760bba1afdc.png" + }, + "name": "Quick Outline", + "publisher": { + "id": "35813", + "name": "Chris Nolet" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "665322", + "name": "4.3", + "publishedDate": "2022-01-04T09:32:16Z" + }, + "downloadSize": "288215056", + "id": "103633", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/0ff283e5-baff-4a39-9854-3e46bec419eb.png" + }, + "name": "Skybox Series Free", + "publisher": { + "id": "31837", + "name": "Avionx" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "699375", + "name": "1.1.1", + "publishedDate": "2022-04-29T06:06:15Z" + }, + "downloadSize": "1799677570", + "id": "29140", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/b48a0552-cab7-479d-b73c-801ce3a16bf5.png" + }, + "name": "Viking Village URP", + "publisher": { + "id": "1", + "name": "Unity Technologies" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "737537", + "name": "1.5.3", + "publishedDate": "2022-09-15T16:17:12Z" + }, + "downloadSize": "2956018592", + "id": "65780", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/735b81ed-59ce-4ae4-a213-c148773e2527.png" + }, + "name": "Eternal Temple", + "publisher": { + "id": "12379", + "name": "Mana Station" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "761161", + "name": "1.0", + "publishedDate": "2022-11-27T21:00:17Z" + }, + "downloadSize": "348447648", + "id": "154257", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/1e9faf26-3081-477f-82ba-6f8a96189774.png" + }, + "name": "Sci-Fi Sound Pack", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "712205", + "name": "1.5.0", + "publishedDate": "2022-06-16T11:23:14Z" + }, + "downloadSize": "336057360", + "id": "73563", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/2c18f034-f017-4320-b66e-9637559cb62b.png" + }, + "name": "Lighting Optimisation Tutorial", + "publisher": { + "id": "1", + "name": "Unity Technologies" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "778177", + "name": "1.0", + "publishedDate": "2023-01-29T21:28:12Z" + }, + "downloadSize": "101036256", + "id": "201671", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/b2245728-ec37-47ef-b125-6813a4d1833f.png" + }, + "name": "Human Vocal Sounds (PRO)", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "729133", + "name": "1.2.1", + "publishedDate": "2022-08-15T14:24:14Z" + }, + "downloadSize": "6956576", + "id": "162025", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/f98cab8f-151b-48ad-9a31-894593cda6a5.png" + }, + "name": "Stylized Water For URP", + "publisher": { + "id": "4931", + "name": "Alexander Ameye" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "758521", + "name": "1.0", + "publishedDate": "2022-11-22T23:02:13Z" + }, + "downloadSize": "4681200", + "id": "151985", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/a8940c88-062c-4d9e-be01-0f5c3563ef45.png" + }, + "name": "Weapon Sounds", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "758522", + "name": "1.0", + "publishedDate": "2022-11-22T23:00:19Z" + }, + "downloadSize": "3630992", + "id": "153777", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/1d9da85a-c559-4bb9-b67e-fc6b4f6a194c.png" + }, + "name": "Shield Sounds", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "694429", + "name": "2.0.0", + "publishedDate": "2022-04-13T14:18:15Z" + }, + "downloadSize": "209465539", + "id": "174461", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/a6c909f0-6db2-42a4-9044-70cee8623d0d.png" + }, + "name": "Tiling Textures - 3D Microgame Add-Ons", + "publisher": { + "id": "1", + "name": "Unity Technologies" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "703276", + "name": "5.1.0", + "publishedDate": "2022-05-16T11:17:13Z" + }, + "downloadSize": "133409047", + "id": "25422", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/ee3d744e-6d87-4dd3-8815-9e2aa85f6777.png" + }, + "name": "Shader Calibration Scene", + "publisher": { + "id": "1", + "name": "Unity Technologies" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "649555", + "name": "1.0", + "publishedDate": "2020-05-07T10:24:11Z" + }, + "downloadSize": "77948704", + "id": "165660", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/d4f24e01-7f9d-4935-9b03-5197e02abf35.png" + }, + "name": "Footsteps Sound Pack", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "563902", + "name": "1.0", + "publishedDate": "2020-10-14T15:46:16Z" + }, + "downloadSize": "75408912", + "id": "180881", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/085cc0a8-1324-4dc8-9d37-86627ed65709.png" + }, + "name": "300+ RPG Flat Icons Pack", + "publisher": { + "id": "34661", + "name": "Kat Grabowska" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "709653", + "name": "1.0", + "publishedDate": "2022-06-08T11:26:21Z" + }, + "downloadSize": "21301264", + "id": "224138", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/3d826d36-dcd4-45a5-a0dd-ddbbd36377e9.png" + }, + "name": "RPG Flat Skills 05", + "publisher": { + "id": "67736", + "name": "A-ravlik" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "503516", + "name": "1.01", + "publishedDate": "2020-02-06T08:11:15Z" + }, + "downloadSize": "117497872", + "id": "82713", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/6e9c83dd-cfd3-4457-a128-bfbe3b11b994.png" + }, + "name": "Flat Skills Icons", + "publisher": { + "id": "13229", + "name": "REXARD" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "656436", + "name": "1.6.3", + "publishedDate": "2021-11-20T18:49:13Z" + }, + "downloadSize": "2987776", + "id": "80667", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/5865c88c-1b3a-48b2-8b3b-c392a8dcd32f.png" + }, + "name": "Soft Mask", + "publisher": { + "id": "26352", + "name": "Oleg Knyazev" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "568583", + "name": "1.2", + "publishedDate": "2020-11-04T18:22:16Z" + }, + "downloadSize": "20175488", + "id": "180105", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/6407ee40-ca4f-40c9-b371-a8c1587481fe.png" + }, + "name": "Stylized RPG Cursors", + "publisher": { + "id": "47074", + "name": "Lid Games" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "633111", + "name": "1.0", + "publishedDate": "2021-05-31T11:51:11Z" + }, + "downloadSize": "893115456", + "id": "194725", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/f4574f0a-bb29-41a2-80c3-f0d917ad33f4.png" + }, + "name": "Fantasy Music Pack Vol 1", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "627207", + "name": "1.0", + "publishedDate": "2021-06-23T12:49:10Z" + }, + "downloadSize": "160195248", + "id": "196149", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/461be42e-63f2-464d-b357-3d337f5fa672.png" + }, + "name": "Fire Sounds Pack", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "544739", + "name": "1.2", + "publishedDate": "2020-07-22T07:23:39Z" + }, + "downloadSize": "14557680", + "id": "150771", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/921b8cf0-7be6-43a6-a6e3-2d8cde538a8e.png" + }, + "name": "Classic UI icons", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "482588", + "name": "1.0", + "publishedDate": "2019-10-17T09:21:09Z" + }, + "downloadSize": "655103888", + "id": "155819", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/a16f1332-ed91-40ff-8e21-c13c46237481.png" + }, + "name": "Flat Icons Megapack", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "610037", + "name": "1.7", + "publishedDate": "2020-08-26T12:22:16Z" + }, + "downloadSize": "9317856", + "id": "129638", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/6665b031-8c41-4aab-a4b3-346c1703b3fa.png" + }, + "name": "Fast Mobile Post Processing: Color Correction(LUT), Blur, Bloom ( URP , VR , AR , LWRP )", + "publisher": { + "id": "39057", + "name": "Rufat's ShaderLab" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "536726", + "name": "1.5", + "publishedDate": "2020-06-22T09:29:20Z" + }, + "downloadSize": "1584368", + "id": "120049", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/5a668076-acd7-4b52-9972-609650c5979b.png" + }, + "name": "Fantasy RPG Cursors (silver)", + "publisher": { + "id": "11690", + "name": "Leonid Deburger" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "534904", + "name": "1.0", + "publishedDate": "2020-06-30T08:47:10Z" + }, + "downloadSize": "1049008", + "id": "173127", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/a12bace8-8c8f-48f6-90fd-aa20444ab0aa.png" + }, + "name": "Low Poly Hex Tiles Vol.2 (Dungeons)", + "publisher": { + "id": "28031", + "name": "InvasionFields" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "547620", + "name": "1.0", + "publishedDate": "2020-03-13T08:51:12Z" + }, + "downloadSize": "266485824", + "id": "163516", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/35b850b9-15e2-41b6-8194-a30d0107a680.png" + }, + "name": "Water Sounds Pack", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "550643", + "name": "1.0", + "publishedDate": "2019-03-20T10:07:09Z" + }, + "downloadSize": "25998768", + "id": "141915", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/ce59d9fb-a75f-487f-b303-8fb2cb9e19d4.png" + }, + "name": "Dragon Mobile UI", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "544945", + "name": "1.2", + "publishedDate": "2020-07-23T08:29:11Z" + }, + "downloadSize": "27039984", + "id": "164168", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/08cdfd01-8682-40ea-9e86-a2c2b21e34ad.png" + }, + "name": "Attribute Icons", + "publisher": { + "id": "38930", + "name": "PONETI" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "532727", + "name": "1.0", + "publishedDate": "2020-09-15T07:22:11Z" + }, + "downloadSize": "43239168", + "id": "171260", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/a9ad2632-eeea-4c14-902e-96161819dd06.png" + }, + "name": "Rock Sounds Pack", + "publisher": { + "id": "16881", + "name": "Cafofo" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "453085", + "name": "1.2", + "publishedDate": "2019-06-26T07:26:12Z" + }, + "downloadSize": "674272", + "id": "136082", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/c8981f98-b1b1-4d5f-9995-a3e09f40184b.png" + }, + "name": "Bézier Path Creator", + "publisher": { + "id": "40478", + "name": "Sebastian Lague" + }, + "state": "deprecated" + } + }, + { + "product": { + "currentVersion": { + "id": "513402", + "name": "2.0", + "publishedDate": "2015-07-14T21:29:53Z" + }, + "downloadSize": "33239824", + "id": "15257", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/23bda8fe-db8c-464a-9253-f48ae97e04f0.png" + }, + "name": "Gothic RPG Buttons", + "publisher": { + "id": "5183", + "name": "Kodiak Graphics" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "370031", + "name": "1.0", + "publishedDate": "2018-07-10T17:50:27Z" + }, + "downloadSize": "2563680", + "id": "120952", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/76a44737-ab2d-4e29-bd79-d51bdac4cb3d.png" + }, + "name": "Low Poly Hex Tiles Vol.1", + "publisher": { + "id": "28031", + "name": "InvasionFields" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "478972", + "name": "1.0", + "publishedDate": "2019-10-29T05:57:12Z" + }, + "downloadSize": "1038320", + "id": "156221", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/2a64dd49-4ded-4340-9b91-46f1f57489d8.png" + }, + "name": "TBS Stylized Hex and Platforms", + "publisher": { + "id": "28031", + "name": "InvasionFields" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "352735", + "name": "1.6", + "publishedDate": "2018-04-13T17:49:33Z" + }, + "downloadSize": "281968", + "id": "69448", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/ab0e423e-ab31-40b7-aa35-9f3d35823d18.png" + }, + "name": "Transition Blocks", + "publisher": { + "id": "23166", + "name": "ATesh Games" + }, + "state": "published" + } + }, + { + "product": { + "currentVersion": { + "id": "161469", + "name": "1.0", + "publishedDate": "2016-01-22T22:00:32Z" + }, + "downloadSize": "99312", + "id": "43321", + "mainImage": { + "icon75": "//assetstorev1-prd-cdn.unity3d.com/key-image/71c45fa0-df9f-4919-9957-e508dd77f146.png" + }, + "name": "RTS camera", + "publisher": { + "id": "12535", + "name": "Denis Sylkin" + }, + "state": "deprecated" + } + } + ], + "total": 176 + } + } + } +] diff --git a/testdata/store/my_assets_p2.json b/testdata/store/my_assets_p2.json new file mode 100644 index 0000000..685d6a3 --- /dev/null +++ b/testdata/store/my_assets_p2.json @@ -0,0 +1,10 @@ +[ + { + "data": { + "searchMyAssets": { + "results": [], + "total": 176 + } + } + } +] diff --git a/testdata/store/package_header_sample.bin b/testdata/store/package_header_sample.bin new file mode 100644 index 0000000..c2412e1 Binary files /dev/null and b/testdata/store/package_header_sample.bin differ diff --git a/unity-sync.example.toml b/unity-sync.example.toml new file mode 100644 index 0000000..7dbdc96 --- /dev/null +++ b/unity-sync.example.toml @@ -0,0 +1,17 @@ +# Example PROJECT manifest. Commit this (as unity-sync.toml) in the repo that consumes the +# assets. unity-sync finds it by walking up from the working directory, or point at it +# with --manifest. Its lockfile is written beside it as unity-sync.lock.json. +# +# This file is project-scoped and carries no account identity: your session stays in the +# user config (~/.config/unity-sync/config.toml). +# +# `unity-sync select` manages these entries through a local page; hand-editing is fine. +# New assets land disabled, so buying one never downloads it behind your back. +# +# Entries key on `id`, not `name`: a publisher renaming their asset must not silently +# deselect it. + +# [[asset]] +# id = "115488" +# name = "Quick Outline" +# enabled = true