diff --git a/.github/scripts/clean_wheel_agent_acceptance.py b/.github/scripts/clean_wheel_agent_acceptance.py index cd9b85d..7655876 100644 --- a/.github/scripts/clean_wheel_agent_acceptance.py +++ b/.github/scripts/clean_wheel_agent_acceptance.py @@ -197,6 +197,52 @@ def _dataset_query( return scope +def _event_parity(crossing: list[dict[str, Any]], reference: list[dict[str, Any]]) -> None: + """Compare stable event values and validate dataset-only ordered evidence.""" + fields = ( + "id", + "action", + "address", + "size", + "stream", + "allocated", + "active", + "reserved", + "callstack", + ) + _require( + len(crossing) == len(reference) == 2 + and all(all(key in row for key in fields) for row in [*crossing, *reference]), + "Cross-shard real events have missing rows or fields", + ) + _require( + [{key: row[key] for key in fields} for row in crossing] + == [{key: row[key] for key in fields} for row in reference], + "Cross-shard real events differ from standalone", + ) + for row in crossing: + frames = row.get("frames") + _require( + row.get("frames_status") == "ordered" + and isinstance(frames, list) + and all( + isinstance(frame, dict) + and isinstance(frame.get("filename"), str) + and type(frame.get("line")) is int + and isinstance(frame.get("name"), str) + for frame in frames + ), + "Cross-shard ordered frame evidence is incomplete", + ) + _require( + "\n".join( + f"{frame['filename']}:{frame['line']} {frame['name']}" for frame in reversed(frames) + ) + == row["callstack"], + "Cross-shard ordered frames differ from standalone stack text", + ) + + def _sharded_acceptance( pt_snap: Path, fixture: Path, @@ -358,10 +404,7 @@ def cli(label: str, *args: object) -> dict[str, Any]: scope.get("slice_indices") == [slices[0]["index"], slices[1]["index"]], "Event range did not cross shards", ) - _require( - crossing["rows"] == reference["rows"] and len(crossing["rows"]) == 2, - "Cross-shard real events differ from standalone", - ) + _event_parity(crossing["rows"], reference["rows"]) _require( crossing.get("total_is_exact") is True and crossing.get("total") == 2, "Cross-shard exact total is invalid", diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 9323fc9..70ac847 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -97,12 +97,11 @@ The required installed-wheel case creates a clean venv without system packages and runs its actual `pt-snap` console script from isolated HOME/CWD. Its trusted, test-owned input has exactly 1,000,000 events; the default exporter produces two full 500,000-event shards. All installed catalog entries are classified: currently -11 dataset-supported templates succeed with installed CLI/API equivalence and -independent known-result checks, while `event_frames` and -`active_memory_frame_tree_at_event` return explicit unsupported-dataset errors. -Both report commands run: `peak-memory` succeeds and `memory-tree` preserves the -corresponding dataset rejection. The allocation/free-request/free-completion -boundary assertions remain covered, alongside static occupancy, offset pages, +12 dataset-supported templates succeed with installed CLI/API equivalence and +independent known-result checks; `event_frames` remains the explicit +unsupported-dataset query. Both `peak-memory` and `memory-tree` reports succeed. +The allocation/free-request/free-completion boundary assertions remain covered, +alongside static occupancy, offset pages, terminal survivors and unchanged source/manifest/member hashes. A second required nightly case separately imports 500,002 events and proves the original final short shard has exactly two real events, with the crossing completed lifetime diff --git a/README.md b/README.md index 147b326..113db91 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,11 @@ byte units, allocated/active/reserved counters, expected output and partial repo - [Replayable pickle splitting](docs/en/splitting.md): one split strategy and a new output directory; separate from dataset import. - [Complete-dataset addressing](docs/en/sharded-snapshotdb.md#complete-dataset-focus-and-addressing-p1), [global built-in support](docs/en/querying.md#dataset-global-built-in-support-p3), and [recorded interoperability acceptance](docs/en/interop-acceptance.md): advanced contracts and evidence, not prerequisites for a first report. +Current compatible exports use parser v3; the pinned v2 revision is a historical +schema reference, with no verified v3 GUI consumer revision yet. Compatible reuse +checks source identity, member inventory and required structure, without comparing +database content digests. See the protocol contract above for limits and evidence. + ## Built-in Query Templates & Reports Use `pt-snap query --list` or `pt-snap capabilities --json` to inspect all 13 built-in templates and their parameter/output schemas: diff --git a/README_zh.md b/README_zh.md index 3e824c1..ffbcc5a 100644 --- a/README_zh.md +++ b/README_zh.md @@ -121,6 +121,10 @@ allocated/active/reserved 指标、预期输出和报告不完整的情况。 - [可回放 pickle 拆分](docs/zh/splitting.md)：只选一种拆分策略并使用新目录，与数据集导入不同。 - [完整数据集寻址](docs/zh/sharded-snapshotdb.md#完整数据集-focus-与寻址p1)、[全局内建支持](docs/zh/querying.md#数据集全局内建支持p3)和[已记录的互操作验收](docs/zh/interop-acceptance.md)：按需查看高级合同与证据，不是第一份报告的前置条件。 +当前兼容导出使用 parser v3，固定的旧 v2 revision 仅为历史 schema 参考，尚无已核验的 +v3 GUI 消费端固定版本。兼容复用检查源身份、成员清单及必要结构，不比较数据库内容摘要。 +限制与验收证据见上方协议合同。 + ## 内置查询模板与报告 可执行 `pt-snap query --list` 或 `pt-snap capabilities --json` 查看全部 13 个内置查询模板及其参数与输出契约： diff --git a/docs/en/database.md b/docs/en/database.md index 212ff13..15850c3 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -381,8 +381,8 @@ format and rebuilds the cache; read-only analysis never migrates a database. ### Standalone frame capability boundary Format 3 applies only to standalone imports. Native sharded imports remain format 2 -and compatible exports remain format 1; neither claims the standalone frame-tree -capability. `callstack_frame_manifest(callstackId INTEGER PRIMARY KEY, +and compatible exports remain format 1. A separate version-2 deduplicated frame +extension supports dataset trees; see [querying](querying.md#deduplicated-dataset-frame-extension-version-2). `callstack_frame_manifest(callstackId INTEGER PRIMARY KEY, frameCount INTEGER NOT NULL)` records the expected length of every captured stack, including zero. Frame fields must be text filename/name and an integer line. `event_frames` returns no rows for absent, known-empty or corrupt arrays, never a @@ -468,16 +468,20 @@ P0 compatibility-v1 table order, inline layout and manifest semantics are unchan `trace_entry_` tables with the ninth column `callstack TEXT`, not views, with the exact positional block/dictionary schema. It removes staging's unprefixed `callstack` table and writes `pt_snap_metadata.import_format_version=1`, retaining -isolated `pt_snap_block_reference` text lifecycle sources. Native standalone and -native datasets keep their interning and v2 metadata. Structured ordered frames -are not emitted or inferred from text. +isolated `pt_snap_block_reference` text lifecycle sources. Native datasets keep their text interning and v2 metadata, plus the separate +version-2 deduplicated frame extension; standalone imports use format 3. Frames are never inferred from text. Inline stack bytes scale with the sum of stack text **over every event**, not unique stacks; conversion temporarily holds native and inline tables and may leave SQLite free pages. Capacity does not bound blocks, loading RSS or total disk. -Compatible datasets additionally carry `ptSnap.identity`, metadata, omitted devices -and full member hashes. Recognized whole-artifact metadata is available only after -all attestation checks; external v1 datasets still need no metadata. +Compatible exports additionally carry `ptSnap.identity`, metadata, omitted devices +and a structural member inventory with empty digest values. Historical digest +fields are accepted as inventories; compatible import reuse compares member names, +not digest values, alongside source identity and required read-only structure. +It does not scan arbitrary row contents, so structurally valid content edits can +be reused. Recognized metadata does not attest database content integrity. +Queries and explicit validators perform independent admission checks; external v1 +datasets still need no metadata. Native dataset content verification is unchanged. For finalized compatibility datasets ONLY, bare validation and cached analysis use `mode=ro&immutable=1` after rejecting all live/dangling sidecars and aliases diff --git a/docs/en/dataset-acceptance.md b/docs/en/dataset-acceptance.md index 1ca5143..443f295 100644 --- a/docs/en/dataset-acceptance.md +++ b/docs/en/dataset-acceptance.md @@ -9,10 +9,10 @@ fixture is not evidence that a producer, collector, or exporter ran. compatibility-v1 and native-v2 layouts; the nightly profile extends the same bank to 200 seeds. The original eight regression seeds and twenty query shapes remain. Each seed now runs 69–73 query cases per layout (2,896 paired comparisons for the -20-seed bank; 28,864 for 200), covering all eleven dataset-supported templates. -The two standalone-only templates, `event_frames` and -`active_memory_frame_tree_at_event`, have separate explicit rejection tests and -never count as successful differential comparisons. +20-seed bank; 28,864 for 200), covering eleven text-query contracts. Dataset frame trees have a separate +real-import single-DB differential suite in `test_dataset_frame_tree.py`, including +recursion, cross-shard sources, corruption, budgets and read-only conservation. +Only `event_frames` remains an explicit dataset rejection. Both layouts use the same independently generated 80-event history and a layout-matched standalone SQL oracle. Capacities 2/3/4/5/10/16 produce 5–40 @@ -131,10 +131,10 @@ separate empty/static/dynamic/mixed regressions across both layouts and reducers - A trusted, test-owned snapshot with 1,000,000 alloc/free-request/free-completion events goes through the clean-venv installed `pt-snap import --format msinsight` with **no capacity override**, producing two full 500,000-event members. All - installed catalog templates are classified: 11 dataset-supported templates - have CLI/API parity and independent result checks; `event_frames` and - `active_memory_frame_tree_at_event` retain explicit dataset rejection. The - `peak-memory` report succeeds; `memory-tree` retains its dataset rejection. + installed catalog templates are classified: 12 dataset-supported templates + have CLI/API parity and independent result checks; only `event_frames` + retains explicit dataset rejection. Both `peak-memory` and `memory-tree` + reports succeed, with root/self occupancy conservation checked. - A second required installed-wheel case preserves the original 500,002-event workload and its 500,000 + 2 member layout. It verifies the crossing completed lifetime, final survivor, pages and point-event results. Both installed cases diff --git a/docs/en/interop-acceptance.md b/docs/en/interop-acceptance.md index 4c25b25..3a2bec0 100644 --- a/docs/en/interop-acceptance.md +++ b/docs/en/interop-acceptance.md @@ -4,7 +4,9 @@ ## Evidence layers and fixed target -The ONLY upstream target is **Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5**. +The historical v2 acceptance target was **Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5**. +This revision and `targetRevision` are historical schema/identity references, not +a verified consumer for current v3 exports. Protocol/schema validation, isolated source-function checks, synthetic CI, reading an actual original-producer artifact, and actual GUI/C++ consumption are DIFFERENT evidence layers. None implies the next. **The bounded real GUI/C++ and @@ -23,7 +25,27 @@ are separately CC BY 4.0, not automatically covered by this project's MIT lineag A port needs snapshot provenance/PR decisions and retained notices. Executable pickle trust is a SEPARATE fixture review; no new committed pickle is added here. -## Forward chain: trusted source → physical export → real GUI +## Current v3 consumer acceptance status + +No matching v3 consumer revision/build has been verified yet; current exports +have no claimed GUI reuse support. The coordinated consumer must match +`cacheHash = SHA256(b"mem_snapshot_parser_v3" + raw_pickle_bytes)` and treat +`parserCache.members` as a structural inventory with empty member values, +without requiring historical DB content digests. A `targetRevision` match alone +does not establish these behaviors. + +Before claiming support, pin the consumer revision/build and verify actual cache +reuse without producer rerun or unintended artifact changes. Use a separately +trusted pickle copy and a new adjacent cache directory, preserving original +artifacts. Do not open current v3 caches with the pinned old v2 GUI: it can +reparse and rewrite the cache. The following steps record the historical v2 +procedure and are not instructions for accepting current CLI exports. + +## Historical v2 forward chain: trusted source → physical export → real GUI + +The pinned historical acceptance below used `mem_snapshot_parser_v2`. Current +exports use `mem_snapshot_parser_v3`; the historical GUI result does not verify +the new parser version. Re-export v2 caches to a new output directory for v3 reuse. 1. Review/trust the source separately: pickle is executable input, not a sandbox. Record exact source SHA256, size, selected device and capacity. Default import diff --git a/docs/en/querying.md b/docs/en/querying.md index 6f9bc3f..d461594 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -99,6 +99,7 @@ across devices and there is no shared cross-device event order. | `memory_peak` | Independent allocated/active/reserved maxima over real events; earliest real ID wins ties; `start_id/end_id` may cross shards | | `allocator_gap` | The same independently selected global events, with each gap computed from counters **at that event** | | `active_blocks_at_event`, `active_memory_callstack_at_event` | Route a real event to its owner and resolve full cross-shard sources before grouping/top_n (details below) | +| `active_memory_frame_tree_at_event` | Event-owner live set with cross-shard allocation frames, grouped by complete caller paths; root/self totals conserve occupancy; no top_n pruning | | `preexisting_live` | Owner-event preexisting occupancy, not a leak count | | `block` | Merge proved allocation-event lifetimes and stable negative tokens once, then apply filters/order/page; latest containing-shard state is an observation, not state at arbitrary E | | `freed_block_lifetime` | Deduplicated captured allocations with a proved `free_completed` source; event-ID distance buckets, not time | @@ -216,7 +217,7 @@ shard. Event/allocation pages apply SQL filters and ordering per shard, merge at `offset + limit` candidates per shard, and count matches separately. Range evidence is measured before non-range filters. Ordinary text-stack statistics merge full per-shard groups before global thresholds; averages use summed sizes and counts. -Canonical ordered-frame statistics retain the bounded source-reading path because +Version-1 canonical ordered-frame statistics retain the bounded source-reading path because formatted text is not their identity. Small block/leak/lifetime reductions use read-only attached shards for proof, conflicts and latest observations before pagination; the attachment limit (normally ten) counts selected observation shards plus any @@ -372,9 +373,9 @@ those merge semantics merely by using a built-in name. ### Optional ordered-frame reader contract -This reader-only pt-snap extension is new, not an existing original-producer -feature. Current native/compatible writers emit **no** ordered-frame tables. -Declare `extensions.ptSnapOrderedFrames={"version":1}` in the manifest and use +This pt-snap extension is independent of original-producer capabilities. New +native/compatible imports write the deduplicated version-2 extension described +below. The version-1 reader remains supported. Declare `extensions.ptSnapOrderedFrames={"version":1}` in the manifest and use isolated actual tables in each covered source shard: - `pt_snap_frame_coverage(eventId INTEGER PRIMARY KEY, frameCount INTEGER)` @@ -779,7 +780,7 @@ and `semantics_version` so that lookup stays tied to the rows. ## Frame Memory Breakdown -After a standalone import with this version, inspect the original ordered frame array or +After an import with this version, inspect the original ordered frame array or decompose live memory after an event: ```bash @@ -830,15 +831,44 @@ node ID to distinguish identical labels. Lists include every match and support Tab then Enter/Space. Parent frame moves up one level; Reset zoom returns to the root without clearing the search. Clear the search field to remove the results. -Frame-tree templates and `report memory-tree` support standalone format-3 databases -only. Dataset roots are rejected explicitly; the analyzer never selects a shard -implicitly. Direct native shards have no standalone frame tables: re-import the -original snapshot as a standalone database to use these templates. Existing -native `pt_snap_frame`/coverage readers remain unchanged. `--timeout` uses the -shared query deadline. Text and HTML renderers reject incomplete or nonconserving -trees. Known-empty stacks have an `empty_callstack` synthetic leaf; -missing/corrupt stacks retain the `missing` leaf. `event_frames` returns no rows -for missing events/stacks, known-empty stacks or invalid frame arrays. +`active_memory_frame_tree_at_event` and `report memory-tree` support standalone +format-3 databases and complete native/compatible datasets. Query the dataset root: +the event owner supplies live blocks and allocation owners supply frames, including +earlier slices. Local IDs and display text never identify cross-shard caller paths. +`frame_id` is query-local; `node_id` hashes the full ordered path of +`(filename, line, name)` triples. `scope.source_coverage.ordered_frames_complete` +reports frame evidence separately from output row completeness. Missing, +unknown-version or corrupt evidence keeps its bytes in `missing`; re-import the +original snapshot to capture frames. Known-empty arrays have an `empty_callstack` +leaf. Work budgets and the shared deadline cover the full operation; failures +return no partial tree. Text and HTML renderers reject truncated/nonconserving trees. +`event_frames` remains standalone-only; use dataset `event` with `id` for its +covered original `frames`. Direct native shards lack standalone frame tables; +query the dataset root for cross-shard allocation sources. + +### Deduplicated dataset frame extension (version 2) + +New imports declare `extensions.ptSnapOrderedFrames={"version":2}` and store: + +| Table | Columns | +| --- | --- | +| `pt_snap_frame_value` | `id INTEGER PRIMARY KEY, frameJson TEXT` | +| `pt_snap_stack` | `stackId INTEGER PRIMARY KEY, frameCount INTEGER` | +| `pt_snap_stack_frame` | `stackId INTEGER, frameIndex INTEGER, frameId INTEGER, PRIMARY KEY(stackId,frameIndex)` | +| `pt_snap_event_stack` | `eventId INTEGER PRIMARY KEY, stackId INTEGER` | + +Each canonical raw frame JSON (including extra typed fields) and each distinct +ordered frame-ID sequence is stored once per shard. Repeated frames retain +positions; empty stacks have an explicit count zero. Only real events receive +references. These independent IDs preserve native text interning and compatible +inline schemas. Native extension cache identity is 2; compatible export identity +is 3, preventing reuse of old text-only exports as frame-capable artifacts. +Readers support versions 1/2 and validate exact schemas, contiguous positions, +counts, references and finite JSON with nesting depth at most 128. Tree frames +require text filename/name and integer line; invalid fields degrade to missing. +No text reconstruction occurs. `callstack_analysis` reduces version-2 events by +local stack and text in SQLite before resolving representative arrays and merging +global identities; repeated stacks do not require fetching every event into Python. The Python API uses the same template and contracts: diff --git a/docs/en/quickstart.md b/docs/en/quickstart.md index 0e70846..cf11550 100644 --- a/docs/en/quickstart.md +++ b/docs/en/quickstart.md @@ -171,6 +171,22 @@ fixed-version evidence have their own [protocol contract](sharded-snapshotdb.md# Read [recorded interoperability acceptance](interop-acceptance.md) before claiming GUI reuse. A matching hash alone is not GUI acceptance. +Current exports use parser v3. The pinned `101f65b` revision and `targetRevision` +are historical v2 schema/identity references, not verified current GUI consumers. +No fixed v3 consumer revision/build has been verified yet. Do not open current +caches with the old GUI: it may reparse and rewrite them. Future v3 acceptance +must use a separately trusted pickle copy and a new adjacent cache, preserving +original artifacts. + +Compatible reuse checks source identity, sourceFile, options, contract versions, +the complete member inventory and required read-only structure. Historical member +hash fields are inventories; their digest values are not compared. Import does +not hash database contents or scan arbitrary row values, so structurally valid +content edits can be reused. Queries and explicit validators perform independent +admission checks. Successful reuse is not proof of database content integrity or +GUI cache reuse. `--output-dir` selects another parent; moving a dataset does not +establish a GUI entrance. + ### Optional: Split a Snapshot `pt-snap split` creates independently replayable files, not a queried dataset. diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 0bf64ec..41e0926 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -155,6 +155,14 @@ conflict. Native v2 remains a separate single-DB layout, not this compatibility base. `pt_snap_metadata` and auxiliary reference tables are optional here. Unknown `extensions` declarations/tables are ignored by base validation. +Compatible cache reuse checks the ordered-frame declaration version, not frame-table +contents or stored digests. Queries independently validate the frame evidence they use. +Legacy and external text-only caches retain their read-only degradation behavior. +Unencodable optional event evidence may remain uncovered; source coverage is reported +separately. Writers check the reader's JSON depth limit of 128 before inserting any +part of a stack. Unsupported deep evidence is omitted as a whole; text callstacks +and memory occupancy remain available. + `DatasetValidation.text_callstacks=True` means an inline text column is available, not that every event has a stack. `structured_frames=False` remains false even when a manifest claims frames: complete structured-frame capability requires a @@ -478,10 +486,15 @@ failure paths; no live upstream GUI or performance acceptance is claimed. ## Explicit msinsight-Compatible Export (P2) `pt-snap import snapshot.pkl --format msinsight --json` (alias -`--format compatibility-v1`) targets **only** -`Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5`. Its supported entrance is -opening the **same original pickle** in that target with adjacent -`.msinsight/`, not direct standalone-directory GUI import. +`--format compatibility-v1`) exports parser v3 caches. The pinned +`Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5` and `targetRevision` +are historical schema/identity references, not proof that the old v2 GUI supports +current exports. No fixed v3 consumer revision/build has been verified yet. +Do not give current caches to that old GUI: it may reparse and rewrite them. +Future v3 acceptance must use a separately trusted pickle copy with a new adjacent +`.msinsight/` cache, preserving the original artifacts; it must pin +the matching consumer revision/build and verify v3 hash/member semantics before +claiming GUI reuse. Direct standalone-directory GUI import is not supported. `--output-dir` selects another parent for pt-snap; relocation alone does not create an upstream GUI entrance. `sourceFile` is the absolute original source path. Capacity defaults to `500000` and can be set with `--events-per-slice`; default @@ -506,40 +519,46 @@ next expected ID, after verifying a unique integer primary key; negative boundar rows are excluded. It does not collect the device's event IDs in Python. This bounds additional ID-validation memory, not snapshot loading, replay, or total import RSS. -`cacheHash = SHA256(b"mem_snapshot_parser_v2" + raw_pickle_bytes)`: the salt is -**prepended**. Independent `ptSnap.identity` fields are `sourceSha256`, +`cacheHash = SHA256(b"mem_snapshot_parser_v3" + raw_pickle_bytes)`: the salt is +**prepended**. Caches using the former `mem_snapshot_parser_v2` salt do not match; +export to a new output directory instead of replacing the old cache. +Independent `ptSnap.identity` fields are `sourceSha256`, `requestedDevice`, `eventsPerSlice`, `outputFormat=compatibility-v1`, -`manifestContractVersion=1`, `exportContractVersion=2`, `referenceContractVersion=1`, +`manifestContractVersion=1`, `exportContractVersion=3`, `referenceContractVersion=1`, `metadataSchemaVersion=1`, `importFormatVersion=1`, and full `targetRevision`. -The `ptSnap` object also contains `metadata`, `omittedDevices`, and `members` -(canonical relative path → logical parser-data SHA256 with `memberHashKind=parser-v1` -for new exports). Whole validated ownership/identity -requires every member/hash and identical per-slice metadata; unknown/external +The `ptSnap` object also contains `metadata` and `omittedDevices`. Ownership/identity +requires a complete member inventory and identical per-slice metadata; unknown/external caches and unrelated files are not claimed as owned. Valid Insight caches can be reused read-only without producer metadata. CLI version alone does not invalidate. -Logical seals cover parser-owned base rows and their dictionary entries, not -consumer tables, indexes or appended columns. Both import and dataset queries -check declared seals; unsealed legacy artifacts receive structural, range, -dictionary-reference and `quick_check` validation only. Without a historical seal, -arbitrary changed values or deleted block rows cannot be proven unchanged. -Digests are not signatures. `quick_check` detects physical damage but does not -prove index/table equivalence against deliberate `writable_schema` tampering. -Default warm resolver reuse still reads all member bytes to verify the generation; -cold admission additionally scans rows. Validation time grows with dataset size. +Compatible imports check source SHA256, complete manifest/member inventory, read-only +database schema and indexed event endpoints. They do not scan base/frame rows, run +`quick_check`, or hash database bytes. Historical digest fields are accepted as member +inventories only; their values are not compared and new exports do not generate content +digests. New `parserCache.members` entries use empty strings as structural inventory values. +There is no strict-cache option. These checks do not detect arbitrary edited values, +internal event gaps or damaged secondary indexes. Ordinary dataset queries and explicit +validators retain their independent admission checks; lightweight import results never +seed their validation cache or receipts. + +Native dataset imports retain disposable validation receipts under +`~/.cache/pt-snap-cli/validation-v1/`. They hash manifest/member bytes and validate +native data independently of compatible-cache reuse. Missing or invalid receipts fall +back to full native validation; `PT_SNAP_DATASET_CACHE=off` disables receipt reuse. Compatible publication is **no-replace even with force**. Identical recognized pt-snap caches reuse without loading pickle (force does not bypass this rule). -Changed source path/content, options/contracts, corrupted parser data or legacy -byte-attested DBs changed by a consumer require a new `--output-dir`; no repair, overwrite or deletion of the old -artifact occurs. Exclusive publication and focus compensation use the existing +Changed source path/content or options/contracts require a new `--output-dir`. +The old artifact is never repaired, overwritten or deleted. Exclusive publication +and focus compensation use the existing checked transaction; source SHA256 is rechecked immediately before publish/reuse. Native force replacement and recovery semantics are untouched. Inline text costs `sum(len(callstack.encode("utf-8")) for each event)` rather than one copy per distinct stack. Existing native interning remains available. Conversion temporarily holds both layouts and can leave free SQLite pages; capacity is not an -RSS/registry/block/disk bound. No optional frame enhancement is emitted. +RSS/registry/block/disk bound. New exports also retain the optional version-2 +ordered-frame extension described above. **The artifact-only runner still reports its own GUI stage as pending/not-run.** The separate fixed-version real GUI/C++ acceptance on 2026-10-11 is recorded in diff --git a/docs/en/snapshot-analyzer-api.md b/docs/en/snapshot-analyzer-api.md index e1b2a0d..34f8728 100644 --- a/docs/en/snapshot-analyzer-api.md +++ b/docs/en/snapshot-analyzer-api.md @@ -231,7 +231,8 @@ project or global focus when no explicit analyzer database is set, then the firs discovered device. An explicit analyzer database without an analyzer device does not inherit a configured device. `max_rows=None`, zero, or a negative value is unlimited. Pass `exact_total=True` for a matching-row `COUNT`. `timeout_s` -is a wall-clock budget for one `execute_query()` call in seconds (or +is a wall-clock budget for one `execute_query()` call, including frame-tree queue +waiting, in seconds (or `PT_SNAP_QUERY_TIMEOUT`); the page query and optional COUNT share it. The same environment variable applies to every template query, including `report peak-memory`. It does not change the row cap. Continue @@ -279,6 +280,17 @@ metadata schema version returns `status="invalid"`. ## Errors and Scope +Concurrent `active_memory_frame_tree_at_event` requests on the same analyzer +use latest-request-wins semantics. A new request interrupts the previous tree's +Python work and SQLite statements; superseded queued requests do not compute. +The old call raises `pt_snap_cli.core.QueryCancelledError` (`QUERY_CANCELLED` +in JSON), never a partial successful tree. Closing the analyzer or invalidating +its context cache also cancels outstanding tree work. Worker-owned connections +are closed in the executing thread before the next tree runs. Other templates +retain their existing concurrency contract; separate analyzers/services and +separate CLI processes do not cancel one another. This does not introduce a +persistent tree-result cache. + - `set_focus()` raises `FileNotFoundError` for a missing database and `ValueError` for an invalid SnapshotDB schema. Device validation uses the shared `InvalidDeviceError`; selecting a device without any resolvable database raises diff --git a/docs/zh/database.md b/docs/zh/database.md index e542e7d..2ebc8e8 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -360,7 +360,7 @@ CREATE UNIQUE INDEX idx_callstack_frame_position ### 单库帧能力边界 格式 3 仅用于单库导入。原生分片导入保持格式 2，兼容导出保持格式 1； -它们不会声称支持单库帧树。新增 `callstack_frame_manifest(callstackId INTEGER PRIMARY KEY, +它们通过独立的版本 2 去重帧扩展支持数据集帧树，详见[查询指南](querying.md#数据集去重帧扩展版本-2)。新增 `callstack_frame_manifest(callstackId INTEGER PRIMARY KEY, frameCount INTEGER NOT NULL)` 保存每个采集调用栈的预期长度，包括零。 filename/name 必须是文本，line 必须是整数。`event_frames` 对缺失、已知空数组 或损坏数组返回零行，不输出部分重建的数组。树将已知空栈与缺失/损坏证据分开。 @@ -437,12 +437,15 @@ P0 兼容 v1 表序、内联布局和 manifest 语义不变。 `trace_entry_` 表，第九列为 `callstack TEXT`，不是 VIEW；block/dictionary 位置 schema 精确匹配。转换去除暂存的未隔离 `callstack` 表，写入 `pt_snap_metadata.import_format_version=1`，保留隔离的 `pt_snap_block_reference` -文本生命周期来源。原生单库/数据集仍使用去重和 v2 metadata，不生成或从文本推断有序 frames。 +文本生命周期来源。原生数据集保留独立的版本 2 帧扩展，不从文本推断有序 frames。 内联调用栈字节按**所有事件**的文本总和增长，不按不同栈计数；转换暂时并存原生与内联表， 可能保留 SQLite free pages。容量不限制 block、加载 RSS 或总磁盘。兼容 manifest 的 -`ptSnap.identity`、metadata、省略设备和全部成员哈希形成整体证明，完整校验后才报告 -metadata available；外部 v1 数据集仍不要求 metadata。 +`ptSnap.identity`、metadata、省略设备及摘要值为空字符串的结构成员清单随导出写入。 +历史摘要字段仅作为清单接受；兼容导入复用检查成员名称、源身份及必要的只读结构， +不比较摘要值，也不扫描任意行内容，因此结构合法的内容编辑可以被复用。 +识别到 metadata 不证明数据库内容完整。查询和显式验证器独立执行准入检查； +外部 v1 数据集仍不要求 metadata。原生数据集的内容校验保持不变。 **仅最终化兼容数据集**的裸校验与缓存分析在首个 SQLite 打开前拒绝全部存活/悬空 sidecar 和别名后使用 `mode=ro&immutable=1`，保留已关闭/checkpoint 原版 WAL 的字节及 diff --git a/docs/zh/dataset-acceptance.md b/docs/zh/dataset-acceptance.md index d14041b..7182ae0 100644 --- a/docs/zh/dataset-acceptance.md +++ b/docs/zh/dataset-acceptance.md @@ -7,8 +7,9 @@ `tests/core/test_dataset_realistic.py` 默认在兼容 v1 和原生 v2 两种布局上各执行 20 个确定性种子；夜间配置将同一批种子扩展至 200 个。原有八个回归种子及二十种 查询形态完整保留。每个种子每种布局运行 69–73 个查询用例（20 种子共 2,896 次 -成对比较，200 种子共 28,864 次），覆盖全部十一个 dataset 支持模板。 -仅单库支持的 `event_frames`、`active_memory_frame_tree_at_event` 另有明确拒绝测试， +成对比较，200 种子共 28,864 次），覆盖十一个文本查询契约。 +帧树由 `test_dataset_frame_tree.py` 单独验证真实导入的单库/数据集对齐、递归、跨片来源、 +损坏、预算及只读守恒；仅 `event_frames` 保留明确数据集拒绝测试， 不会计为差分成功。 两种布局使用相同的独立生成 80-event 历史，并分别与同布局单库 SQL 参考结果比较。 @@ -105,10 +106,8 @@ fallback 三条路径的这些契约。`top_n=0` 完整性另有空/static/dynam 64-MiB 上限，无需创建 8-GB 数据文件。 - 受信、测试自行构造的 1,000,000 个 alloc/free-request/free-completion 事件快照， 经干净 venv 中安装的 `pt-snap import --format msinsight`，**不覆盖容量默认值**， - 生成两个完整的 500,000-event 成员。所有已安装目录模板均被分类：11 个支持数据集的 - 模板核对 CLI/API 一致性及独立结果；`event_frames` 和 - `active_memory_frame_tree_at_event` 保留明确的数据集拒绝。`peak-memory` 报告成功， - `memory-tree` 保留数据集拒绝。 + 生成两个完整的 500,000-event 成员。所有已安装目录模板均被分类：12 个支持数据集的 + 模板核对 CLI/API 一致性及独立结果。帧树模板及 `memory-tree` 报告成功，并验证根节点/自身占用守恒；仅 `event_frames` 保留数据集拒绝。 - 第二个必需的安装包用例保留原有 500,002-event 工作负载及 500,000 + 2 成员布局， 验证跨界完成生命周期、最终存活分配、分页及定点事件结果。两个安装包用例均要求 查询前后输入、manifest 及成员 hash 不变。 diff --git a/docs/zh/interop-acceptance.md b/docs/zh/interop-acceptance.md index a391028..beec75d 100644 --- a/docs/zh/interop-acceptance.md +++ b/docs/zh/interop-acceptance.md @@ -4,7 +4,8 @@ ## 证据分层与固定目标 -唯一上游目标是 **Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5**。 +历史 v2 验收目标是 **Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5**。 +该 revision 及 `targetRevision` 仅为历史 schema/身份参考，不是当前 v3 导出的已核验消费端。 协议/schema、隔离源函数、合成 CI、真实原版生产端产物的读取差分、GUI/C++ 实测是 不同证据层，不能互相替代。**2026-10-11 的有界真实 GUI/C++ 与原版生产端反向验收已通过**， 详见下方记录。[#205 待验收说明](https://github.com/codeasier/pt-snap-cli/issues/205#issuecomment-5987264017) @@ -19,7 +20,23 @@ 本项目 MIT lineage。真正移植须走 snapshot provenance/PR 声明并保留 notices。 可执行 pickle 的信任是独立 fixture 审核；本验收不新增已提交 pickle。 -## 正向：可信源 → 物理兼容导出 → 真实 GUI +## 当前 v3 消费端验收状态 + +目前尚无已核验的配套 v3 消费端固定 revision/build，不宣称当前导出支持 GUI 复用。 +联动消费端必须匹配 `cacheHash = SHA256(b"mem_snapshot_parser_v3" + raw_pickle_bytes)`， +并将 `parserCache.members` 作为值为空字符串的结构清单，不要求历史 DB 内容摘要。 +仅 `targetRevision` 匹配不能证明这些行为。 + +宣称支持前，应固定消费端 revision/build，实测缓存复用、未重跑生产端及无意外产物修改。 +必须使用独立可信 pickle 副本和新建邻接缓存目录，保留原始产物。 +不要用固定的旧 v2 GUI 打开当前 v3 缓存，它可能重新解析并改写缓存。 +以下步骤仅记录历史 v2 流程，不是当前 CLI 导出的验收操作指南。 + +## 历史 v2 正向：可信源 → 物理兼容导出 → 真实 GUI + +以下固定历史验收使用 `mem_snapshot_parser_v2`，当前导出已使用 +`mem_snapshot_parser_v3`；历史 GUI 结果不代表新版解析器已验收。 +需要按 v3 复用时，请换新的输出目录重新导出 v2 缓存。 1. 先独立审核/信任源：pickle 可执行，不是沙箱。记录 SHA256、大小、设备与容量。 默认 import 仍单库，原生/兼容模式显式区分。 diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 5faa28f..7c9d539 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -89,6 +89,7 @@ pt-snap query '' --device 0 --template-use event --params '{"id":123}' | `memory_peak` | allocated/active/reserved 独立真实峰值；同值选最早真实 ID；`start_id/end_id` 可跨片 | | `allocator_gap` | 使用同样独立选择的全局事件，每个 gap 由**该事件**计数器计算 | | `active_blocks_at_event`, `active_memory_callstack_at_event` | 路由实际事件到所有者片，分组/top_n 前解析完整跨片来源（见下文） | +| `active_memory_frame_tree_at_event` | 事件所属片的存活集合，跨片解析分配帧后按完整调用路径聚合；根节点与自身占用守恒，不做 top_n 剪枝 | | `preexisting_live` | 所有者事件的 preexisting 占用，不是泄漏数 | | `block` | 已证明的分配事件生命周期及稳定负 token 去重后筛选/排序/分页；最新出现片 state 是观察，不是任意 E 的状态 | | `freed_block_lifetime` | 已采集分配与已证明 `free_completed` 来源的去重生命周期；event-ID 距离桶，不是时间 | @@ -300,8 +301,8 @@ cache 仍由调用者所有。不生成合并临时数据库。`ReportService.ev ### 可选有序 frame reader 合同 -这是新增的仅 reader pt-snap 扩展，不是原版 producer 已有功能。当前原生/兼容 writer -**不生成**有序 frame 表。manifest 声明 `extensions.ptSnapOrderedFrames={"version":1}`， +此 pt-snap 扩展独立于原版 producer 能力。新原生/兼容导入生成下文的去重版本 2 +扩展；保留版本 1 原始数组读取兼容。manifest 声明 `extensions.ptSnapOrderedFrames={"version":1}`， 并在有覆盖的来源片使用隔离的实际表： - `pt_snap_frame_coverage(eventId INTEGER PRIMARY KEY, frameCount INTEGER)` @@ -699,13 +700,35 @@ HTML 图始终按字节比例绘制。即使帧过小或占用为零，也可以 列表包含全部匹配项，支持 Tab 定位后按 Enter/空格进入。Parent frame 返回父节点， Reset zoom 回到根节点且保留搜索；清空搜索框可清除搜索结果。 -帧树模板和 `report memory-tree` 仅支持单库格式 3 数据库。dataset 根目录会 -明确拒绝，分析器不会隐式选取分片。直接访问原生分片也没有这些单库帧表； -使用新模板须将原始 snapshot 重新导入为单库。已有 `pt_snap_frame`/coverage -读取能力不变。`--timeout` 使用共享查询截止时间。文本和 HTML 渲染器拒绝 -不完整或不守恒的树。已知空栈使用 `empty_callstack` 合成叶节点，缺失/损坏 -调用栈使用 `missing` 叶节点。`event_frames` 对缺失事件/栈、已知空栈以及 -无效帧数组返回零行。 +`active_memory_frame_tree_at_event` 和 `report memory-tree` 支持单库格式 3 +以及完整的原生/兼容分片数据集。查询数据集根目录：事件所属片提供存活块集合， +分配事件所属片提供原始帧，可追溯到更早分片。局部 ID 和显示文本不能作为跨片分组身份。 +`frame_id` 仅在本次查询有效，`node_id` 是完整有序 `(filename, line, name)` 调用路径的哈希。 +`scope.source_coverage.ordered_frames_complete` 单独报告帧证据完整性，与输出行完整性不同。 +缺失、未知版本或损坏证据保留在 `missing` 的占用中；重新导入原始快照可获得采集帧。 +已知空数组使用 `empty_callstack` 叶节点。工作预算和共享截止时间覆盖整个操作， +失败不返回部分树；文本和 HTML 拒绝截断或不守恒的树。 +`event_frames` 仍仅支持单库；数据集使用 `event` 的 `id` 参数读取覆盖的原始 `frames`。 +直接访问原生分片没有单库帧表，应查询数据集根目录，以跨片解析分配来源。 + +### 数据集去重帧扩展（版本 2） + +新导入声明 `extensions.ptSnapOrderedFrames={"version":2}`，使用以下表： + +| 表 | 列 | +| --- | --- | +| `pt_snap_frame_value` | `id INTEGER PRIMARY KEY, frameJson TEXT` | +| `pt_snap_stack` | `stackId INTEGER PRIMARY KEY, frameCount INTEGER` | +| `pt_snap_stack_frame` | `stackId INTEGER, frameIndex INTEGER, frameId INTEGER, PRIMARY KEY(stackId,frameIndex)` | +| `pt_snap_event_stack` | `eventId INTEGER PRIMARY KEY, stackId INTEGER` | + +每个 canonical 原始帧 JSON（包括额外类型字段）和每个不同的有序帧 ID 序列在每片只存一份。 +重复帧保留位置，空栈有明确的零 frameCount，仅真实事件保存引用。 +独立 ID 不改变原生文本去重或兼容内联 schema。原生扩展缓存身份更新为 2，兼容导出身份为 3， +避免把旧文本产物复用成帧能力产物。保留版本 1/2 读取兼容，校验精确 schema、连续位置、 +数量、引用以及深度至多 128 的有限数字 JSON。帧树要求 filename/name 为文本、line 为整数， +不合规帧降级为缺失叶，不从文本重建。`callstack_analysis` 对版本 2 先在 SQLite 中按 +局部栈和文本归约，再解析代表数组并合并全局身份，重复栈无需将每个事件取入 Python。 Python API 使用相同模板和契约： diff --git a/docs/zh/quickstart.md b/docs/zh/quickstart.md index 7adb359..9b79617 100644 --- a/docs/zh/quickstart.md +++ b/docs/zh/quickstart.md @@ -155,6 +155,17 @@ RSS。输出路径、完整成员校验、force/恢复行为及支持的内建 [协议合同](sharded-snapshotdb.md#显式-msinsight-兼容导出p2)。声称 GUI 复用前请阅读 [已记录的互操作验收](interop-acceptance.md)，仅 hash 相同不等于 GUI 验收通过。 +当前导出使用 parser v3。固定的 `101f65b` revision 及 `targetRevision` 仅为历史 +v2 schema/身份参考，不是当前导出的已核验 GUI 消费端；目前尚无已核验的 v3 消费端 +固定 revision/build。不要用旧 GUI 打开当前缓存，它可能重新解析并改写缓存。 +后续 v3 验收必须使用独立可信 pickle 副本及新建邻接缓存，保留原始产物。 + +兼容复用检查源身份、sourceFile、选项、合同版本、完整成员清单及必要的只读结构。 +历史成员哈希字段仅作为清单，不比较摘要值。导入不对数据库内容取哈希，也不扫描任意 +行值，因此结构合法的内容编辑可以被复用。查询和显式验证器独立执行准入检查； +复用成功不证明数据库内容完整或 GUI 缓存复用。`--output-dir` 选择其他父目录， +搬迁数据集不意味着建立 GUI 入口。 + ### 可选：拆分快照 `pt-snap split` 生成可独立回放的文件，不是待查询数据集。它同样需要可信 pickle， diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index 73ce759..e10008f 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -124,6 +124,12 @@ dictionary 的十进制整数 key 以文本存储，使用真实带设备后缀 `callstack` 表**：现有布局检测会正确报告 v1/v2 冲突。原生 v2 仍是独立单库布局， 不是本兼容基础格式。这里不要求 `pt_snap_metadata` 或辅助引用表。 +兼容缓存复用只检查有序帧扩展声明版本，不扫描帧表内容或比较历史摘要。 +查询独立校验其使用的帧证据。旧缓存和外部纯文本缓存保留只读降级行为； +无法编码的可选事件帧仍可不覆盖，查询继续独立报告 source coverage。 +写入前按 reader 的 JSON 深度上限 128 预检整栈；超深证据整栈省略， +保留文本调用栈及内存占用，不因可选证据导致整次导入失败。 + 基础校验忽略未知 `extensions` 声明/表。`DatasetValidation.text_callstacks=True` 表示有内联文本列，不保证每个事件有栈。即使 manifest 声称 frames， `structured_frames=False` 仍为假：完整结构化 frame 能力要求已识别的独立扩展版本、 @@ -382,9 +388,13 @@ pickle 仍可能整体加载。分库只限制**单库真实事件数**，不限 ## 显式 msinsight 兼容导出（P2） `pt-snap import snapshot.pkl --format msinsight --json`（别名 -`--format compatibility-v1`）**仅**对齐 -`Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5`。支持入口是在该目标版本 -打开**同一原始 pickle**及邻接 `<原始路径>.msinsight/`，不是 GUI 直接导入独立目录。 +`--format compatibility-v1`）导出 parser v3 缓存。固定的 +`Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5` 及 `targetRevision` +仅为历史 schema/身份参考，不证明旧 v2 GUI 支持当前导出。尚无已核验的 v3 +消费端固定 revision/build。不要将当前缓存交给该旧 GUI，它可能重新解析并改写缓存。 +后续 v3 验收必须使用独立可信 pickle 副本及新建邻接 `<原始路径>.msinsight/` 缓存， +保留原始产物，并记录配套消费端 revision/build、验证 v3 hash/member 语义后再宣称 +GUI 复用通过。不支持 GUI 直接导入独立目录。 `--output-dir` 为 pt-snap 选择其他父目录，搬迁本身不创建上游入口。`sourceFile` 记录 绝对原始源路径。容量默认 `500000`，可用 `--events-per-slice` 调整；默认单库及显式 native-v2 输出仍独立且不变。 @@ -403,29 +413,34 @@ raw-frame 丢失与字符串 OOM action 强制转换属于已知局限，不应 边界行，不再在 Python 中收集整个设备的事件 ID。这只限制额外的 ID 校验内存， 不代表快照加载、回放或完整导入的 RSS 有界。 -`cacheHash = SHA256(b"mem_snapshot_parser_v2" + 原始_pickle_字节)`，salt **前置**。 +`cacheHash = SHA256(b"mem_snapshot_parser_v3" + 原始_pickle_字节)`，salt **前置**。 +旧 `mem_snapshot_parser_v2` 缓存不再匹配，请指定新的输出目录重新导出，不覆盖旧缓存。 独立 `ptSnap.identity` 字段为 `sourceSha256`、`requestedDevice`、`eventsPerSlice`、 -`outputFormat=compatibility-v1`、`manifestContractVersion=1`、`exportContractVersion=2`、 +`outputFormat=compatibility-v1`、`manifestContractVersion=1`、`exportContractVersion=3`、 `referenceContractVersion=1`、`metadataSchemaVersion=1`、`importFormatVersion=1`、完整 -`targetRevision`。`ptSnap` 另含 `metadata`、`omittedDevices` 和 `members`（规范相对路径 -→ 基础解析数据逻辑 SHA256，`memberHashKind=parser-v1`）。整体已校验所有权/身份要求每个成员/哈希及逐片 metadata 相同； -不接管未知/外部缓存或无关文件。CLI 版本变化本身不使缓存失效。 - -逻辑摘要覆盖解析器基础行及对应字典项，不覆盖消费者表、索引和追加列。导入和数据集查询 -均校验已声明的摘要；无摘要的旧产物仅执行结构、事件范围、字典引用及 `quick_check` -校验。缺少历史摘要时，不能追溯证明任意字段未改动或 block 行未被删除。摘要不是签名， -`quick_check` 可检测物理损坏，但不能证明刻意使用 `writable_schema` 篡改后的索引与表 -仍然一致。默认 resolver 热缓存仍读取全部成员字节核对 generation，冷准入还会扫描行， -因此校验耗时随数据量增长。 +`targetRevision`。`ptSnap` 另含 `metadata` 和 `omittedDevices`。所有权/身份校验 +要求成员清单完整且逐片 metadata 相同，不接管未知/外部缓存或无关文件。 +有效 Insight 缓存可以只读复用，不伪造生产者 metadata。CLI 版本变化本身不使缓存失效。 + +兼容产物导入检查源文件 SHA256、完整 manifest/成员清单、只读数据库表结构及 +索引上的事件端点；不扫描基础行或 frame 行、不执行 `quick_check`，也不读取全部 +DB 字节计算摘要。历史摘要字段仅作为成员清单兼容，不比较摘要值，新产物不再生成 +内容摘要。新的 `parserCache.members` 使用空字符串作为结构清单的值。 +不再提供缓存强校验开关。这些检查不保证发现任意字段修改、区间内部事件 +缺失或二级索引损坏。正常数据集查询和显式校验器保留独立准入检查，轻量导入结果 +不会写入其校验缓存或持久化校验记录。 + +原生数据集导入保留 `~/.cache/pt-snap-cli/validation-v1/` 下可丢弃的校验记录， +独立执行原生格式的文件摘要及内容校验，不属于兼容缓存复用逻辑。记录缺失或无效时 +回退完整原生校验，`PT_SNAP_DATASET_CACHE=off` 禁用记录复用。 兼容发布 **force 也 no-replace**。已识别、身份相同的 pt-snap 缓存不加载 pickle 即复用， -force 不绕过该规则。源路径/内容、选项/合同变化、基础解析数据损坏，或旧物理摘要产物被调用方修改后须换新 -`--output-dir`，不修复、不覆盖、不删除旧产物。排他发布与 focus 补偿使用已有受检事务， +force 不绕过该规则。源路径/内容、选项/合同变化后须换新 `--output-dir`。不修复、不覆盖、不删除旧产物。排他发布与 focus 补偿使用已有受检事务， 发布/复用前再次核对源 SHA256；原生 force 替换与恢复语义不变。 内联文本成本按每个事件的 UTF-8 调用栈字节求和，不按不同栈各存一份；原生去重仍可用。 转换暂时并存两种布局，可能留下 SQLite free pages；容量不是 RSS/registry/block/磁盘 -上限。不生成可选 frame 增强。 +上限。另保留独立的版本 2 去重 frame 扩展，详见[查询指南](querying.md#数据集去重帧扩展版本-2)。 **仅检查已有产物的 runner 仍将自身 GUI 阶段标为待验收/not-run。** 2026-10-11 独立固定版本真实 GUI/C++ 实测见[互操作验收](interop-acceptance.md)， diff --git a/docs/zh/snapshot-analyzer-api.md b/docs/zh/snapshot-analyzer-api.md index 2cc5bc7..fa42efb 100644 --- a/docs/zh/snapshot-analyzer-api.md +++ b/docs/zh/snapshot-analyzer-api.md @@ -210,6 +210,7 @@ for row in result["rows"]: analyzer 设备；未设置显式 analyzer 数据库时，再使用已解析项目或全局 focus 的设备； 最后使用发现的第一个设备。只有显式 analyzer 数据库而没有 analyzer 设备时，不会继承 配置中的设备。`max_rows=None`、零或负数均表示不限制行数。传入 +帧树排队等待也计入同一次调用的超时预算。 `exact_total=True` 可得到匹配行 `COUNT`。`timeout_s` 是一次 `execute_query()` 调用的墙钟预算（秒），也可由 `PT_SNAP_QUERY_TIMEOUT` 提供；页面查询与可选 COUNT 共用该预算。该环境变量作用于所有模板查询， @@ -252,6 +253,14 @@ other_metadata = analyzer.get_database_metadata("/path/to/other.db") ## 错误与范围 +同一个 analyzer 上并发调用 `active_memory_frame_tree_at_event` 时，只执行最新请求。 +新请求会中断旧任务的 Python 计算和 SQLite 语句，排队中已过期的请求不会继续计算。 +旧调用抛出 `pt_snap_cli.core.QueryCancelledError`（JSON 错误码为 `QUERY_CANCELLED`）， +不会返回成功的部分树。关闭 analyzer 或清理其 context 缓存也会取消未结束的拆解任务。 +每个任务在执行线程中关闭自己持有的数据库连接后，下一任务才开始计算。其它模板保留 +原有并发约定；不同 analyzer/service 实例、不同 CLI 进程之间互不取消。这不会增加 +持久化的拆解结果缓存。 + - `set_focus()` 在数据库不存在时抛出 `FileNotFoundError`，在 SnapshotDB schema 无效时 抛出 `ValueError`。设备校验使用共享的 `InvalidDeviceError`；没有任何可解析数据库时只 设置设备会抛出 `RuntimeError`。 diff --git a/src/pt_snap_cli/api.py b/src/pt_snap_cli/api.py index fe7d9a1..7b7cf12 100644 --- a/src/pt_snap_cli/api.py +++ b/src/pt_snap_cli/api.py @@ -140,7 +140,7 @@ def invalidate_context_cache(self, db_path: Path | str | None = None) -> None: This cleanup operation remains available after close(). While open, the analyzer can create fresh contexts on its next query or overview. """ - self._context_cache.invalidate(db_path) + self._query_service.invalidate_context_cache(db_path) def get_focus(self) -> FocusState: self._ensure_open() diff --git a/src/pt_snap_cli/core/__init__.py b/src/pt_snap_cli/core/__init__.py index 807155b..c1c37b9 100644 --- a/src/pt_snap_cli/core/__init__.py +++ b/src/pt_snap_cli/core/__init__.py @@ -13,6 +13,7 @@ InvalidParameterError, InvalidSkillTargetError, PtSnapCoreError, + QueryCancelledError, QueryExecutionError, QueryTimeoutError, SkillCatalogError, @@ -65,6 +66,7 @@ from pt_snap_cli.core.split_service import SplitService __all__ = [ + "QueryCancelledError", "PtSnapCoreError", "FocusNotConfiguredError", "FocusFileInvalidError", diff --git a/src/pt_snap_cli/core/dataset_active_sql.py b/src/pt_snap_cli/core/dataset_active_sql.py index 10e035b..24768d6 100644 --- a/src/pt_snap_cli/core/dataset_active_sql.py +++ b/src/pt_snap_cli/core/dataset_active_sql.py @@ -95,7 +95,7 @@ def reduce_active( used only to obtain their owning shard's full text, never as global identity. A frame extension, even partially covered, must use ordered-frame attribution. """ - if sources.dataset.ordered_frames_version == 1: + if sources.dataset.ordered_frames_version in (1, 2): return None required = _required_slices(sources, [item], _attachment_limit()) if required is None: diff --git a/src/pt_snap_cli/core/dataset_frame_tree.py b/src/pt_snap_cli/core/dataset_frame_tree.py new file mode 100644 index 0000000..3d23247 --- /dev/null +++ b/src/pt_snap_cli/core/dataset_frame_tree.py @@ -0,0 +1,195 @@ +"""Caller-path occupancy over the event owner's live set and allocation owners.""" + +from __future__ import annotations + +import hashlib +import json +from typing import cast + +from .dataset_attribution import _rounding_cursor, _scope +from .dataset_sources import DatasetSourceResolver +from .models import QueryResult + +METRICS = ( + ("block_count", "self_block_count"), + ("size_bytes", "self_bytes"), + ("requested_bytes", "self_requested_bytes"), +) + + +def _node( + node_id: str, + parent: str | None, + depth: int, + category: str, + label: str, + frame_id: int | None = None, + frame: tuple[str, int, str] | None = None, +) -> dict[str, object]: + return { + "node_id": node_id, + "parent_id": parent, + "depth": depth, + "frame_id": frame_id, + "category": category, + "label": label, + "filename": frame[0] if frame else None, + "line": frame[1] if frame else None, + "name": frame[2] if frame else None, + **{field: 0 for pair in METRICS for field in pair}, + } + + +def frame_tree( + sources: DatasetSourceResolver, + template: str, + params: dict[str, object], + max_rows: int | None, + exact_total: bool, + slice_index: int | None, + semantics_version: int | None, +) -> QueryResult: + event_id = cast(int, params["event_id"]) + owner = sources.target(event_id, slice_index) + groups = sources.read( + sources.dataset.root / owner.file, + "SELECT allocEventId, CASE WHEN allocEventId=-1 AND freeEventId=-1 THEN 'static' " + "WHEN allocEventId=-1 THEN 'preexisting_live_at_event' ELSE 'dynamic_live_at_event' END " + "AS category, COUNT(*) AS block_count, SUM(size) AS size_bytes, " + f'SUM(COALESCE(requestedSize,0)) AS requested_bytes FROM "block_{sources.device.device_id}" ' + "WHERE size>=? AND (allocEventId=-1 OR allocEventId<=?) " + "AND (freeEventId IS NULL OR freeEventId<0 OR freeEventId>?) " + + ("AND allocEventId!=-1 " if not params["include_static"] else "") + + "GROUP BY allocEventId,category ORDER BY allocEventId,category", + [params["min_size"], event_id, event_id], + ) + references = sources.events( + cast(int, g["allocEventId"]) for g in groups if cast(int, g["allocEventId"]) >= 0 + ) + root = _node("root", None, 0, "mixed", "[all active blocks]") + nodes = {"root": root} + frame_ids: dict[tuple[str, int, str], int] = {} + ordered_blocks = dynamic_blocks = resolved_blocks = dynamic_bytes = captured_bytes = 0 + for group in groups: + sources.budget.remaining() + category = cast(str, group["category"]) + source = references.get(cast(int, group["allocEventId"])) + frames = source.frames if source is not None and source.event["action"] == 4 else None + if category == "dynamic_live_at_event": + dynamic_blocks += cast(int, group["block_count"]) + dynamic_bytes += cast(int, group["size_bytes"]) + if source is not None and source.event["action"] == 4: + resolved_blocks += cast(int, group["block_count"]) + # A covered raw array can have nonstandard dictionaries. Preserve its + # occupancy, but never invent file/line/name from the rendered text. + usable = frames is not None and all( + isinstance(f.get("filename"), str) + and type(f.get("line")) is int + and isinstance(f.get("name"), str) + for f in frames + ) + parent = root + if category == "dynamic_live_at_event" and usable: + ordered_blocks += cast(int, group["block_count"]) + captured_bytes += cast(int, group["size_bytes"]) + path = [root] + if category == "dynamic_live_at_event" and usable and frames: + for depth, value in enumerate(reversed(frames), 1): + sources.budget.remaining() + frame = ( + cast(str, value["filename"]), + cast(int, value["line"]), + cast(str, value["name"]), + ) + frame_ids.setdefault(frame, len(frame_ids)) + identity = json.dumps( + [parent["node_id"], frame], ensure_ascii=True, separators=(",", ":") + ) + node_id = "frame/" + hashlib.sha256(identity.encode("utf-8")).hexdigest() + if node_id not in nodes: + nodes[node_id] = _node( + node_id, + cast(str, parent["node_id"]), + depth, + category, + f"{frame[0]}:{frame[1]} {frame[2]}", + frame_ids[frame], + frame, + ) + sources.budget.consume([nodes[node_id]]) + parent = nodes[node_id] + path.append(parent) + else: + kind = ( + category + if category != "dynamic_live_at_event" + else ("empty_callstack" if usable and frames == [] else "missing") + ) + label = { + "static": "[static]", + "preexisting_live_at_event": "[preexisting live]", + "empty_callstack": "[empty captured callstack]", + "missing": "[missing callstack]", + }[kind] + if kind not in nodes: + nodes[kind] = _node( + kind, + "root", + 1, + "empty_callstack" if kind == "empty_callstack" else category, + label, + ) + sources.budget.consume([nodes[kind]]) + parent = nodes[kind] + path.append(parent) + for total, own in METRICS: + amount = cast(int, group[total]) + for node in path: + node[total] = cast(int, node[total]) + amount + parent[own] = cast(int, parent[own]) + amount + rows = list(nodes.values()) + with _rounding_cursor(sources) as cursor: + for row in rows: + sources.budget.remaining() + row["percent_of_total"] = cursor.execute( + "SELECT ROUND(? * 100.0 / NULLIF(?,0),4)", (row["size_bytes"], root["size_bytes"]) + ).fetchone()[0] + rows.sort( + key=lambda r: ( + cast(int, r["depth"]), + cast(str, r["parent_id"] or ""), + -cast(int, r["size_bytes"]), + cast(str, r["node_id"]), + ) + ) + selected = rows[:max_rows] if max_rows is not None and max_rows > 0 else rows + sources.budget.consume(selected) + coverage = { + "range_complete": True, + "event_position": "after", + "active_blocks": root["block_count"], + "active_bytes": root["size_bytes"], + "dynamic_blocks": dynamic_blocks, + "dynamic_bytes": dynamic_bytes, + "allocation_events_resolved": resolved_blocks, + "allocation_source_complete": resolved_blocks == dynamic_blocks, + "ordered_frame_bytes": captured_bytes, + "ordered_frame_blocks": ordered_blocks, + "ordered_frames_complete": ordered_blocks == dynamic_blocks, + "frame_degradation": None if ordered_blocks == dynamic_blocks else "text_only_or_uncovered", + "frame_identity": "filename_line_name; frame_id is query-local, node_id is caller-path hash", + "source_queries": sources.query_count, + } + return QueryResult( + total=len(rows) if exact_total else len(selected), + returned=len(selected), + device_id=sources.device.device_id, + rows=selected, + template=template, + semantics_version=semantics_version, + has_more=len(selected) < len(rows), + truncated=len(selected) < len(rows), + total_is_exact=exact_total or len(selected) == len(rows), + timeout_s=sources.budget.timeout_s, + scope=_scope(sources, owner.index, event_id, coverage), + ) diff --git a/src/pt_snap_cli/core/dataset_frames.py b/src/pt_snap_cli/core/dataset_frames.py new file mode 100644 index 0000000..58b7f6e --- /dev/null +++ b/src/pt_snap_cli/core/dataset_frames.py @@ -0,0 +1,131 @@ +"""Read the version-2 normalized frame extension without reconstructing text.""" + +from __future__ import annotations + +import hashlib +import json +from collections import defaultdict +from pathlib import Path +from typing import Protocol, cast + +from .dataset_contract import _unique_object +from .frame_json import _bounded_depth, _finite_float, _invalid_constant + + +class FrameBudget(Protocol): + def remaining(self) -> float | None: ... + + +class FrameResolver(Protocol): + _frame_schemas: dict[Path, bool] + + @property + def budget(self) -> FrameBudget: ... + + def read( + self, path: Path, sql: str, values: list[object] | None = None + ) -> list[dict[str, object]]: ... + + +SCHEMA = { + "pt_snap_frame_value": [("id", "INTEGER", 1), ("frameJson", "TEXT", 0)], + "pt_snap_stack": [("stackId", "INTEGER", 1), ("frameCount", "INTEGER", 0)], + "pt_snap_stack_frame": [ + ("stackId", "INTEGER", 1), + ("frameIndex", "INTEGER", 2), + ("frameId", "INTEGER", 0), + ], + "pt_snap_event_stack": [("eventId", "INTEGER", 1), ("stackId", "INTEGER", 0)], +} + + +def attach_frames( + resolver: FrameResolver, path: Path, ids: list[int] +) -> dict[int, tuple[list[dict[str, object]], str]]: + result: dict[int, tuple[list[dict[str, object]], str]] = {} + batch_size = 256 + + if path not in resolver._frame_schemas: + valid = True + for table, columns in SCHEMA.items(): + kind = resolver.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) + schema = resolver.read(path, f'PRAGMA table_info("{table}")') + if ( + kind != [{"type": "table"}] + or [(r["name"], r["type"], r["pk"]) for r in schema] != columns + ): + valid = False + break + resolver._frame_schemas[path] = valid + if not resolver._frame_schemas[path]: + return result + for start in range(0, len(ids), batch_size): + batch = ids[start : start + batch_size] + marks = ",".join("?" for _ in batch) + coverage = resolver.read( + path, + "SELECT e.eventId,e.stackId,s.frameCount FROM pt_snap_event_stack e " + f"LEFT JOIN pt_snap_stack s ON s.stackId=e.stackId WHERE e.eventId IN ({marks})", + list(batch), + ) + entries = resolver.read( + path, + "SELECT sf.stackId,sf.frameIndex,sf.frameId,f.frameJson FROM pt_snap_stack_frame sf " + "LEFT JOIN pt_snap_frame_value f ON f.id=sf.frameId " + f"WHERE sf.stackId IN (SELECT stackId FROM pt_snap_event_stack WHERE eventId IN ({marks})) " + "ORDER BY sf.stackId,sf.frameIndex", + list(batch), + ) + grouped: dict[int, list[dict[str, object]]] = defaultdict(list) + for row in entries: + grouped[cast(int, row["stackId"])].append(row) + decoded_stacks: dict[int, tuple[list[dict[str, object]], str] | None] = {} + for row in coverage: + resolver.budget.remaining() + event, stack, count = row["eventId"], row["stackId"], row["frameCount"] + # The SQL IN clause already restricts eventId to this batch. Do not + # scan the full shard's allocation list for each returned event. + if type(stack) is not int or type(count) is not int or count < 0: + continue + stack = cast(int, stack) + if stack not in decoded_stacks: + decoded_stacks[stack] = None + frames = grouped[stack] + if len(frames) != count or any( + type(frame["frameIndex"]) is not int + or frame["frameIndex"] != index + or type(frame["frameId"]) is not int + or not isinstance(frame["frameJson"], str) + for index, frame in enumerate(frames) + ): + continue + try: + decoded = [ + json.loads( + cast(str, frame["frameJson"]), + object_pairs_hook=_unique_object, + parse_constant=_invalid_constant, + parse_float=_finite_float, + ) + for frame in frames + ] + if not all(isinstance(frame, dict) for frame in decoded) or not _bounded_depth( + decoded + ): + continue + canonical = json.dumps( + decoded, + sort_keys=True, + ensure_ascii=True, + separators=(",", ":"), + allow_nan=False, + ) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + except (ValueError, TypeError, RecursionError, OverflowError): + continue + decoded_stacks[stack] = (decoded, digest) + value = decoded_stacks[stack] + if value is not None: + decoded, digest = value + result[cast(int, event)] = (decoded, digest) + return result diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 4c6101e..fd8415c 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -471,7 +471,7 @@ def _finish_stacks(sources, grouped, params, scope, events, ordered): def _stacks(sources, low, high, items, params, scope): - if sources.dataset.ordered_frames_version != 1: + if sources.dataset.ordered_frames_version not in (1, 2): return _text_stacks(sources, low, high, items, params, scope) grouped = {} events = ordered = 0 @@ -485,13 +485,34 @@ def _stacks(sources, low, high, items, params, scope): join = " LEFT JOIN callstack cs ON t.callstackId=cs.id" else: projection += ", t.callstack" - batch = sources.read( - sources.dataset.root / item.file, - f"SELECT {projection} FROM {trace} t{join} " - "WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", - [low, high], - ) - refs = sources.events_from_rows(item, batch) + weights = None + if sources.dataset.ordered_frames_version == 2: + from .dataset_frames import attach_frames + + path = sources.dataset.root / item.file + attach_frames(sources, path, []) # Validate optional schema, no frame fetch. + text = "cs.callstack" if sources.dataset.callstack_layout == "v2" else "t.callstack" + stack_group = "" + if sources._frame_schemas[path]: + join += " LEFT JOIN pt_snap_event_stack es ON es.eventId=t.id" + stack_group = "es.stackId," + batch = sources.read( + path, + "SELECT MIN(t.id) AS id, COUNT(*) AS alloc_count, " + f"SUM(t.size) AS total_size, MAX(t.size) AS max_size FROM {trace} t{join} " + f"WHERE t.id>=0 AND t.id>=? AND t.id<=? GROUP BY {stack_group}{text} COLLATE BINARY", + [low, high], + ) + weights = {row["id"]: row for row in batch} + refs = sources.events(cast(int, row["id"]) for row in batch) + else: + batch = sources.read( + sources.dataset.root / item.file, + f"SELECT {projection} FROM {trace} t{join} " + "WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", + [low, high], + ) + refs = sources.events_from_rows(item, batch) for source in refs.values(): sources.budget.remaining() text = source.event.get("callstack") @@ -521,14 +542,23 @@ def _stacks(sources, low, high, items, params, scope): "frames_status": source.frames_status, }, ) - row["alloc_count"] += 1 - row["total_size"] += source.event["size"] - row["max_size"] = max(row["max_size"], source.event["size"]) + weight = ( + weights[source.event["id"]] + if weights is not None + else { + "alloc_count": 1, + "total_size": source.event["size"], + "max_size": source.event["size"], + } + ) + row["alloc_count"] += weight["alloc_count"] + row["total_size"] += weight["total_size"] + row["max_size"] = max(row["max_size"], weight["max_size"]) if row["text_kind"] != "captured" and source.text_kind == "captured": row["callstack"], row["stack_event_id"] = text, source.event["id"] row["text_kind"] = "captured" - events += 1 - ordered += source.frames is not None + events += weight["alloc_count"] + ordered += weight["alloc_count"] if source.frames is not None else 0 return _finish_stacks(sources, grouped, params, scope, events, ordered) @@ -636,7 +666,8 @@ def global_query( if template == "event": refs = sources.events(cast(int, r["id"]) for r in selected) detailed = ( - sources.dataset.ordered_frames_version == 1 or cast(int, params["stack_bytes"]) >= 0 + sources.dataset.ordered_frames_version in (1, 2) + or cast(int, params["stack_bytes"]) >= 0 ) selected = [dict(refs[cast(int, r["id"])].event) for r in selected] if detailed: diff --git a/src/pt_snap_cli/core/dataset_resolver.py b/src/pt_snap_cli/core/dataset_resolver.py index bf241a4..4004697 100644 --- a/src/pt_snap_cli/core/dataset_resolver.py +++ b/src/pt_snap_cli/core/dataset_resolver.py @@ -36,6 +36,7 @@ DatabaseSchemaError, InvalidDeviceError, InvalidParameterError, + QueryCancelledError, QueryTimeoutError, ) from pt_snap_cli.core.native_dataset_contract import ( @@ -47,7 +48,10 @@ validate_native_dataset, validate_native_scope, ) -from pt_snap_cli.core.snapshot_cache import validate_cache +from pt_snap_cli.core.snapshot_cache import ( + parse_cache_manifest, + validate_cache, +) from pt_snap_cli.core.validation_budget import ( ValidationBudget, check_budget, @@ -55,6 +59,12 @@ sqlite_timeout, validation_progress, ) +from pt_snap_cli.core.validation_receipts import ( + import_validation_active, + load_receipt, + receipts_enabled, + save_receipt, +) _FileStamp = tuple[int, int, int, int, int, int, int] _Generation = tuple[tuple[str, _FileStamp], ...] @@ -67,6 +77,7 @@ class ResolvedDataset: fingerprint: str ordered_frames_version: int | None = None generation: _Generation | None = None + content_verified: bool = True def require_unchanged(self, budget: ValidationBudget | None = None) -> None: """Guard a reused report/query resolution without repeating full validation.""" @@ -76,7 +87,7 @@ def require_unchanged(self, budget: ValidationBudget | None = None) -> None: current = _generation(self.root, self.validation.manifest, budget) if self.generation is None or current != self.generation: raise DatabaseSchemaError("Dataset members changed during operation; retry.") - if not _cacheable(current): + if self.content_verified and not _cacheable(current): if ( _content_fingerprint(self.root, self.validation.manifest, budget) != self.fingerprint @@ -140,7 +151,7 @@ def to_dict(self) -> dict[str, object]: "cross_slice_queries": True, "supported_queries": dict(DATASET_SUPPORT), "cross_slice_sources": True, - "ordered_frames_reader_version": 1, + "ordered_frames_reader_version": 2, "structured_frames_scope": "validation_only; query source coverage is separate", "devices": [ { @@ -303,7 +314,10 @@ class DatasetResolver: """ def inspect( - self, path: Path | str, *, budget: ValidationBudget | None = None + self, + path: Path | str, + *, + budget: ValidationBudget | None = None, ) -> ResolvedDataset | None: candidate = Path(path).expanduser().absolute() with _CACHE_LOCK: @@ -320,6 +334,10 @@ def inspect( _ = _VALIDATED.pop(cache_key, None) return None cache_key = root + if import_validation_active(): + raw = parse_cache_manifest((root / "manifest.json").read_bytes()) + if raw.get("format") != NATIVE_FORMAT: + return self._inspect_compatible_structure(root, raw, budget) strong_cache = os.environ.get("PT_SNAP_DATASET_CACHE") is None with _CACHE_LOCK: cached = _VALIDATED.get(root) @@ -380,16 +398,32 @@ def inspect( # Native validation already checks both ends against declared hashes. compatibility_before = ( _content_fingerprint(root, planned, budget, manifest_hash=before) - if strong_cache and not isinstance(planned, NativeManifest) + if strong_cache and (not isinstance(planned, NativeManifest) or receipts_enabled()) else None ) - validation = ( - validate_native_dataset(root, planned, budget=budget) - if isinstance(planned, NativeManifest) - else validate_dataset(root, budget=budget) + receipt = ( + load_receipt(root, "dataset", compatibility_before) + if compatibility_before is not None + else None ) + counts = ( + [receipt.get(key) for key in ("real", "boundary")] if receipt is not None else [] + ) + receipt_hit = len(counts) == 2 and all(type(n) is int and n >= 0 for n in counts) + if receipt_hit: + assert receipt is not None + real, boundary = receipt["real"], receipt["boundary"] + assert isinstance(real, int) and isinstance(boundary, int) + validation = DatasetValidation(planned, real, boundary) + else: + validation = ( + validate_native_dataset(root, planned, budget=budget) + if isinstance(planned, NativeManifest) + else validate_dataset(root, budget=budget) + ) if ( - not isinstance(planned, NativeManifest) + not receipt_hit + and not isinstance(planned, NativeManifest) and isinstance(raw, dict) and ("parserCache" in raw or "ptSnap" in raw) ): @@ -417,7 +451,7 @@ def inspect( ) from exc digest = hashlib.sha256() digest.update(before.encode("ascii")) - for device in checked_rows(validation.manifest.devices, budget): + for device in checked_rows(() if receipt_hit else validation.manifest.devices, budget): for item in checked_rows(device.slices, budget): digest.update(item.file.encode("utf-8")) # Native validation already verifies this digest before and @@ -432,7 +466,8 @@ def inspect( if wal.exists(): digest.update(b"-wal") digest.update(_hash_file(wal, budget=budget).encode("ascii")) - fingerprint = digest.hexdigest() + fingerprint = compatibility_before if receipt_hit else digest.hexdigest() + assert fingerprint is not None if compatibility_before is not None and compatibility_before != fingerprint: raise DatabaseSchemaError("Dataset content changed during inspection; retry.") if before != _hash_file(manifest_path, budget=budget): @@ -449,6 +484,16 @@ def inspect( version if type(version) is int else None, generation, ) + check_budget(budget) + save_receipt( + root, + "dataset", + fingerprint, + { + "real": validation.real_event_count, + "boundary": validation.boundary_event_count, + }, + ) if strong_cache or _cacheable(generation): entry = _ValidatedEntry( generation, @@ -475,11 +520,70 @@ def inspect( with _CACHE_LOCK: _ = _VALIDATED.pop(cache_key, None) raise DatabaseSchemaError("Invalid dataset: manifest nesting is too deep") from exc - except (DatabaseSchemaError, QueryTimeoutError): + except (DatabaseSchemaError, QueryTimeoutError, QueryCancelledError): with _CACHE_LOCK: _ = _VALIDATED.pop(cache_key, None) raise + @staticmethod + def _inspect_compatible_structure( + root: Path, raw: dict[str, object], budget: ValidationBudget | None + ) -> ResolvedDataset: + manifest = parse_manifest(raw, budget=budget) + _generation(root, manifest, budget) # Preflight all members before SQLite opens. + _prepare_compatibility_transport(root, manifest, budget) + generation = _generation(root, manifest, budget) + attestation = raw.get("parserCache", {}) + producer = raw.get("ptSnap", {}) + if not isinstance(attestation, dict) or not isinstance(producer, dict): + raise DatabaseSchemaError("Invalid cache extension") + identity = producer.get("identity", {}) + if not isinstance(identity, dict): + raise DatabaseSchemaError("Invalid producer identity") + selected = attestation.get("requestedDevice") + if selected is None: + selected = identity.get("requestedDevice") + cache_hash = raw.get("cacheHash") + if not isinstance(cache_hash, str): + raise DatabaseSchemaError("Invalid cacheHash") + checked = validate_cache(root, cache_hash, selected, budget=budget) + if checked != raw: + raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") + boundary = 0 + for device in manifest.devices: + for item in device.slices: + with ( + closing( + sqlite3.connect( + readonly_member_uri(root / item.file), + uri=True, + timeout=sqlite_timeout(budget), + ) + ) as conn, + validation_progress(conn, budget), + ): + # Count only synthetic IDs through the integer primary key; + # never fetch/encode callstacks or scan block contents. + boundary += conn.execute( + f'SELECT COUNT(*) FROM "trace_entry_{device.device_id}" WHERE id<0' # nosec B608 + ).fetchone()[0] + if raw != parse_cache_manifest( + (root / "manifest.json").read_bytes() + ) or generation != _generation(root, manifest, budget): + raise DatabaseSchemaError("Dataset changed during inspection; retry.") + extensions = raw.get("extensions", {}) + frames = extensions.get("ptSnapOrderedFrames", {}) if isinstance(extensions, dict) else {} + version = frames.get("version") if isinstance(frames, dict) else None + fingerprint = "structure-" + hashlib.sha256(repr(generation).encode()).hexdigest() + return ResolvedDataset( + root, + DatasetValidation(manifest, sum(d.event_count for d in manifest.devices), boundary), + fingerprint, + version if type(version) is int else None, + generation, + False, + ) + def completion_device_ids(self, path: Path | str) -> list[int] | None: """Bounded manifest-only hints, never evidence of dataset validity. diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 550dfb8..5343ea5 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -8,12 +8,11 @@ import hashlib import json -import math import sqlite3 import time from bisect import bisect_right from collections import defaultdict -from collections.abc import Iterable +from collections.abc import Callable, Iterable from dataclasses import dataclass from pathlib import Path from typing import cast @@ -26,9 +25,11 @@ from pt_snap_cli.core.errors import ( DatabaseSchemaError, InvalidParameterError, + QueryCancelledError, QueryExecutionError, QueryTimeoutError, ) +from pt_snap_cli.core.frame_json import _bounded_depth, _finite_float, _invalid_constant from pt_snap_cli.core.query_limits import ( DEFAULT_MAX_SCRATCH_DB_BYTES, DEFAULT_MAX_WORK_BYTES, @@ -36,36 +37,12 @@ QueryLimits, ) from pt_snap_cli.query.executor import QueryExecutionError as ExecutorError -from pt_snap_cli.query.executor import QueryExecutor +from pt_snap_cli.query.executor import QueryExecutor, cancellation_options from pt_snap_cli.query.executor import QueryTimeoutError as ExecutorTimeout SOURCE_BATCH_SIZE = 256 -def _invalid_constant(value: str) -> object: - raise ValueError(f"Non-JSON numeric constant: {value}") - - -def _finite_float(value: str) -> float: - result = float(value) - if not math.isfinite(result): - raise ValueError("Non-finite frame number") - return result - - -def _bounded_depth(value: object) -> bool: - pending = [(value, 0)] - while pending: - current, depth = pending.pop() - if depth > 128: - return False - if isinstance(current, dict): - pending.extend((item, depth + 1) for item in current.values()) - elif isinstance(current, list): - pending.extend((item, depth + 1) for item in current) - return True - - @dataclass class QueryBudget: timeout_s: float | None @@ -77,6 +54,7 @@ class QueryBudget: # Per-operation derived database pages, not total SQLite temp space or RSS. # Zero retains the bounded in-Python fallback. max_scratch_db_bytes: int = DEFAULT_MAX_SCRATCH_DB_BYTES + cancelled: Callable[[], bool] | None = None @classmethod def from_limits( @@ -121,6 +99,8 @@ def consume(self, rows: list[dict[str, object]]) -> None: ) def remaining(self) -> float | None: + if self.cancelled is not None and self.cancelled(): + raise QueryCancelledError("Frame tree query cancelled by a newer request or reset.") if self.timeout_s is None: return None left = self.timeout_s - (time.monotonic() - self.started) @@ -228,13 +208,19 @@ def read( ) self.query_count += 1 return QueryExecutor(context).execute( - sql, values, timeout_s=self.budget.remaining(), consume=self._consume + sql, + values, + timeout_s=self.budget.remaining(), + consume=self._consume, + **cancellation_options(self.budget.cancelled), ) except ExecutorTimeout as exc: + self.budget.remaining() raise QueryTimeoutError( f"Query timed out after {self.budget.timeout_s} seconds" ) from exc except ExecutorError as exc: + self.budget.remaining() raise QueryExecutionError(str(exc)) from exc except (DatabaseNotFoundError, SchemaVersionError, sqlite3.DatabaseError) as exc: raise DatabaseSchemaError(str(exc)) from exc @@ -332,6 +318,22 @@ def _ensure_frames(self, item: SliceRecord, ids: list[int]) -> None: self._frames_resolved.update(pending) def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSource]) -> None: + if self.dataset.ordered_frames_version == 2: + from .dataset_frames import attach_frames + + decoded = attach_frames(self, path, ids) + for event, (frames, digest) in decoded.items(): + old = sources[event] + sources[event] = EventSource( + old.event, + old.slice_index, + f"{self.namespace}:frames:sha256:{digest}", + "captured", + frames, + "ordered", + old.local_stack_id, + ) + return # This is a NEW, explicit read contract, not a guessed original-producer # schema. Existing pt_snap_block_reference has formatted text only. if self.dataset.ordered_frames_version != 1: diff --git a/src/pt_snap_cli/core/dataset_support.py b/src/pt_snap_cli/core/dataset_support.py index 0c60053..3a712de 100644 --- a/src/pt_snap_cli/core/dataset_support.py +++ b/src/pt_snap_cli/core/dataset_support.py @@ -9,6 +9,7 @@ "allocator_gap": "global_peaks_same_event_counters", "active_blocks_at_event": "owner_event_active_set_cross_slice_sources", "active_memory_callstack_at_event": "owner_event_global_source_groups_before_top_n", + "active_memory_frame_tree_at_event": "owner_event_caller_paths_cross_slice_ordered_frames", "preexisting_live": "owner_event_preexisting_occupancy", "block": "proved_lifecycles_dedup_latest_observation", "freed_block_lifetime": "proved_completed_lifecycles_once", diff --git a/src/pt_snap_cli/core/error_codes.py b/src/pt_snap_cli/core/error_codes.py index 54b9ed4..ec4ac72 100644 --- a/src/pt_snap_cli/core/error_codes.py +++ b/src/pt_snap_cli/core/error_codes.py @@ -14,6 +14,7 @@ InvalidCategoryError, InvalidDeviceError, InvalidParameterError, + QueryCancelledError, QueryExecutionError, QueryTimeoutError, SkillError, @@ -32,6 +33,7 @@ DATABASE_SCHEMA_INVALID = "DATABASE_SCHEMA_INVALID" QUERY_FAILED = "QUERY_FAILED" QUERY_TIMEOUT = "QUERY_TIMEOUT" +QUERY_CANCELLED = "QUERY_CANCELLED" IMPORT_FAILED = "IMPORT_FAILED" IMPORT_BACKEND_MISSING = "IMPORT_BACKEND_MISSING" SNAPSHOT_INVALID = "SNAPSHOT_INVALID" @@ -87,6 +89,8 @@ def classify_error(exc: BaseException) -> tuple[str, str | None]: INVALID_PARAMETER, "Use a numeric --timeout, or set PT_SNAP_QUERY_TIMEOUT to a number of seconds.", ) + if isinstance(exc, QueryCancelledError): + return QUERY_CANCELLED, "Use the result of the latest frame-tree request." if isinstance(exc, QueryTimeoutError): return ( QUERY_TIMEOUT, diff --git a/src/pt_snap_cli/core/errors.py b/src/pt_snap_cli/core/errors.py index 7f841df..64a6dc2 100644 --- a/src/pt_snap_cli/core/errors.py +++ b/src/pt_snap_cli/core/errors.py @@ -56,6 +56,10 @@ class QueryTimeoutError(QueryExecutionError): pass +class QueryCancelledError(QueryExecutionError): + """An obsolete query was interrupted; no partial result is returned.""" + + class ImportToolMissingError(PtSnapCoreError): """Raised when the built-in snapshot import backend is unavailable. diff --git a/src/pt_snap_cli/core/frame_json.py b/src/pt_snap_cli/core/frame_json.py new file mode 100644 index 0000000..a45cda9 --- /dev/null +++ b/src/pt_snap_cli/core/frame_json.py @@ -0,0 +1,11 @@ +"""Strict JSON validation shared by optional ordered frame readers.""" + +from pt_snap_cli.snapshot.util.frame_json import ( + _bounded_depth as _bounded_depth, +) +from pt_snap_cli.snapshot.util.frame_json import ( + _finite_float as _finite_float, +) +from pt_snap_cli.snapshot.util.frame_json import ( + _invalid_constant as _invalid_constant, +) diff --git a/src/pt_snap_cli/core/import_service.py b/src/pt_snap_cli/core/import_service.py index 8f5df42..0525fbc 100644 --- a/src/pt_snap_cli/core/import_service.py +++ b/src/pt_snap_cli/core/import_service.py @@ -7,9 +7,13 @@ from dataclasses import replace from pathlib import Path -from pt_snap_cli.core.dataset_contract import parse_manifest, validate_dataset +from pt_snap_cli.core.dataset_contract import parse_manifest from pt_snap_cli.core.dataset_import_backend import DatasetImportBackend -from pt_snap_cli.core.dataset_resolver import DatasetResolver, _require_readonly_member +from pt_snap_cli.core.dataset_resolver import ( + DatasetResolver, + ResolvedDataset, + _require_readonly_member, +) from pt_snap_cli.core.errors import ( DatabaseSchemaError, FocusFileInvalidError, @@ -43,9 +47,15 @@ build_native_manifest, cache_identity, ) +from pt_snap_cli.core.ordered_frame_integrity import frame_member_digest, require_frame_declaration from pt_snap_cli.core.sharded_replay_service import ShardedReplayResult -from pt_snap_cli.core.snapshot_cache import parse_cache_manifest, source_hashes, validate_cache +from pt_snap_cli.core.snapshot_cache import ( + parse_cache_manifest, + source_hashes, + validate_cache, +) from pt_snap_cli.core.snapshot_import_backend import SnapshotImportBackend +from pt_snap_cli.core.validation_receipts import import_receipts, load_receipt, save_receipt logger = logging.getLogger(__name__) @@ -69,6 +79,10 @@ def __init__( self._msinsight_backend = MsinsightImportBackend() def import_snapshot(self, options: ImportOptions) -> ImportResult: + with import_receipts(): + return self._import_snapshot(options) + + def _import_snapshot(self, options: ImportOptions) -> ImportResult: self._validate_options(options) self._validate_snapshot_file(options.snapshot_file) if options.format in (COMPATIBILITY_FORMAT, "msinsight"): @@ -204,6 +218,20 @@ def _import_dataset(self, options: ImportOptions) -> ImportResult: and dataset is not None and manifest.identity == cache_identity(source_hash, options.device, capacity) ): + try: + require_frame_declaration( + parse_cache_manifest((target / "manifest.json").read_bytes()) + ) + if load_receipt(target, "native-frames", dataset.fingerprint) != {}: + for device in manifest.devices: + for item in device.slices: + frame_member_digest(target, target / item.file, device.device_id) + dataset.require_unchanged() + save_receipt(target, "native-frames", dataset.fingerprint, {}) + except (ValueError, OSError, sqlite3.DatabaseError, DatabaseSchemaError) as exc: + raise ImportExecutionError( + f"Existing native ordered-frame evidence is invalid; use --force to rebuild. Existing target and focus preserved: {exc}" + ) from exc self._recheck_source(options.snapshot_file, source_hash) focus_state = self._set_focus_if_requested(target, options) return ImportResult( @@ -239,7 +267,9 @@ def finalize(result: ShardedReplayResult) -> None: for item in result.slices: _require_readonly_member(result.directory, item.file) self._metadata_service.write(item.file, metadata) + frame_member_digest(result.directory, item.file, item.device) manifest = build_native_manifest(result, metadata, capacity) + require_frame_declaration(manifest) with (result.directory / "manifest.json").open("x", encoding="utf-8") as file: json.dump(manifest, file, indent=2) validated = DatasetResolver().inspect(result.directory) @@ -247,6 +277,8 @@ def finalize(result: ShardedReplayResult) -> None: raise ImportExecutionError( "Staged dataset validation did not produce a complete artifact." ) + self._remember_published_validation(target, validated) + save_receipt(target, "native-frames", validated.fingerprint, {}) completed, generated = metadata, result focus_state: FocusState | None = None @@ -310,9 +342,6 @@ def _import_compatible_dataset(self, options: ImportOptions) -> ImportResult: "Explicit events_per_slice does not match the existing compatible " "cache; choose a new output directory." ) - # A self-consistent seal is not proof of semantic validity. - # Use shared P0 without repeating the resolver's seal scan. - validate_dataset(target) self._recheck_source(options.snapshot_file, source_hash) return ImportResult( target, @@ -420,6 +449,22 @@ def focus(path: Path) -> None: generated.omitted_devices, ) + @staticmethod + def _remember_published_validation(target: Path, validated: ResolvedDataset) -> None: + if not validated.content_verified: + return + # Publication only renames the staged directory. The receipt is bound to + # all manifest/member bytes, so another artifact at target cannot hit it. + save_receipt( + target, + "dataset", + validated.fingerprint, + { + "real": validated.validation.real_event_count, + "boundary": validated.validation.boundary_event_count, + }, + ) + def _hash_source(self, source: Path) -> str: try: return self._metadata_service.calculate_sha256(source) diff --git a/src/pt_snap_cli/core/latest_query.py b/src/pt_snap_cli/core/latest_query.py new file mode 100644 index 0000000..da3abf4 --- /dev/null +++ b/src/pt_snap_cli/core/latest_query.py @@ -0,0 +1,53 @@ +"""Latest-request-wins coordination without sharing SQLite connections.""" + +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from threading import Event, Lock + +from .errors import QueryCancelledError + + +class LatestQuery: + def __init__(self) -> None: + self._state = Lock() + self._work = Lock() + self._active: Event | None = None + + def cancel(self) -> None: + with self._state: + if self._active is not None: + self._active.set() + + @contextmanager + def run( + self, remaining: Callable[[], float | None] | None = None + ) -> Iterator[Callable[[], bool]]: + token = Event() + with self._state: + if self._active is not None: + self._active.set() + self._active = token + try: + # The old task must unwind and close its own SQLite handles first. + while True: + self._check(token) + left = remaining() if remaining is not None else None + if self._work.acquire(timeout=min(left, 0.05) if left is not None else 0.05): + break + try: + self._check(token) + if remaining is not None: + remaining() + yield token.is_set + self._check(token) + finally: + self._work.release() + finally: + with self._state: + if self._active is token: + self._active = None + + @staticmethod + def _check(token: Event) -> None: + if token.is_set(): + raise QueryCancelledError("Frame tree query cancelled by a newer request or reset.") diff --git a/src/pt_snap_cli/core/msinsight_export.py b/src/pt_snap_cli/core/msinsight_export.py index 6ff24e2..f1ec7df 100644 --- a/src/pt_snap_cli/core/msinsight_export.py +++ b/src/pt_snap_cli/core/msinsight_export.py @@ -23,20 +23,26 @@ TRACE_COLUMNS, DatasetContractError, DatasetManifest, - validate_dataset, + parse_manifest, ) -from .dataset_files import hash_file, readonly_member_uri, require_readonly_member +from .dataset_files import readonly_member_uri, require_readonly_member from .dataset_import_backend import DatasetImportBackend +from .dataset_resolver import DatasetResolver from .errors import ImportExecutionError from .import_metadata_contract import metadata_from_mapping from .models import ImportMetadata +from .ordered_frame_integrity import require_frame_declaration from .sharded_replay_service import ShardedReplayResult -from .snapshot_cache import parse_cache_manifest, seal_manifest, source_hashes, validate_cache +from .snapshot_cache import ( + parse_cache_manifest, + seal_manifest, + source_hashes, +) COMPATIBILITY_FORMAT = "compatibility-v1" DEFAULT_CAPACITY = 500000 # Independent semantic versions, not the installed CLI version or GUI cacheHash. -EXPORT_VERSION = 2 +EXPORT_VERSION = 3 REFERENCE_VERSION = 1 @@ -219,7 +225,6 @@ def build_compatible_manifest( cache_hash: str, ) -> dict[str, object]: devices: dict[str, object] = {} - files: dict[str, str] = {} for device in sorted({s.device for s in result.slices}): slices = [s for s in result.slices if s.device == device] devices[str(device)] = { @@ -244,18 +249,16 @@ def build_compatible_manifest( "cacheHash": cache_hash, "eventsPerSlice": capacity, "devices": devices, + "extensions": {"ptSnapOrderedFrames": {"version": 2}}, "ptSnap": { "identity": compatible_identity( metadata.source_sha256, metadata.requested_device, capacity ), "metadata": asdict(metadata), "omittedDevices": list(result.omitted_devices), - "members": files, }, } seal_manifest(result.directory, manifest, metadata.requested_device) - manifest["ptSnap"]["members"] = manifest["parserCache"]["members"] - manifest["ptSnap"]["memberHashKind"] = "parser-v1" return manifest @@ -270,8 +273,8 @@ class CompatibleCache: def inspect_owned_cache(target: Path) -> CompatibleCache: """Whole valid recognized identity only; salted hash alone is never ownership. - Base tables are attested logically. Consumer-owned tables and indexes may - change without invalidating parser data. Legacy exports retain byte checks. + Reuse checks source identity, manifest inventory and required DB structure. + Consumer-owned tables and indexes do not invalidate the cache. """ data = parse_cache_manifest((target / "manifest.json").read_bytes()) extension = data.get("ptSnap") @@ -280,9 +283,8 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: identity = extension.get("identity") if not isinstance(identity, dict): raise ValueError("manifest.ptSnap.identity: expected object") - for key in ("metadata", "members"): - if not isinstance(extension.get(key), dict): - raise ValueError(f"manifest.ptSnap.{key}: expected object") + if not isinstance(extension.get("metadata"), dict): + raise ValueError("manifest.ptSnap.metadata: expected object") if not isinstance(extension.get("omittedDevices"), list): raise ValueError("manifest.ptSnap.omittedDevices: expected array") version = identity.get("exportContractVersion") @@ -290,9 +292,8 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: raise ValueError( f"Legacy export contract v{version}; choose a new --output-dir to re-export as v{EXPORT_VERSION}" ) - # Logical seals establish consistency, not semantic validity. Keep shared P0 - # checks for every producer, including imports that do not register focus. - manifest = validate_dataset(target).manifest + # Reject declaration/identity failures before inspecting member structure. + manifest = parse_manifest(data) extension = data["ptSnap"] metadata = metadata_from_mapping(extension["metadata"]) identity = extension["identity"] @@ -310,31 +311,23 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: ): raise ValueError("Metadata must match physical v1 layout") paths = {s.file for d in manifest.devices for s in d.slices} + require_frame_declaration(data) + resolved = DatasetResolver().inspect(target) + if resolved is None: + raise ValueError("Compatible dataset is missing") + if parse_cache_manifest((target / "manifest.json").read_bytes()) != data: + raise ValueError("Compatible manifest changed during inspection") if any((target / (relative + "-journal")).exists() for relative in paths): raise ValueError( "Snapshot member has a rollback journal; a writer or recovery may be active, retry later" ) - if set(extension["members"]) != paths: - raise ValueError("Incomplete member attestation") expected = {"manifest.json", "parse.log", *paths, *(str(Path(p).parent) for p in paths)} expected.update(relative + suffix for relative in paths for suffix in ("-wal", "-shm")) actual = {p.relative_to(target).as_posix() for p in target.rglob("*")} if not actual <= expected or any(p.is_symlink() for p in target.rglob("*")): raise ValueError("Unrecognized files or aliases in compatible cache") - logical_members = None - if extension.get("memberHashKind") == "parser-v1": - checked = validate_cache(target, data["cacheHash"], metadata.requested_device) - if checked != data: - raise ValueError("Compatible manifest changed during inspection") - # validate_cache checks the producer seal even without parserCache. - logical_members = checked["ptSnap"]["members"] - elif "memberHashKind" in extension: - raise ValueError("Unsupported member hash contract") for relative in paths: member = target / relative - digest = logical_members[relative] if logical_members is not None else hash_file(member) - if digest != extension["members"][relative]: - raise ValueError("Compatible member hash changed") require_readonly_member(target, member, immutable=True) with closing(sqlite3.connect(readonly_member_uri(member), uri=True)) as conn: conn.row_factory = sqlite3.Row @@ -364,4 +357,5 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: raise ValueError("Device selection disagrees with identity") if Path(manifest.source_file).name != metadata.source_name: raise ValueError("Source path disagrees with metadata") + resolved.require_unchanged() return CompatibleCache(manifest, identity, metadata, tuple(omitted)) diff --git a/src/pt_snap_cli/core/native_dataset_contract.py b/src/pt_snap_cli/core/native_dataset_contract.py index ffbd8bc..cc687ae 100644 --- a/src/pt_snap_cli/core/native_dataset_contract.py +++ b/src/pt_snap_cli/core/native_dataset_contract.py @@ -53,7 +53,7 @@ # Semantic contracts, deliberately independent of the installed CLI __version__. NATIVE_FORMAT_VERSION = 1 NATIVE_IMPORT_VERSION = 1 -NATIVE_EXTENSION_VERSION = 1 +NATIVE_EXTENSION_VERSION = 2 @dataclass(frozen=True) @@ -314,6 +314,11 @@ def validate_native_dataset( manifest.metadata.requested_device, manifest.events_per_slice, ) + # Read old text-only artifacts; import reuse still requires the current + # identity in ImportService. Optional frame support is declared separately. + extension = manifest.identity.get("extensionContractVersion") + if extension in (1, NATIVE_EXTENSION_VERSION): + expected_identity["extensionContractVersion"] = extension _need( manifest.identity == expected_identity, "identity", @@ -624,5 +629,6 @@ def build_native_manifest( "identity": cache_identity(metadata.source_sha256, metadata.requested_device, capacity), "metadata": asdict(metadata), "omittedDevices": list(result.omitted_devices), + "extensions": {"ptSnapOrderedFrames": {"version": 2}}, "devices": devices, } diff --git a/src/pt_snap_cli/core/ordered_frame_integrity.py b/src/pt_snap_cli/core/ordered_frame_integrity.py new file mode 100644 index 0000000..e64a0d9 --- /dev/null +++ b/src/pt_snap_cli/core/ordered_frame_integrity.py @@ -0,0 +1,98 @@ +"""Producer-owned frame validation, separate from consumer/base-table seals.""" + +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +from .dataset_contract import _unique_object +from .dataset_files import readonly_member_uri, require_readonly_member +from .dataset_frames import SCHEMA +from .frame_json import _bounded_depth, _finite_float, _invalid_constant + + +def require_frame_declaration(manifest: dict[str, object]) -> None: + extensions = manifest.get("extensions") + frames = extensions.get("ptSnapOrderedFrames") if isinstance(extensions, dict) else None + if ( + not isinstance(frames, dict) + or type(frames.get("version")) is not int + or frames["version"] != 2 + ): + raise ValueError("Producer ordered-frame declaration is missing or unsupported") + + +def frame_member_digest(root: Path, member: Path, device: int) -> str: + """Validate normalized evidence and stream its exact logical content. + + Event coverage may be partial: unencodable optional evidence is intentionally + omitted by the writer. Every stored reference and frame must still be valid. + Consumer tables, columns and indexes outside this extension are not hashed. + """ + if type(device) is not int or device < 0: + raise ValueError("Invalid ordered-frame device") + require_readonly_member(root, member, immutable=True) + digest = hashlib.sha256() + with closing(sqlite3.connect(readonly_member_uri(member), uri=True)) as conn: + conn.execute("PRAGMA query_only=ON") + conn.execute("BEGIN") + for table, schema in SCHEMA.items(): + if conn.execute("SELECT type FROM sqlite_master WHERE name=?", (table,)).fetchall() != [ + ("table",) + ]: + raise ValueError(f"Ordered-frame table is missing: {table}") + actual = [(r[1], r[2], r[5]) for r in conn.execute(f'PRAGMA table_info("{table}")')] + if actual != schema: + raise ValueError(f"Invalid ordered-frame schema: {table}") + for column, kind, _ in schema: + if ( + kind == "INTEGER" + and conn.execute( + f'SELECT 1 FROM "{table}" WHERE typeof("{column}") != \'integer\' OR "{column}" < 0 LIMIT 1' + ).fetchone() + ): + raise ValueError(f"Invalid ordered-frame integer: {table}.{column}") + + checks = ( + "SELECT 1 FROM pt_snap_stack_frame f LEFT JOIN pt_snap_stack s ON s.stackId=f.stackId " + "LEFT JOIN pt_snap_frame_value v ON v.id=f.frameId WHERE s.stackId IS NULL OR v.id IS NULL LIMIT 1", + "SELECT 1 FROM pt_snap_stack s LEFT JOIN (SELECT stackId,COUNT(*) AS n,MIN(frameIndex) AS lo," + "MAX(frameIndex) AS hi FROM pt_snap_stack_frame GROUP BY stackId) f ON f.stackId=s.stackId " + "WHERE s.frameCount!=COALESCE(f.n,0) OR (s.frameCount>0 AND (f.lo!=0 OR f.hi!=s.frameCount-1)) LIMIT 1", + f"SELECT 1 FROM pt_snap_event_stack e LEFT JOIN pt_snap_stack s ON s.stackId=e.stackId " + f'LEFT JOIN "trace_entry_{device}" t ON t.id=e.eventId ' + "WHERE s.stackId IS NULL OR t.id IS NULL LIMIT 1", + ) + for sql in checks: + if conn.execute(sql).fetchone(): + raise ValueError("Invalid ordered-frame reference, frame count or sequence") + + for table, schema in SCHEMA.items(): + columns = ",".join(f'"{name}"' for name, _, _ in schema) + keys = ",".join(f'"{name}"' for name, _, pk in schema if pk) + digest.update(table.encode("ascii") + b"\n") + for row in conn.execute(f'SELECT {columns} FROM "{table}" ORDER BY {keys}'): + if table == "pt_snap_frame_value": + if not isinstance(row[1], str): + raise ValueError("Ordered frame must contain JSON text") + try: + frame = json.loads( + row[1], + object_pairs_hook=_unique_object, + parse_constant=_invalid_constant, + parse_float=_finite_float, + ) + if not isinstance(frame, dict) or not _bounded_depth([frame]): + raise ValueError("Ordered frame must be a bounded JSON object") + except (RecursionError, OverflowError) as exc: + raise ValueError("Invalid ordered-frame JSON") from exc + digest.update( + json.dumps( + row, ensure_ascii=True, separators=(",", ":"), allow_nan=False + ).encode("ascii") + ) + digest.update(b"\n") + return digest.hexdigest() diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index 83ad1c5..4229b89 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -12,6 +12,7 @@ from pt_snap_cli.core.context_cache import ContextCache from pt_snap_cli.core.dataset_attribution import event_attribution, summarize_dataset_stacks from pt_snap_cli.core.dataset_contract import QueryScope +from pt_snap_cli.core.dataset_frame_tree import frame_tree from pt_snap_cli.core.dataset_global import global_query from pt_snap_cli.core.dataset_resolver import DatasetResolver, ResolvedDataset from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget @@ -29,6 +30,7 @@ TemplateRenderError, ) from pt_snap_cli.core.focus_service import FocusService +from pt_snap_cli.core.latest_query import LatestQuery from pt_snap_cli.core.models import ( QueryResult, ResolvedFocus, @@ -39,7 +41,7 @@ from pt_snap_cli.core.query_limits import QueryLimits, resolve_query_limits from pt_snap_cli.core.stack_summary import summarize_stacks from pt_snap_cli.query.executor import QueryExecutionError as ExecutorQueryExecutionError -from pt_snap_cli.query.executor import QueryExecutor +from pt_snap_cli.query.executor import QueryExecutor, cancellation_options from pt_snap_cli.query.executor import QueryTimeoutError as ExecutorQueryTimeoutError from pt_snap_cli.query.executor import TemplateRenderError as ExecutorTemplateRenderError from pt_snap_cli.query.registry import ( @@ -106,6 +108,8 @@ def __init__( self._owns_context_cache: bool = context_cache is None self._executor: QueryExecutor | None = None self._executor_context: Context | None = None + self._executor_source: QueryExecutor | None = None + self._frame_tree_queries = LatestQuery() @property def context_cache(self) -> ContextCache: @@ -117,13 +121,16 @@ def close(self) -> None: An injected cache is borrowed and must be closed by its owner. """ + self._frame_tree_queries.cancel() self._executor = None self._executor_context = None + self._executor_source = None if self._owns_context_cache: self._context_cache.close() def invalidate_context_cache(self, db_path: Path | str | None = None) -> None: """Drop a cached context (or all of them when ``db_path`` is None).""" + self._frame_tree_queries.cancel() self._context_cache.invalidate(db_path) def list_templates( @@ -243,6 +250,75 @@ def execute_query( query_limits: QueryLimits | None = None, _budget: QueryBudget | None = None, _resolved: _ResolvedQuery | None = None, + ) -> QueryResult: + """Frame-tree calls on this service supersede unfinished frame-tree calls. + + Separate service instances do not cancel one another. Other templates + retain their existing ownership/concurrency contract. + """ + if template != "active_memory_frame_tree_at_event": + return self._execute_query( + template, + params, + db_path, + device_id, + start_dir, + max_rows, + exact_total=exact_total, + timeout_s=timeout_s, + slice_index=slice_index, + query_limits=query_limits, + _budget=_budget, + _resolved=_resolved, + ) + budget = _budget if _budget is not None else self._new_budget(timeout_s, query_limits) + with self._frame_tree_queries.run(budget.remaining) as cancelled: + previous = budget.cancelled + budget.cancelled = lambda: cancelled() or (previous is not None and previous()) + # Each task opens and closes its handles in its executing thread. + # The borrowed/standalone cache must never move between click workers. + worker = QueryService(self._focus_service, query_limits=self._query_limits) + # Retain override detection and custom templates, never the source connection. + worker._executor = self._executor + worker._executor_source = self._executor + try: + result = worker._execute_query( + template, + params, + db_path, + device_id, + start_dir, + max_rows, + exact_total=exact_total, + timeout_s=timeout_s, + slice_index=slice_index, + query_limits=query_limits, + _budget=budget, + _resolved=_resolved, + ) + budget.remaining() + return result + finally: + try: + worker.close() + finally: + budget.cancelled = previous + + def _execute_query( + self, + template: str, + params: dict[str, Any] | None = None, + db_path: Path | str | None = None, + device_id: int | None = None, + start_dir: Path | None = None, + max_rows: int | None = None, + *, + exact_total: bool = False, + timeout_s: float | None = None, + slice_index: int | None = None, + query_limits: QueryLimits | None = None, + _budget: QueryBudget | None = None, + _resolved: _ResolvedQuery | None = None, ) -> QueryResult: budget = _budget if _budget is not None else self._new_budget(timeout_s, query_limits) timeout_s, started = budget.timeout_s, budget.started @@ -285,9 +361,13 @@ def execute_query( if slice_index is not None: dataset.paths(QueryScope("slice", selected.device_id, slice_index)) implementation = ( - event_attribution - if template in ("active_blocks_at_event", "active_memory_callstack_at_event") - else global_query + frame_tree + if template == "active_memory_frame_tree_at_event" + else ( + event_attribution + if template in ("active_blocks_at_event", "active_memory_callstack_at_event") + else global_query + ) ) result = implementation( sources, @@ -325,6 +405,7 @@ def execute_query( device_id=target_device, max_rows=max_rows, timeout_s=budget.remaining(), + **cancellation_options(budget.cancelled), ) offset = _validated_offset(template, query_params) rank_cap = _finite_top_n(template, query_params) @@ -338,6 +419,7 @@ def execute_query( device_id=target_device, timeout_s=_remaining_timeout(timeout_s, started), ignore_row_window=True, + **cancellation_options(budget.cancelled), ) total_is_exact = True if offset == 0: @@ -349,8 +431,10 @@ def execute_query( except ExecutorTemplateRenderError as exc: raise TemplateRenderError(str(exc)) from exc except ExecutorQueryTimeoutError as exc: + budget.remaining() raise QueryTimeoutError(f"Query timed out after {timeout_s} seconds") from exc except ExecutorQueryExecutionError as exc: + budget.remaining() if get_query(template) is None: raise TemplateNotFoundError(f"Template '{template}' not found") from exc raise QueryExecutionError(str(exc)) from exc @@ -416,7 +500,12 @@ def _get_executor(self, ctx: Context) -> QueryExecutor: template directory of its own. """ if self._executor is None or self._executor_context is not ctx: - self._executor = QueryExecutor(ctx) + self._executor = ( + self._executor_source.clone_for_context(ctx) + if self._executor_source is not None + else QueryExecutor(ctx) + ) + self._executor_source = None self._executor_context = ctx return self._executor diff --git a/src/pt_snap_cli/core/report_service.py b/src/pt_snap_cli/core/report_service.py index d099b3d..6d5057e 100644 --- a/src/pt_snap_cli/core/report_service.py +++ b/src/pt_snap_cli/core/report_service.py @@ -56,7 +56,7 @@ def memory_tree_report( timeout_s: float | None = None, query_limits: QueryLimits | None = None, ) -> QueryResult: - """Return frame occupancy through the shared standalone query contract.""" + """Return standalone or dataset frame occupancy through the shared query.""" return self._query_service.execute_query( "active_memory_frame_tree_at_event", params={"event_id": event_id, "include_static": include_static, "min_size": min_size}, diff --git a/src/pt_snap_cli/core/snapshot_cache.py b/src/pt_snap_cli/core/snapshot_cache.py index 5f23341..49daccc 100644 --- a/src/pt_snap_cli/core/snapshot_cache.py +++ b/src/pt_snap_cli/core/snapshot_cache.py @@ -7,8 +7,6 @@ import os import re import sqlite3 -import sys -from collections import OrderedDict from contextlib import closing from pathlib import Path from typing import Any @@ -16,12 +14,11 @@ from .validation_budget import ( ValidationBudget, check_budget, - checked_rows, sqlite_timeout, validation_progress, ) -PARSER_HASH_SALT = b"mem_snapshot_parser_v2" +PARSER_HASH_SALT = b"mem_snapshot_parser_v3" CACHE_CONTRACT_VERSION = 1 TRACE_COLUMNS = ( "id", @@ -37,37 +34,6 @@ BLOCK_COLUMNS = ("id", "address", "size", "requestedSize", "state", "allocEventId", "freeEventId") -class _RowEncoder: - """Bound repeated callstack encoding; preserve the v1 digest bytes.""" - - def __init__(self) -> None: - self.stacks: OrderedDict[str, bytes] = OrderedDict() - self.size = 0 - - def encode(self, row: tuple[Any, ...], inline_stack: bool) -> bytes: - if not inline_stack or not isinstance(row[-1], str): - return json.dumps( - row, ensure_ascii=True, separators=(",", ":"), allow_nan=False - ).encode() - stack = row[-1] - encoded = self.stacks.get(stack) - if encoded is None: - encoded = json.dumps(stack, ensure_ascii=True, allow_nan=False).encode() - cost = sys.getsizeof(stack) + sys.getsizeof(encoded) - if cost <= 8 * 1024 * 1024: - while self.stacks and ( - self.size + cost > 8 * 1024 * 1024 or len(self.stacks) >= 2048 - ): - key, value = self.stacks.popitem(last=False) - self.size -= sys.getsizeof(key) + sys.getsizeof(value) - self.stacks[stack] = encoded - self.size += cost - else: - self.stacks.move_to_end(stack) - prefix = json.dumps(row[:-1], separators=(",", ":"), allow_nan=False).encode() - return prefix[:-1] + b"," + encoded + b"]" - - def source_hashes(source: Path) -> tuple[str, str]: """Return raw and salted SHA-256 in one streaming pass, without unpickling.""" raw = hashlib.sha256() @@ -112,13 +78,11 @@ def _member(directory: Path, relative: str) -> Path: return candidate -def _table_rows( +def _validate_table_schema( conn: sqlite3.Connection, name: str, columns: tuple[str, ...], - where: str = "", - parameters: tuple[str, ...] = (), -) -> sqlite3.Cursor: +) -> None: if conn.execute("SELECT type FROM sqlite_master WHERE name=?", (name,)).fetchone() != ( "table", ): @@ -131,75 +95,48 @@ def _table_rows( raise ValueError(f"Invalid parser column: {name}.{column}") if "id" in columns and [row[1] for row in schema if row[5]] != ["id"]: raise ValueError(f"Missing parser primary key: {name}.id") - selected = ",".join(f'"{column}"' for column in columns) - order = '"id"' if "id" in columns else selected - # Identifiers and the optional predicate are internal constants/validated device IDs. - return conn.execute( - f'SELECT {selected} FROM "{name}" {where} ORDER BY {order}', parameters - ) # nosec B608 -def member_digest( +def validate_member_structure( path: Path, device: str, start: int, end: int, *, budget: ValidationBudget | None = None -) -> str: - """Hash fixed base columns; consumer columns, tables, indexes, views and triggers are allowed.""" - if re.fullmatch(r"0|[1-9][0-9]*", str(device)) is None: - raise ValueError("Invalid device ID") - digest = hashlib.sha256() - encoder = _RowEncoder() - # Consumers may start writing after the sidecar check. Always use SQLite's - # locking/WAL protocol and one read snapshot, even for a currently closed DB. +) -> None: + """Check required schema and indexed endpoints, without scanning rows/pages.""" sidecars = [Path(str(path) + suffix) for suffix in ("-wal", "-shm", "-journal")] if any(item.is_symlink() for item in sidecars): raise ValueError("Snapshot has an aliased sidecar") if sidecars[2].exists(): - raise ValueError( - "Snapshot has a rollback journal; a writer or recovery may be active, retry later" - ) - uri = path.resolve().as_uri() + "?mode=ro" + raise ValueError("Snapshot has a rollback journal; retry later") with ( - closing(sqlite3.connect(uri, uri=True, timeout=sqlite_timeout(budget))) as conn, + closing( + sqlite3.connect( + path.resolve().as_uri() + "?mode=ro", uri=True, timeout=sqlite_timeout(budget) + ) + ) as conn, validation_progress(conn, budget), ): conn.execute("PRAGMA query_only=ON") conn.execute("BEGIN") - if conn.execute("PRAGMA quick_check").fetchall() != [("ok",)]: - raise ValueError("Snapshot member failed integrity check") - tables = ((f"trace_entry_{device}", TRACE_COLUMNS), (f"block_{device}", BLOCK_COLUMNS)) - for table, columns in tables: - digest.update(json.dumps((table, columns), separators=(",", ":")).encode()) - for row in checked_rows(_table_rows(conn, table, columns), budget): - digest.update(encoder.encode(row, columns == TRACE_COLUMNS)) - digest.update(b"\n") - count, low, high = conn.execute( - f'SELECT COUNT(*), MIN(id), MAX(id) FROM "trace_entry_{device}" WHERE id>=0' # nosec B608 - device validated above - ).fetchone() - if (count, low, high) != (end - start + 1, start, end): - raise ValueError("Snapshot event range is incomplete") - for row in _table_rows( - conn, - "dictionary", - ("table", "column", "key", "value"), - 'WHERE "table" IN (?, ?)', - tuple(table for table, _ in tables), + for name, columns in ( + (f"trace_entry_{device}", TRACE_COLUMNS), + (f"block_{device}", BLOCK_COLUMNS), + ("dictionary", ("table", "column", "key", "value")), ): - digest.update(json.dumps(row, ensure_ascii=True, separators=(",", ":")).encode()) - digest.update(b"\n") - for table, column in ((f"trace_entry_{device}", "action"), (f"block_{device}", "state")): - # Identifiers are internal constants with the device validated above. - missing = conn.execute( - f'SELECT 1 FROM (SELECT DISTINCT "{column}" AS value FROM "{table}") AS base WHERE NOT EXISTS ' # nosec B608 - '(SELECT 1 FROM dictionary AS dict WHERE dict."table"=? AND dict."column"=? ' - 'AND dict."key"=CAST(base.value AS TEXT)) LIMIT 1', - (table, column), - ).fetchone() - if missing: - raise ValueError("Snapshot dictionary reference is missing") - return digest.hexdigest() + _validate_table_schema(conn, name, columns) + low = conn.execute( + f'SELECT id FROM "trace_entry_{device}" WHERE id>=0 ORDER BY id LIMIT 1' + ).fetchone() # nosec B608 + high = conn.execute( + f'SELECT id FROM "trace_entry_{device}" WHERE id>=0 ORDER BY id DESC LIMIT 1' + ).fetchone() # nosec B608 + if low != (start,) or high != (end,): + raise ValueError("Snapshot event range is incomplete") def parser_members( - directory: Path, manifest: dict[str, Any], *, budget: ValidationBudget | None = None + directory: Path, + manifest: dict[str, Any], + *, + budget: ValidationBudget | None = None, ) -> dict[str, str]: """Validate the complete manifest and all DBs; never deserialize the source.""" if _integer(manifest.get("schemaVersion"), 1) != 1: @@ -240,13 +177,14 @@ def parser_members( relative = item["file"] if relative in members: raise ValueError("Duplicate snapshot member") - members[relative] = member_digest( - _member(directory, relative), device, start, end, budget=budget - ) + path = _member(directory, relative) + validate_member_structure(path, device, start, end, budget=budget) + members[relative] = "" return members def seal_manifest(directory: Path, manifest: dict[str, Any], device: int | None = None) -> None: + """Record structural inventory and device selection, without content digests.""" manifest["parserCache"] = { "version": CACHE_CONTRACT_VERSION, "requestedDevice": device, @@ -309,33 +247,25 @@ def validate_cache( raise ValueError("Snapshot device selection changed") members = parser_members(directory, manifest, budget=budget) attestation = manifest.get("parserCache") - # Legacy producers have no logical attestation; still validate every base table and range. + # Historical digest declarations are accepted as member inventories only. if attestation is not None and ( _integer(attestation.get("version"), 1) != CACHE_CONTRACT_VERSION - or attestation.get("members") != members + or not isinstance(attestation.get("members"), dict) + or set(attestation["members"]) != set(members) ): raise ValueError("Snapshot parser data changed") - # Both attestations must hold; parserCache must not hide an older producer seal. + # Historical producer declarations must list the same manifest members. producer = manifest.get("ptSnap", {}) if "memberHashKind" in producer and producer["memberHashKind"] != "parser-v1": raise ValueError("Unsupported member hash contract") - if producer.get("memberHashKind") == "parser-v1" and producer.get("members") != members: - raise ValueError("Snapshot producer member hash changed") + if producer.get("memberHashKind") == "parser-v1" and ( + not isinstance(producer.get("members"), dict) or set(producer["members"]) != set(members) + ): + raise ValueError("Snapshot producer member inventory changed") if "members" in producer and producer.get("memberHashKind") != "parser-v1": physical = producer["members"] if not isinstance(physical, dict) or set(physical) != set(members): raise ValueError("Invalid legacy snapshot member inventory") - from .dataset_files import hash_file - - for relative, expected in physical.items(): - member = _member(directory, relative) - wal = Path(str(member) + "-wal") - # A read-only open can create an empty WAL and SHM. They contain no - # committed rows; only the DB bytes need the legacy physical digest. - if Path(str(member) + "-journal").exists() or (wal.exists() and wal.stat().st_size): - raise ValueError("Legacy physical snapshot hash cannot validate live sidecars") - if hash_file(member, budget=budget) != expected: - raise ValueError("Legacy snapshot member hash changed") if path.read_bytes() != original: raise ValueError("Snapshot manifest changed during validation") return manifest diff --git a/src/pt_snap_cli/core/validation_budget.py b/src/pt_snap_cli/core/validation_budget.py index ec60cf0..2bb0dc3 100644 --- a/src/pt_snap_cli/core/validation_budget.py +++ b/src/pt_snap_cli/core/validation_budget.py @@ -12,7 +12,7 @@ from contextlib import contextmanager from typing import Protocol, TypeVar -from .errors import QueryTimeoutError +from .errors import QueryCancelledError, QueryTimeoutError class ValidationBudget(Protocol): @@ -50,16 +50,16 @@ def checked_rows(rows: Iterable[_T], budget: ValidationBudget | None) -> Iterato def validation_progress( conn: sqlite3.Connection, budget: ValidationBudget | None ) -> Iterator[None]: - """Interrupt SQLite and preserve QUERY_TIMEOUT rather than a schema error.""" + """Preserve timeout/cancellation instead of reporting an invalid schema.""" check_budget(budget) - timed_out: QueryTimeoutError | None = None + interrupted: QueryTimeoutError | QueryCancelledError | None = None def progress() -> int: - nonlocal timed_out + nonlocal interrupted try: check_budget(budget) - except QueryTimeoutError as exc: - timed_out = exc + except (QueryTimeoutError, QueryCancelledError) as exc: + interrupted = exc return 1 return 0 @@ -69,8 +69,8 @@ def progress() -> int: try: yield except sqlite3.DatabaseError: - if timed_out is not None: - raise timed_out from None + if interrupted is not None: + raise interrupted from None check_budget(budget) raise check_budget(budget) diff --git a/src/pt_snap_cli/core/validation_receipts.py b/src/pt_snap_cli/core/validation_receipts.py new file mode 100644 index 0000000..9c48be9 --- /dev/null +++ b/src/pt_snap_cli/core/validation_receipts.py @@ -0,0 +1,103 @@ +"""Disposable, content-addressed validation receipts for repeated imports. + +Receipts live outside datasets. They never substitute file timestamps for byte +hashes, and are only consulted inside an import operation. Bump VERSION whenever +the validation semantics change, independently of the package version. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from pathlib import Path + +VERSION = 1 +_ENABLED: ContextVar[bool] = ContextVar("import_validation_receipts", default=False) + + +@contextmanager +def import_receipts() -> Iterator[None]: + token = _ENABLED.set(True) + try: + yield + finally: + _ENABLED.reset(token) + + +def _path(root: Path, scope: str) -> Path | None: + if not receipts_enabled(): + return None + key = hashlib.sha256(str(root).encode("utf-8")).hexdigest() + try: + return ( + Path.home() + / ".cache" + / "pt-snap-cli" + / f"validation-v{VERSION}" + / f"{key}-{scope}.json" + ) + except (OSError, RuntimeError): + return None + + +def import_validation_active() -> bool: + """Import admission policy is independent of receipt/cache configuration.""" + return _ENABLED.get() + + +def receipts_enabled() -> bool: + return _ENABLED.get() and os.environ.get("PT_SNAP_DATASET_CACHE") is None + + +def _encoded(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") + + +def load_receipt(root: Path, scope: str, fingerprint: str) -> dict[str, object] | None: + path = _path(root, scope) + if path is None: + return None + try: + if path.is_symlink() or path.stat().st_size > 4096: + return None + envelope = json.loads(path.read_bytes()) + data = envelope["data"] + if ( + envelope["sha256"] != hashlib.sha256(_encoded(data)).hexdigest() + or data["fingerprint"] != fingerprint + or not isinstance(data["payload"], dict) + ): + return None + return data["payload"] + except (OSError, ValueError, KeyError, TypeError, RecursionError): + return None + + +def save_receipt(root: Path, scope: str, fingerprint: str, payload: dict[str, object]) -> None: + path = _path(root, scope) + if path is None: + return + temporary: Path | None = None + try: + path.parent.mkdir(parents=True, exist_ok=True) + data = {"fingerprint": fingerprint, "payload": payload} + raw = _encoded({"data": data, "sha256": hashlib.sha256(_encoded(data)).hexdigest()}) + # One bounded record per dataset/scope, atomically replaced on change. + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as file: + temporary = Path(file.name) + file.write(raw) + os.replace(temporary, path) + except OSError: + # Read-only homes, full disks and stale receipts affect speed, not validity. + pass + finally: + if temporary is not None: + try: + temporary.unlink(missing_ok=True) + except OSError: + pass diff --git a/src/pt_snap_cli/query/executor.py b/src/pt_snap_cli/query/executor.py index b9ff0d7..ffc43fd 100644 --- a/src/pt_snap_cli/query/executor.py +++ b/src/pt_snap_cli/query/executor.py @@ -2,13 +2,14 @@ from __future__ import annotations +import copy import re import sqlite3 import time from collections.abc import Callable, Generator, Mapping from contextlib import contextmanager from pathlib import Path -from typing import Any +from typing import Any, TypedDict from jinja2 import Environment, StrictUndefined, Template, TemplateSyntaxError @@ -90,6 +91,14 @@ def reported_sql_limit( return injected_row_limit(validated, max_rows) +class _CancellationOptions(TypedDict, total=False): + cancelled: Callable[[], bool] + + +def cancellation_options(cancelled: Callable[[], bool] | None) -> _CancellationOptions: + return {"cancelled": cancelled} if cancelled is not None else {} + + class QueryExecutionError(Exception): """Raised when query execution fails.""" @@ -115,7 +124,9 @@ def _is_sqlite_interrupt(error: BaseException) -> bool: @contextmanager def _sqlite_timeout( - conn: sqlite3.Connection, timeout_s: float | None + conn: sqlite3.Connection, + timeout_s: float | None, + cancelled: Callable[[], bool] | None = None, ) -> Generator[None, None, None]: """Abort the current SQLite statement after ``timeout_s`` wall-clock seconds. @@ -124,13 +135,18 @@ def _sqlite_timeout( the query should run without a time bound. The handler is always cleared so persistent connections do not leak a deadline into the next statement. """ - if timeout_s is None or timeout_s <= 0: + if (timeout_s is None or timeout_s <= 0) and cancelled is None: yield return - deadline = time.monotonic() + float(timeout_s) + deadline = ( + time.monotonic() + float(timeout_s) if timeout_s is not None and timeout_s > 0 else None + ) def _handler() -> int: - return 1 if time.monotonic() >= deadline else 0 + return int( + (cancelled is not None and cancelled()) + or (deadline is not None and time.monotonic() >= deadline) + ) conn.set_progress_handler(_handler, _PROGRESS_HANDLER_INTERVAL) try: @@ -329,6 +345,12 @@ def render( return rendered_sql + def clone_for_context(self, context: Context) -> QueryExecutor: + """Copy runtime registrations to a worker without borrowing its connection.""" + worker = QueryExecutor(context) + worker._configs = copy.deepcopy(self._configs) + return worker + def execute( self, sql: str, @@ -336,6 +358,7 @@ def execute( timeout_s: float | None = None, *, consume: Callable[[list[dict[str, Any]]], None] | None = None, + cancelled: Callable[[], bool] | None = None, ) -> list[dict[str, Any]]: """Execute raw SQL query. @@ -353,7 +376,7 @@ def execute( """ try: with self._context.connect() as conn: - with _sqlite_timeout(conn, timeout_s): + with _sqlite_timeout(conn, timeout_s, cancelled): cursor = conn.cursor() try: if params: @@ -386,13 +409,19 @@ def execute( except Exception as e: raise QueryExecutionError(f"Query execution failed: {e}") from e - def count(self, sql: str, timeout_s: float | None = None) -> int: + def count( + self, + sql: str, + timeout_s: float | None = None, + *, + cancelled: Callable[[], bool] | None = None, + ) -> int: """Count the rows produced by a rendered query without materializing them.""" query = sql.strip().rstrip(";").rstrip() count_sql = f"SELECT COUNT(*) FROM ({query}) AS pt_snap_count" try: with self._context.connect() as conn: - with _sqlite_timeout(conn, timeout_s): + with _sqlite_timeout(conn, timeout_s, cancelled): cursor = conn.cursor() cursor.execute(count_sql) row = cursor.fetchone() @@ -456,6 +485,7 @@ def execute_template_page( timeout_s: float | None = None, *, probe_has_more: bool = True, + cancelled: Callable[[], bool] | None = None, ) -> tuple[list[dict[str, Any]], bool, int | None]: """Execute a named template and report whether more rows exist. @@ -480,7 +510,11 @@ def execute_template_page( fetch_sql = ( bump_trailing_limit(sql, 1) if probe_has_more and applied_limit is not None else sql ) - rows = self.execute(fetch_sql, timeout_s=timeout_s) + rows = self.execute( + fetch_sql, + timeout_s=timeout_s, + **cancellation_options(cancelled), + ) has_more = applied_limit is not None and len(rows) > applied_limit if has_more and applied_limit is not None: rows = rows[:applied_limit] @@ -495,6 +529,7 @@ def count_template( timeout_s: float | None = None, *, ignore_row_window: bool = False, + cancelled: Callable[[], bool] | None = None, ) -> int: """Count rows returned by a named template without materializing them. @@ -518,7 +553,7 @@ def count_template( if "top_n" in template.parameters: count_params["top_n"] = -1 sql = self.render(template, count_params, device_id) - return self.count(sql, timeout_s=timeout_s) + return self.count(sql, timeout_s=timeout_s, **cancellation_options(cancelled)) def execute_on_all_devices( self, diff --git a/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml b/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml index 1c8d7f1..2cb7691 100644 --- a/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml +++ b/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml @@ -2,9 +2,10 @@ version: "1.0" queries: active_memory_frame_tree_at_event: description: Decompose live memory after an event into a caller-first frame tree - semantics_version: 1 + semantics_version: 2 interpretation_limits: - - "Requires structured frame tables from import_format_version >= 3; old text-only databases must be re-imported." + - "Standalone databases require structured frame tables from import_format_version >= 3. Datasets resolve original allocation events across shards using ptSnapOrderedFrames version 1 or 2; absent, unknown or corrupt evidence contributes to missing stacks without reconstructing frames from text. Re-import to capture ordered frames." + - "Dataset frame_id is query-local; node_id identifies the full ordered path of filename/line/name triples. Shard-local IDs and display text are never cross-shard grouping keys. scope.source_coverage reports frame completeness separately from result row completeness." - "Uses allocation stacks of blocks live after event_id: allocEventId <= event_id and freeEventId > event_id (or absent). freeEventId is free_completed, so pending-free memory remains included." - "Traversal reverses the original innermost-first frame array. Node identity is the whole ordered caller path, preserving recursion and separating identical frames under different callers." - "size_bytes includes descendant occupancy; self_bytes counts only allocations ending at this node. size_bytes = self_bytes + sum(direct child size_bytes). Do not sum inclusive bytes across depths." diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index 40df201..7b2ebff 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -299,3 +299,25 @@ Read-only queries preserve path identity, classify absent/corrupt evidence, and conserve live block, requested-byte and occupancy totals. No upstream code or new executable fixtures were incorporated; source mappings and licenses remain unchanged. This supersedes the unmerged implementation proposed in PR #207. + +## 2026-10-21: deduplicated dataset frames and cross-shard occupancy trees + +First-party ordered-frame evidence is now retained by sharded replay in an +isolated version-2 extension: raw frame JSON and ordered frame sequences are +interned independently of native text callstack IDs, with references only for +real events. Compatible conversion preserves these extension tables without +changing base inline trace columns or boundary-event normalization. Native +extension and compatible export cache identities advance; old native text-only +artifacts remain readable. Dataset trees resolve allocation owners before +caller-path grouping and preserve missing/empty/static/preexisting occupancy. +No allocator transition, event ID, source mapping, license or executable fixture +was changed, and no external implementation was incorporated. + +## 2026-11-03: preflight optional ordered-frame JSON before publication + +First-party frame writers and readers now share the existing bounded JSON rules. +Serialized stacks are checked before any optional evidence is inserted, including +JSON tuple-to-array normalization; unsupported deep evidence is omitted while +the original text stack and snapshot import remain accepted. No allocator, +event identity, source mapping, license or executable fixture changed. No +external implementation was incorporated. diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/ordered_frames.py b/src/pt_snap_cli/snapshot/tools/adaptors/ordered_frames.py new file mode 100644 index 0000000..b265c33 --- /dev/null +++ b/src/pt_snap_cli/snapshot/tools/adaptors/ordered_frames.py @@ -0,0 +1,75 @@ +"""Deduplicated, optional frame evidence independent of native text stack IDs.""" + +import json +import sqlite3 + +from ...base import Frame, TraceEntry +from ...util.frame_json import _bounded_depth + + +class OrderedFrameWriter: + def __init__(self, connection: sqlite3.Connection): + self.connection = connection + connection.executescript( + "CREATE TABLE pt_snap_frame_value (id INTEGER PRIMARY KEY, frameJson TEXT);" + "CREATE TABLE pt_snap_stack (stackId INTEGER PRIMARY KEY, frameCount INTEGER);" + "CREATE TABLE pt_snap_stack_frame (stackId INTEGER, frameIndex INTEGER, frameId INTEGER," + " PRIMARY KEY(stackId,frameIndex));" + "CREATE TABLE pt_snap_event_stack (eventId INTEGER PRIMARY KEY, stackId INTEGER);" + ) + self.frames: dict[str, int] = {} + self.stacks: dict[tuple[int, ...], int] = {} + self.containers: dict[tuple[int, ...], tuple[object, int]] = {} + + def insert(self, event: TraceEntry) -> None: + if event.idx is None or event.idx < 0: + return + frames = event.callstack_frames() + key = tuple(map(id, frames)) + cached = self.containers.get(key) + if cached is not None: + stack_id = cached[1] + else: + try: + encoded = [ + json.dumps( + frame.to_dict() if isinstance(frame, Frame) else frame, + sort_keys=True, + ensure_ascii=True, + separators=(",", ":"), + allow_nan=False, + ) + for frame in frames + ] + # Check the JSON shape readers see, including tuple-to-list + # normalization, before writing any part of this optional stack. + if not _bounded_depth([json.loads(value) for value in encoded]): + return + except (ValueError, TypeError, RecursionError, OverflowError): + # Optional evidence must not change acceptance of native text inputs. + return + ids = [] + for value in encoded: + if value not in self.frames: + frame_id = len(self.frames) + self.frames[value] = frame_id + self.connection.execute( + "INSERT INTO pt_snap_frame_value VALUES (?,?)", (frame_id, value) + ) + ids.append(self.frames[value]) + stack = tuple(ids) + if stack not in self.stacks: + stack_id = len(self.stacks) + self.stacks[stack] = stack_id + self.connection.execute( + "INSERT INTO pt_snap_stack VALUES (?,?)", (stack_id, len(stack)) + ) + self.connection.executemany( + "INSERT INTO pt_snap_stack_frame VALUES (?,?,?)", + [(stack_id, index, frame_id) for index, frame_id in enumerate(stack)], + ) + stack_id = self.stacks[stack] + self.containers[key] = (frames, stack_id) + self.connection.execute( + "INSERT INTO pt_snap_event_stack VALUES (?,?)", (event.idx, stack_id) + ) diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py b/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py index 1d2d130..45c5d58 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py @@ -15,6 +15,7 @@ from ...base import Block, BlockState, DeviceSnapshot, TraceEntry from ...simulate import AllocatorHooker, SimulateDeviceSnapshot, SimulateHooker from .database import CallstackInterner, event2record +from .ordered_frames import OrderedFrameWriter from .snapshot2db import SnapshotDbHandler @@ -153,11 +154,13 @@ def __init__(self, item: ReplaySlice, registry: BlockRegistry): self.registry = registry self.writer = SnapshotDbHandler(str(item.file), [item.device]) self.stacks = CallstackInterner() + self.frames = OrderedFrameWriter(self.writer.db.conn) self.current_event: TraceEntry | None = None def pre_undo_event(self, wait4undo_event: TraceEntry, current_snapshot: DeviceSnapshot) -> bool: event, snapshot = wait4undo_event, current_snapshot self.current_event = event + self.frames.insert(event) self.writer.insert_event( event2record( event, diff --git a/src/pt_snap_cli/snapshot/util/frame_json.py b/src/pt_snap_cli/snapshot/util/frame_json.py new file mode 100644 index 0000000..0989839 --- /dev/null +++ b/src/pt_snap_cli/snapshot/util/frame_json.py @@ -0,0 +1,27 @@ +"""JSON limits shared by optional ordered-frame writers and readers.""" + +import math + + +def _invalid_constant(value: str) -> object: + raise ValueError(f"Non-JSON numeric constant: {value}") + + +def _finite_float(value: str) -> float: + result = float(value) + if not math.isfinite(result): + raise ValueError("Non-finite frame number") + return result + + +def _bounded_depth(value: object) -> bool: + pending = [(value, 0)] + while pending: + current, depth = pending.pop() + if depth > 128: + return False + if isinstance(current, dict): + pending.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + pending.extend((item, depth + 1) for item in current) + return True diff --git a/tests/core/AGENTS.md b/tests/core/AGENTS.md index 4855dce..28d4867 100644 --- a/tests/core/AGENTS.md +++ b/tests/core/AGENTS.md @@ -57,11 +57,10 @@ The same acceptance is stored in the JUnit `dataset_acceptance` property (use The real importer exports exactly 1,000,000 events using the unchanged 500,000 default capacity. The complete installed catalog must be classified: currently -11 dataset-supported templates succeed with CLI/API result parity and independent -known-result assertions; `event_frames` and `active_memory_frame_tree_at_event` -are explicit dataset rejections. Both report commands run through the console: -`peak-memory` succeeds, while `memory-tree` preserves its documented unsupported -dataset contract. Source and every exported member remain unchanged. This is +12 dataset-supported templates succeed with CLI/API result parity and independent +known-result assertions; `event_frames` is the explicit dataset rejection. +Frame trees have additional real-import differential tests in `test_dataset_frame_tree.py`. Both report commands run through the console: +`peak-memory` and `memory-tree` succeed with shared CLI/API semantics. Source and every exported member remain unchanged. This is trusted test-owned producer/replay acceptance, not live capture or upstream GUI acceptance. Do not turn a missing wheel/skip into a successful nightly gate. diff --git a/tests/core/dataset_differential_cases.py b/tests/core/dataset_differential_cases.py index 589d1c4..65760e0 100644 --- a/tests/core/dataset_differential_cases.py +++ b/tests/core/dataset_differential_cases.py @@ -25,7 +25,8 @@ "active_memory_callstack_at_event", ) ) -UNSUPPORTED_TEMPLATES = frozenset(("event_frames", "active_memory_frame_tree_at_event")) +FRAME_TEMPLATES = frozenset(("active_memory_frame_tree_at_event",)) +UNSUPPORTED_TEMPLATES = frozenset(("event_frames",)) def configured_seeds(environ=None): diff --git a/tests/core/test_critical_coverage_cache.py b/tests/core/test_critical_coverage_cache.py index 3411a30..b2c5962 100644 --- a/tests/core/test_critical_coverage_cache.py +++ b/tests/core/test_critical_coverage_cache.py @@ -12,7 +12,7 @@ from pt_snap_cli.core import snapshot_cache from pt_snap_cli.core.snapshot_cache import ( - member_digest, + parser_members, seal_manifest, source_hashes, validate_cache, @@ -233,11 +233,19 @@ def test_matching_producer_and_parser_identities_validate_read_only(cache_artifa @pytest.mark.parametrize("device", ["01", "0; DROP TABLE block_0"]) -def test_member_digest_rejects_invalid_device_before_database_access(cache_artifact, device): - directory = cache_artifact[0] +def test_member_inventory_rejects_invalid_device_before_database_access( + cache_artifact, device, monkeypatch +): + directory, manifest, _ = cache_artifact + manifest["devices"][device] = manifest["devices"].pop("0") + + def unexpected_database_access(*args, **kwargs): + pytest.fail("Invalid device IDs must be rejected before opening a database") + + monkeypatch.setattr(snapshot_cache, "validate_member_structure", unexpected_database_access) before = _tree_state(directory) with pytest.raises(ValueError, match="Invalid device ID"): - member_digest(directory / "slice.db", device, 0, 0) + parser_members(directory, manifest) assert _tree_state(directory) == before diff --git a/tests/core/test_dataset_attribution.py b/tests/core/test_dataset_attribution.py index cd4da5f..34eed1f 100644 --- a/tests/core/test_dataset_attribution.py +++ b/tests/core/test_dataset_attribution.py @@ -544,6 +544,8 @@ def test_native_bare_local_stack_ids_do_not_alias_foreign_sources(tmp_path): .db_path ) raw = json.loads((root / "manifest.json").read_text()) + # Exercise legacy text-only identity independently of captured frame evidence. + raw.pop("extensions", None) for event in (17, 67): item = next( s diff --git a/tests/core/test_dataset_differential_cases.py b/tests/core/test_dataset_differential_cases.py index ae19278..f71c8de 100644 --- a/tests/core/test_dataset_differential_cases.py +++ b/tests/core/test_dataset_differential_cases.py @@ -12,6 +12,7 @@ from pt_snap_cli.query import registry from tests.core.dataset_differential_assertions import compare_case from tests.core.dataset_differential_cases import ( + FRAME_TEMPLATES, REGRESSION_SEEDS, SUPPORTED_TEMPLATES, UNSUPPORTED_TEMPLATES, @@ -61,7 +62,9 @@ def test_invalid_seed_configuration_fails_instead_of_silently_skipping(name, val def test_query_bank_preserves_original_shapes_and_all_supported_templates(monkeypatch): - assert set(DATASET_SUPPORT) == SUPPORTED_TEMPLATES + # Text-only SQL differential fixtures cannot serve the structured single-DB + # oracle; frame paths have their own real-import differential suite. + assert set(DATASET_SUPPORT) == SUPPORTED_TEMPLATES | FRAME_TEMPLATES # Other suites legitimately register custom SQL. Check packaged discovery # in a fresh registry, and restore both singleton and identity bookkeeping. with monkeypatch.context() as patch: @@ -69,7 +72,10 @@ def test_query_bank_preserves_original_shapes_and_all_supported_templates(monkey patch.setattr(registry, "_registry", registry.QueryRegistry()) patch.setattr(registry, "_packaged", {}) registry._load_all_templates() - assert set(registry.list_queries()) == SUPPORTED_TEMPLATES | UNSUPPORTED_TEMPLATES + assert ( + set(registry.list_queries()) + == SUPPORTED_TEMPLATES | FRAME_TEMPLATES | UNSUPPORTED_TEMPLATES + ) for seed in configured_seeds({"PT_SNAP_DIFFERENTIAL_SEEDS": "200"}): capacity = capacity_for_seed(seed) cases = query_cases(seed, capacity) diff --git a/tests/core/test_dataset_frame_tree.py b/tests/core/test_dataset_frame_tree.py new file mode 100644 index 0000000..b047d6c --- /dev/null +++ b/tests/core/test_dataset_frame_tree.py @@ -0,0 +1,465 @@ +"""Full frame paths, real imports and read-only cross-shard occupancy contracts.""" + +import dataclasses +import json +import sqlite3 +import time +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +from pathlib import Path +from threading import Event + +import pytest +from typer.testing import CliRunner + +import pt_snap_cli.core.query_service as query_service_module +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.cli import app +from pt_snap_cli.core.dataset_frames import attach_frames +from pt_snap_cli.core.errors import InvalidParameterError, QueryCancelledError, QueryExecutionError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.models import ImportOptions +from pt_snap_cli.core.query_limits import QueryLimits +from pt_snap_cli.core.query_service import QueryService +from pt_snap_cli.snapshot.base import TraceEntry +from pt_snap_cli.snapshot.representation import load_snapshot_representation +from pt_snap_cli.snapshot.tools.adaptors.ordered_frames import OrderedFrameWriter +from tests.core.test_dataset_attribution import case +from tests.core.test_msinsight_export import options +from tests.core.wheel_scale_acceptance import write_scale_source +from tests.snapshot.test_sharded_replay import write_source +from tests.test_dataset_focus import hashes + +TEMPLATE = "active_memory_frame_tree_at_event" +ROOT = {"filename": "train.py", "line": 1, "name": "train"} +SHARED = {"filename": "model.py", "line": 2, "name": "forward"} +LEAF = {"filename": "tensor.py", "line": 3, "name": "alloc"} +STACKS = { + 0: [LEAF, SHARED, ROOT], + 1: [LEAF, SHARED, ROOT], + 3: [LEAF, ROOT], + 4: [SHARED, SHARED, ROOT], + 5: [SHARED, ROOT], + 6: [], + 11: [LEAF, SHARED, ROOT], +} + + +@pytest.mark.parametrize("reset", [False, True]) +def test_new_frame_tree_or_close_interrupts_old_work_and_releases_connections( + tmp_path, monkeypatch, reset +): + root = framed_dataset(tmp_path) + before = hashes(root) + started = Event() + original = query_service_module.frame_tree + old_caches = [] + + def slow_tree(sources, template, params, *args): + if params["event_id"] == 8: + old_caches.append(sources.cache) + started.set() + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + sources.budget.remaining() + Event().wait(0.005) + pytest.fail("The obsolete tree was not cancelled") + return original(sources, template, params, *args) + + monkeypatch.setattr(query_service_module, "frame_tree", slow_tree) + with closing(QueryService()) as service, ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit( + service.execute_query, TEMPLATE, {"event_id": 8}, db_path=root, timeout_s=0 + ) + assert started.wait(3) + if reset: + service.close() + second = pool.submit(service.execute_query, TEMPLATE, {"event_id": 9}, db_path=root) + with pytest.raises(QueryCancelledError): + first.result(timeout=5) + result = second.result(timeout=5) + assert_conserved(result.rows) + assert all(len(cache) == 0 for cache in old_caches) + # Completion must not leave cancellation installed for the next call. + assert service.execute_query(TEMPLATE, {"event_id": 9}, db_path=root).rows == result.rows + assert hashes(root) == before + + +@pytest.mark.parametrize("reset", ["close", "invalidate_context_cache"]) +def test_analyzer_reset_cancels_its_inflight_tree(tmp_path, monkeypatch, reset): + root = framed_dataset(tmp_path) + started = Event() + original = query_service_module.frame_tree + + def slow_tree(sources, template, params, *args): + if params["event_id"] == 8: + started.set() + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + sources.budget.remaining() + Event().wait(0.005) + pytest.fail("Analyzer reset did not cancel the obsolete tree") + return original(sources, template, params, *args) + + monkeypatch.setattr(query_service_module, "frame_tree", slow_tree) + with SnapshotAnalyzer(root) as analyzer, ThreadPoolExecutor(max_workers=1) as pool: + first = pool.submit(analyzer.execute_query, TEMPLATE, {"event_id": 8}, timeout_s=0) + assert started.wait(3) + getattr(analyzer, reset)() + with pytest.raises(QueryCancelledError): + first.result(timeout=5) + if reset == "invalidate_context_cache": + assert_conserved(analyzer.execute_query(TEMPLATE, {"event_id": 9})["rows"]) + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_QUERY_TIMEOUT", raising=False) + + +def framed_dataset(tmp_path, version=2): + root = case(tmp_path) + manifest = root / "manifest.json" + raw = json.loads(manifest.read_text()) + raw["extensions"] = {"ptSnapOrderedFrames": {"version": version}} + manifest.write_text(json.dumps(raw)) + for device in (0, 2): + for index in range(8): + with ( + closing(sqlite3.connect(root / f"device_{device}/slice_{index:05d}.db")) as conn, + conn, + ): + if version == 2: + writer = OrderedFrameWriter(conn) + else: + conn.executescript( + "CREATE TABLE pt_snap_frame_coverage (eventId INTEGER PRIMARY KEY,frameCount INTEGER);" + "CREATE TABLE pt_snap_frame (eventId INTEGER,frameIndex INTEGER,frameJson TEXT, PRIMARY KEY(eventId,frameIndex));" + ) + for event in range(index * 2, index * 2 + 2): + if event not in STACKS: + continue + if version == 2: + entry = TraceEntry.from_dict( + {"id": event, "action": "alloc", "frames": STACKS[event]} + ) + writer.insert(entry) + else: + conn.execute( + "INSERT INTO pt_snap_frame_coverage VALUES (?,?)", + (event, len(STACKS[event])), + ) + conn.executemany( + "INSERT INTO pt_snap_frame VALUES (?,?,?)", + [ + (event, i, json.dumps(frame)) + for i, frame in enumerate(STACKS[event]) + ], + ) + return root + + +def assert_conserved(rows): + assert rows[0]["node_id"] == "root" + for total, own in ( + ("size_bytes", "self_bytes"), + ("requested_bytes", "self_requested_bytes"), + ("block_count", "self_block_count"), + ): + assert sum(row[own] for row in rows) == rows[0][total] + for row in rows: + assert row[total] == row[own] + sum( + child[total] for child in rows if child["parent_id"] == row["node_id"] + ) + + +@pytest.mark.parametrize("version", (1, 2)) +def test_paths_recursion_source_owners_and_devices_conserve_occupancy(tmp_path, version): + root = framed_dataset(tmp_path, version) + before = hashes(root) + with SnapshotAnalyzer(root) as analyzer: + result = analyzer.execute_query(TEMPLATE, {"event_id": 8}, exact_total=True) + rows = result["rows"] + assert_conserved(rows) + assert (rows[0]["size_bytes"], rows[0]["block_count"]) == (295, 9) + assert sorted(r["size_bytes"] for r in rows if r["name"] == "alloc") == [40, 150] + assert sorted( + (r["depth"], r["size_bytes"], r["self_bytes"]) for r in rows if r["name"] == "forward" + ) == [(2, 205, 25), (3, 30, 30)] + assert next(r for r in rows if r["node_id"] == "missing")["size_bytes"] == 15 + assert next(r for r in rows if r["node_id"] == "empty_callstack")["size_bytes"] == 20 + assert not result["scope"]["source_coverage"]["ordered_frames_complete"] + assert rows == analyzer.execute_query(TEMPLATE, {"event_id": 8}, device_id=2)["rows"] + assert analyzer.execute_query(TEMPLATE, {"event_id": 11})["rows"][0]["size_bytes"] == 285 + filtered = analyzer.execute_query( + TEMPLATE, {"event_id": 8, "include_static": False, "min_size": 30} + )["rows"] + assert_conserved(filtered) + assert filtered[0]["size_bytes"] == 220 + empty = analyzer.execute_query(TEMPLATE, {"event_id": 8, "min_size": 9999})["rows"] + assert ( + len(empty) == 1 and empty[0]["size_bytes"] == 0 and empty[0]["percent_of_total"] is None + ) + capped = analyzer.execute_query(TEMPLATE, {"event_id": 8}, max_rows=2, exact_total=True) + assert capped["rows"] == rows[:2] and capped["truncated"] and capped["total"] == len(rows) + with pytest.raises(InvalidParameterError): + analyzer.execute_query(TEMPLATE, {"event_id": 8}, slice_index=0) + assert hashes(root) == before + + +@pytest.mark.parametrize("damage", ("tail", "frame", "json", "type", "schema", "version")) +def test_bad_evidence_degrades_to_missing_without_losing_bytes(tmp_path, damage): + root = framed_dataset(tmp_path) + path = root / "device_0/slice_00000.db" + with closing(sqlite3.connect(path)) as conn, conn: + if damage == "tail": + conn.execute("DELETE FROM pt_snap_stack_frame WHERE frameIndex=2") + elif damage == "frame": + conn.execute("DELETE FROM pt_snap_frame_value") + elif damage == "json": + conn.execute("UPDATE pt_snap_frame_value SET frameJson='[]'") + elif damage == "type": + conn.execute( + "UPDATE pt_snap_frame_value SET frameJson=?", + ('{"filename":"x","line":true,"name":"n"}',), + ) + elif damage == "schema": + conn.execute("DROP TABLE pt_snap_stack_frame") + if damage == "version": + manifest = root / "manifest.json" + data = json.loads(manifest.read_text()) + data["extensions"]["ptSnapOrderedFrames"]["version"] = 999 + manifest.write_text(json.dumps(data)) + before = hashes(root) + with SnapshotAnalyzer(root) as analyzer: + rows = analyzer.execute_query(TEMPLATE, {"event_id": 8})["rows"] + assert_conserved(rows) + assert rows[0]["size_bytes"] == 295 + assert next(r for r in rows if r["node_id"] == "missing")["size_bytes"] >= 165 + assert hashes(root) == before + + +@pytest.mark.parametrize("fmt", ("pt-snap-native-v2", "msinsight")) +def test_real_import_matches_single_tree_and_cli_report(tmp_path, fmt): + request = options(tmp_path, capacity=2, focus=False) + # Same complete raw triple under two parents; preserve deep recursion too. + data = load_snapshot_representation(request.snapshot_file) # test-owned generated input + data["device_traces"][0][0]["frames"] = [LEAF, SHARED, ROOT] + data["device_traces"][0][1]["frames"] = [SHARED, SHARED, ROOT] + data["device_traces"][0][-1]["frames"] = [LEAF, ROOT] + write_source(tmp_path, data) + service = ImportService() + dataset = service.import_snapshot(dataclasses.replace(request, format=fmt, events_per_slice=2)) + single = service.import_snapshot( + ImportOptions(request.snapshot_file, tmp_path / "single", set_focus=False) + ) + before = hashes(dataset.db_path) + + def paths(rows): + by_id = {r["node_id"]: r for r in rows} + result = {} + for row in rows: + labels = [] + current = row + while current is not None: + labels.append(current["label"]) + current = by_id.get(current["parent_id"]) + result[tuple(reversed(labels))] = tuple( + row[k] + for k in ( + "size_bytes", + "self_bytes", + "requested_bytes", + "self_requested_bytes", + "block_count", + "self_block_count", + "percent_of_total", + ) + ) + return result + + with ( + SnapshotAnalyzer(dataset.db_path) as analyzer, + SnapshotAnalyzer(single.db_path) as reference, + ): + for event in range(5): + actual = analyzer.execute_query(TEMPLATE, {"event_id": event}) + assert_conserved(actual["rows"]) + assert paths(actual["rows"]) == paths( + reference.execute_query(TEMPLATE, {"event_id": event})["rows"] + ) + rows = analyzer.execute_query(TEMPLATE, {"event_id": 3})["rows"] + assert analyzer.execute_query("event", {"id": 0})["rows"][0]["frames"] == [ + LEAF, + SHARED, + ROOT, + ] + runner = CliRunner() + args = ["report", "memory-tree", str(dataset.db_path), "--event-id", "3"] + report = runner.invoke(app, [*args, "--json"]) + assert report.exit_code == 0, report.output + assert json.loads(report.output)["rows"] == rows + html = runner.invoke(app, [*args, "--format", "html"]) + assert html.exit_code == 0 and "Reset zoom" in html.output + assert hashes(dataset.db_path) == before + + +def test_dedup_storage_and_budget_failure_are_explicit(tmp_path): + root = framed_dataset(tmp_path) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn: + assert conn.execute("SELECT COUNT(*) FROM pt_snap_frame_value").fetchone()[0] == 3 + assert conn.execute("SELECT COUNT(*) FROM pt_snap_stack").fetchone()[0] == 1 + assert conn.execute("SELECT COUNT(*) FROM pt_snap_event_stack").fetchone()[0] == 2 + service = QueryService() + try: + with pytest.raises(QueryExecutionError, match="budget"): + service.execute_query( + TEMPLATE, {"event_id": 8}, db_path=root, query_limits=QueryLimits(max_work_rows=1) + ) + assert ( + service.execute_query(TEMPLATE, {"event_id": 8}, db_path=root).rows[0]["size_bytes"] + == 295 + ) + finally: + service.close() + + +def test_same_rendered_stack_is_not_a_frame_path_identity(tmp_path): + root = framed_dataset(tmp_path) + # Both arrays render as "a:1 b\nc:2 d" when traversed caller-first. + arrays = [ + [{"filename": "a", "line": 1, "name": "b\nc:2 d"}], + [{"filename": "c", "line": 2, "name": "d"}, {"filename": "a", "line": 1, "name": "b"}], + ] + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("DELETE FROM pt_snap_event_stack") + conn.execute("DELETE FROM pt_snap_stack_frame") + conn.execute("DELETE FROM pt_snap_stack") + conn.execute("DELETE FROM pt_snap_frame_value") + for table in ( + "pt_snap_event_stack", + "pt_snap_stack_frame", + "pt_snap_stack", + "pt_snap_frame_value", + ): + conn.execute(f"DROP TABLE {table}") + writer = OrderedFrameWriter(conn) + for event, frames in enumerate(arrays): + writer.insert(TraceEntry.from_dict({"id": event, "action": "alloc", "frames": frames})) + conn.execute("UPDATE trace_entry_0 SET callstack=? WHERE id IN (0,1)", ("a:1 b\nc:2 d",)) + with SnapshotAnalyzer(root) as analyzer: + rows = analyzer.execute_query(TEMPLATE, {"event_id": 1})["rows"] + assert_conserved(rows) + assert len([row for row in rows if row["filename"] == "a"]) == 2 + assert sorted(row["size_bytes"] for row in rows if row["filename"] == "a") == [60, 90] + + +def test_repeated_ordered_stacks_reduce_before_fetching_with_real_work_budget(tmp_path): + source = write_scale_source(tmp_path / "scale.pkl", 10_003) + result = ImportService().import_snapshot( + ImportOptions( + source, tmp_path / "output", set_focus=False, events_per_slice=5_000, format="msinsight" + ) + ) + before = hashes(result.db_path) + with SnapshotAnalyzer(result.db_path) as analyzer: + stats = analyzer.execute_query( + "callstack_analysis", {"min_count": 1}, query_limits=QueryLimits(max_work_rows=1000) + ) + assert len(stats["rows"]) == 6 + assert sum(row["alloc_count"] for row in stats["rows"]) == 10_003 + assert sum(row["total_size"] for row in stats["rows"]) == 16 * 10_003 + 64 + assert stats["scope"]["source_coverage"]["ordered_frame_events"] == 10_003 + for event, expected in ((4_999, 176), (5_000, 160), (10_002, 176)): + tree = analyzer.execute_query(TEMPLATE, {"event_id": event}) + assert tree["rows"][0]["size_bytes"] == expected + assert tree["scope"]["source_coverage"]["ordered_frames_complete"] + assert_conserved(tree["rows"]) + assert hashes(result.db_path) == before + + +def test_legacy_native_artifact_is_readable_but_frame_tree_degrades(tmp_path): + request = options(tmp_path, focus=False) + result = ImportService().import_snapshot(dataclasses.replace(request, format=None)) + manifest = result.db_path / "manifest.json" + raw = json.loads(manifest.read_text()) + raw["identity"]["extensionContractVersion"] = 1 + raw.pop("extensions") + manifest.write_text(json.dumps(raw)) + before = hashes(result.db_path) + with SnapshotAnalyzer(result.db_path) as analyzer: + result = analyzer.execute_query(TEMPLATE, {"event_id": 3}) + assert_conserved(result["rows"]) + assert {row["name"] for row in result["rows"]} == {None} + assert not result["scope"]["source_coverage"]["ordered_frames_complete"] + assert hashes(manifest.parent) == before + + +def test_statistics_representative_is_earliest_event_not_local_stack_id(tmp_path): + root = framed_dataset(tmp_path) + # Equivalent arrays may use different local IDs in optional frame evidence. + # SQLite's GROUP BY order must not select event 1 ahead of event 0. + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute( + "INSERT INTO pt_snap_stack SELECT 99,frameCount FROM pt_snap_stack WHERE stackId=0" + ) + conn.execute( + "INSERT INTO pt_snap_stack_frame SELECT 99,frameIndex,frameId " + "FROM pt_snap_stack_frame WHERE stackId=0" + ) + conn.execute("UPDATE pt_snap_event_stack SET stackId=99 WHERE eventId=0") + before = hashes(root) + with SnapshotAnalyzer(root) as analyzer: + rows = analyzer.execute_query("callstack_analysis", {"min_count": 1})["rows"] + shared = next(row for row in rows if row["callstack"].startswith("shared:")) + assert shared["stack_event_id"] == 0 + assert shared["alloc_count"] == 3 # Events 0, 1 and 11 share the raw array. + assert shared["total_size"] == 24 # Trace-event sizes, not live block sizes. + assert hashes(root) == before + + +def test_large_shared_stack_batch_never_searches_all_event_ids(tmp_path): + class EventIds(list): + def __contains__(self, event): + pytest.fail("Frame loading must not perform linear event-list membership") + + class Budget: + def remaining(self): + return None + + path = tmp_path / "many-allocations.db" + with closing(sqlite3.connect(path)) as conn: + conn.executescript( + "CREATE TABLE pt_snap_event_stack(eventId INTEGER PRIMARY KEY,stackId INTEGER);" + "CREATE TABLE pt_snap_stack(stackId INTEGER PRIMARY KEY,frameCount INTEGER);" + "CREATE TABLE pt_snap_stack_frame(stackId INTEGER,frameIndex INTEGER,frameId INTEGER);" + "CREATE TABLE pt_snap_frame_value(id INTEGER PRIMARY KEY,frameJson TEXT);" + "INSERT INTO pt_snap_stack VALUES(0,1);" + "INSERT INTO pt_snap_stack_frame VALUES(0,0,0);" + ) + conn.execute( + "INSERT INTO pt_snap_frame_value VALUES(0,?)", + (json.dumps({"filename": "alloc.py", "line": 1, "name": "alloc"}),), + ) + conn.executemany( + "INSERT INTO pt_snap_event_stack VALUES(?,0)", ((i,) for i in range(30000)) + ) + conn.row_factory = sqlite3.Row + + class Resolver: + _frame_schemas = {path: True} + budget = Budget() + reads = 0 + + def read(self, path, sql, values=None): + self.reads += 1 + return [dict(row) for row in conn.execute(sql, values or [])] + + resolver = Resolver() + frames = attach_frames(resolver, path, EventIds(range(30000))) + assert len(frames) == 30000 + assert resolver.reads == 2 * ((30000 + 255) // 256) + assert frames[0] == frames[29999] diff --git a/tests/core/test_dataset_global.py b/tests/core/test_dataset_global.py index 7f89b6b..b267894 100644 --- a/tests/core/test_dataset_global.py +++ b/tests/core/test_dataset_global.py @@ -483,6 +483,7 @@ def test_all_dataset_row_keys_are_declared_by_shared_template_schema(tmp_path, l "stack_bytes": 0 if detailed else -1, }, "preexisting_live": {"event_id": event}, + "active_memory_frame_tree_at_event": {"event_id": event}, } with SnapshotAnalyzer(root) as analyzer: for template in DATASET_SUPPORT: diff --git a/tests/core/test_dataset_id_pagination.py b/tests/core/test_dataset_id_pagination.py index 56e4254..70d5aa8 100644 --- a/tests/core/test_dataset_id_pagination.py +++ b/tests/core/test_dataset_id_pagination.py @@ -59,6 +59,23 @@ def paired_dataset(tmp_path_factory, request): ] +def assert_shared_rows(actual, expected): + """Compare common columns while checking new dataset event frame evidence.""" + assert len(actual) == len(expected) + for row, reference in zip(actual, expected, strict=True): + assert {key: row[key] for key in reference} == reference + if "frames_status" in row: + assert row["frames_status"] == "ordered" + assert isinstance(row["frames"], list) + assert ( + "\n".join( + f"{frame['filename']}:{frame['line']} {frame['name']}" + for frame in reversed(row["frames"]) + ) + == reference["callstack"] + ) + + @pytest.mark.parametrize("template", ["event", "allocation"]) @pytest.mark.parametrize("direction", ["ASC", "DESC"]) def test_id_windows_match_standalone_with_filters_and_sparse_ids( @@ -100,7 +117,7 @@ def test_id_windows_match_standalone_with_filters_and_sparse_ids( expected = service.execute_query( template, params, single, max_rows=max_rows, exact_total=exact ) - assert actual.rows == expected.rows, params + assert_shared_rows(actual.rows, expected.rows) for field in ("total", "returned", "has_more", "truncated", "total_is_exact"): assert getattr(actual, field) == getattr(expected, field), (params, field) assert actual.scope["actual_range"] == { @@ -131,7 +148,7 @@ def test_id_slice_pages_preserve_scope_and_exact_total(paired_dataset, template, template, params, root, slice_index=item.index, exact_total=True ) expected = service.execute_query(template, params, single, exact_total=True) - assert actual.rows == expected.rows + assert_shared_rows(actual.rows, expected.rows) assert actual.total == expected.total assert actual.has_more == expected.has_more assert actual.truncated == expected.truncated diff --git a/tests/core/test_dataset_scale_acceptance.py b/tests/core/test_dataset_scale_acceptance.py index 415b8ed..d0ccbd5 100644 --- a/tests/core/test_dataset_scale_acceptance.py +++ b/tests/core/test_dataset_scale_acceptance.py @@ -375,7 +375,7 @@ def test_installed_trusted_producer_default_export_cli_and_api( assert {case["template"] for case in cases} == set(by_name) supported = {name for name, entry in by_name.items() if entry["dataset_support"]["supported"]} unsupported = set(by_name) - supported - assert unsupported == {"event_frames", "active_memory_frame_tree_at_event"} + assert unsupported == {"event_frames"} cases_file = runtime.cwd / "cases.json" cases_file.write_text(json.dumps(cases)) api = runtime.python_json("api", API_QUERIES, root, cases_file, boundary - 1) @@ -419,10 +419,12 @@ def test_installed_trusted_producer_default_export_cli_and_api( "--event-id", boundary - 1, "--json", - expected=1, + expected=0, ) - assert tree["error"]["code"] == api["tree_report"]["code"] == "QUERY_FAILED" - assert "Unsupported dataset-global" in tree["error"]["message"] + assert tree["rows"] == api["tree_report"]["rows"] + assert tree["rows"] == cli_queries["active_memory_frame_tree_at_event"]["rows"] + assert tree["rows"][0]["size_bytes"] == 176 + assert sum(row["self_bytes"] for row in tree["rows"]) == 176 assert hashes(root) == before assert sha256(source) == source_hash assert not (runtime.cwd / ".pt-snap").exists() @@ -438,7 +440,7 @@ def test_installed_trusted_producer_default_export_cli_and_api( "slices": 2, "templates_supported": sorted(supported), "templates_expected_rejections": sorted(unsupported), - "reports": {"peak-memory": "passed", "memory-tree": "expected dataset rejection"}, + "reports": {"peak-memory": "passed", "memory-tree": "passed"}, "commands": runtime.commands, "read_only_verified": True, } diff --git a/tests/core/test_dataset_sql_pushdown.py b/tests/core/test_dataset_sql_pushdown.py index 01b2ac3..b5f4ffc 100644 --- a/tests/core/test_dataset_sql_pushdown.py +++ b/tests/core/test_dataset_sql_pushdown.py @@ -390,6 +390,9 @@ def test_global_text_identity_and_candidates_ignore_source_collation( conn.execute("UPDATE trace_entry_0 SET callstackId=1 WHERE id=?", [ids[1]]) conn.execute("UPDATE trace_entry_0 SET size=1 WHERE id>=0") if layout == "v2": + # This fixture tests text collation, independently of the original raw + # frames retained by new imports. Frame identity has separate coverage. + manifest.pop("extensions", None) item["sha256"] = hashlib.sha256(member.read_bytes()).hexdigest() manifest_path.write_text(json.dumps(manifest)) expected_text = {event: texts[int(event == ids[1])] for event in ids} diff --git a/tests/core/test_dataset_validation_cache.py b/tests/core/test_dataset_validation_cache.py index 8a356ef..527c6f9 100644 --- a/tests/core/test_dataset_validation_cache.py +++ b/tests/core/test_dataset_validation_cache.py @@ -13,7 +13,7 @@ from pt_snap_cli.core.dataset_contract import validate_dataset from pt_snap_cli.core.dataset_files import hash_file from pt_snap_cli.core.dataset_resolver import DatasetResolver -from pt_snap_cli.core.errors import DatabaseSchemaError, QueryTimeoutError +from pt_snap_cli.core.errors import DatabaseSchemaError, QueryCancelledError, QueryTimeoutError from pt_snap_cli.core.validation_budget import validation_progress from tests.test_dataset_focus import make_dataset from tests.test_native_dataset import dataset as native_dataset @@ -66,7 +66,9 @@ def changed(*args, **kwargs): @pytest.mark.parametrize("cache_mode", [None, "immutable", "disabled"]) @pytest.mark.parametrize("seal", ["parserCache", "ptSnap"]) -def test_declared_seals_are_checked_in_every_cache_mode(tmp_path, monkeypatch, cache_mode, seal): +def test_historical_digests_do_not_reject_valid_content_changes( + tmp_path, monkeypatch, cache_mode, seal +): from pt_snap_cli.core.snapshot_cache import seal_manifest if cache_mode is None: @@ -87,8 +89,7 @@ def test_declared_seals_are_checked_in_every_cache_mode(tmp_path, monkeypatch, c assert DatasetResolver().inspect(root) is not None with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: conn.execute("UPDATE trace_entry_0 SET allocated=allocated+1") - with pytest.raises(DatabaseSchemaError, match="seal"): - DatasetResolver().inspect(root) + assert DatasetResolver().inspect(root) is not None def forbidden(*args, **kwargs): @@ -462,6 +463,22 @@ def test_strong_operation_guard_hashes_even_when_filesystem_tokens_look_unchange resolved.require_unchanged() +@pytest.mark.parametrize("error", [QueryTimeoutError, QueryCancelledError]) +def test_cancelled_admission_evicts_an_existing_entry(tmp_path, monkeypatch, error): + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + DatasetResolver().inspect(root) + assert root in dataset_resolver._VALIDATED + + class Budget: + def remaining(self): + raise error("interrupted admission") + + with pytest.raises(error, match="interrupted admission"): + DatasetResolver().inspect(root, budget=Budget()) + assert root not in dataset_resolver._VALIDATED + + def test_disabling_cache_does_not_reuse_an_existing_entry(tmp_path, monkeypatch): monkeypatch.delenv("PT_SNAP_DATASET_CACHE") root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) diff --git a/tests/core/test_import_validation_receipts.py b/tests/core/test_import_validation_receipts.py new file mode 100644 index 0000000..3f65ff0 --- /dev/null +++ b/tests/core/test_import_validation_receipts.py @@ -0,0 +1,186 @@ +"""Import receipts avoid row scans across processes without trusting timestamps.""" + +import dataclasses +import json +import os +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.core import dataset_resolver, import_service +from pt_snap_cli.core.errors import ImportExecutionError, QueryTimeoutError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.validation_receipts import import_receipts, load_receipt, save_receipt +from tests.core.test_dataset_validation_cache import CancellableBudget +from tests.core.test_msinsight_export import options + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_DATASET_CACHE", raising=False) + dataset_resolver._VALIDATED.clear() + yield + dataset_resolver._VALIDATED.clear() + + +def unexpected(*args, **kwargs): + pytest.fail("unchanged imported bytes must not require another row validation") + + +def test_first_import_seeds_cold_repeat_including_focus(tmp_path, monkeypatch): + request = dataclasses.replace(options(tmp_path), format=None) + first = ImportService().import_snapshot(request) + dataset_resolver._VALIDATED.clear() # A fresh CLI process has no in-memory entries. + for name in ("validate_native_dataset", "validate_dataset", "validate_cache"): + monkeypatch.setattr(dataset_resolver, name, unexpected) + monkeypatch.setattr(import_service, "frame_member_digest", unexpected) + hashed = [] + original = dataset_resolver._hash_file + + def hash_file(path, **kwargs): + hashed.append(path) + return original(path, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", hash_file) + second = ImportService().import_snapshot(request) + assert second.reused and second.metadata == first.metadata + assert second.focus_state.available_devices == first.focus_state.available_devices + members = list(first.db_path.glob("device_*/*.db")) + assert members and all(path in hashed for path in members) + + +def test_same_size_and_mtime_frame_edit_misses_receipt(tmp_path): + request = dataclasses.replace(options(tmp_path, focus=False), format=None) + first = ImportService().import_snapshot(request) + member = next(first.db_path.glob("device_*/*.db")) + before = member.stat() + with closing(sqlite3.connect(member)) as conn, conn: + row = conn.execute("SELECT id,frameJson FROM pt_snap_frame_value LIMIT 1").fetchone() + # Same-length valid JSON, without changing the number or size of rows. + changed = row[1].replace("name", "nAme", 1) + assert changed != row[1] + conn.execute("UPDATE pt_snap_frame_value SET frameJson=? WHERE id=?", (changed, row[0])) + assert member.stat().st_size == before.st_size + os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns)) + dataset_resolver._VALIDATED.clear() + with pytest.raises(ImportExecutionError): + ImportService().import_snapshot(request) + assert member.exists() + + +def test_compatible_consumer_table_change_reuses_without_receipt(tmp_path, monkeypatch): + request = options(tmp_path, focus=False) + first = ImportService().import_snapshot(request) + member = next(first.db_path.glob("device_*/*.db")) + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute("CREATE TABLE gui_cache(value TEXT)") + conn.execute("INSERT INTO gui_cache VALUES ('consumer')") + dataset_resolver._VALIDATED.clear() + monkeypatch.setattr(dataset_resolver, "validate_dataset", unexpected) + assert ImportService().import_snapshot(request).reused + dataset_resolver._VALIDATED.clear() + assert ImportService().import_snapshot(request).reused + + +@pytest.mark.parametrize("damage", ["missing", "malformed", "checksum", "version"]) +def test_missing_or_damaged_receipt_falls_back(tmp_path, monkeypatch, damage): + request = dataclasses.replace(options(tmp_path, focus=False), format=None) + ImportService().import_snapshot(request) + folder = tmp_path / ".cache/pt-snap-cli/validation-v1" + receipts = list(folder.glob("*.json")) + assert receipts + for path in receipts: + if damage == "missing": + path.unlink() + elif damage == "malformed": + path.write_text("{", encoding="utf-8") + elif damage == "checksum": + data = json.loads(path.read_bytes()) + data["sha256"] = "0" * 64 + path.write_text(json.dumps(data), encoding="utf-8") + else: + path.rename(path.with_suffix(".old")) + dataset_resolver._VALIDATED.clear() + calls = [] + original = dataset_resolver.validate_native_dataset + + def validate(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(dataset_resolver, "validate_native_dataset", validate) + assert ImportService().import_snapshot(request).reused + assert calls == [1] + + +def test_receipt_is_import_only_and_off_disables_it(tmp_path, monkeypatch): + request = dataclasses.replace(options(tmp_path, focus=False), format=None) + result = ImportService().import_snapshot(request) + dataset_resolver._VALIDATED.clear() + calls = [] + original = dataset_resolver.validate_native_dataset + + def validate(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(dataset_resolver, "validate_native_dataset", validate) + dataset_resolver.DatasetResolver().inspect(result.db_path) + assert calls == [1] # Ordinary analysis retains its existing admission contract. + calls.clear() + dataset_resolver._VALIDATED.clear() + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "off") + assert ImportService().import_snapshot(request).reused + assert calls == [1] + + +def test_unwritable_receipt_directory_is_only_a_cache_miss(tmp_path): + (tmp_path / ".cache").write_text("file blocks directory creation", encoding="utf-8") + with import_receipts(): + save_receipt(tmp_path, "dataset", "fingerprint", {}) + assert load_receipt(tmp_path, "dataset", "fingerprint") is None + request = dataclasses.replace(options(tmp_path, focus=False), format=None) + assert not ImportService().import_snapshot(request).reused + dataset_resolver._VALIDATED.clear() + assert ImportService().import_snapshot(request).reused + + +def test_receipt_hit_hashing_respects_cancellation(tmp_path, monkeypatch): + request = dataclasses.replace(options(tmp_path, focus=False), format=None) + first = ImportService().import_snapshot(request) + dataset_resolver._VALIDATED.clear() + budget = CancellableBudget() + original = dataset_resolver._hash_file + + def cancelled(path, **kwargs): + if path.suffix == ".db": + budget.cancelled = True + return original(path, **kwargs) + + with monkeypatch.context() as patch, import_receipts(): + patch.setattr(dataset_resolver, "_hash_file", cancelled) + with pytest.raises(QueryTimeoutError): + dataset_resolver.DatasetResolver().inspect(first.db_path, budget=budget) + assert first.db_path not in dataset_resolver._VALIDATED + assert ImportService().import_snapshot(request).reused + + +def test_compatible_committed_wal_frame_edits_do_not_trigger_content_checks(tmp_path): + request = options(tmp_path, focus=False) + first = ImportService().import_snapshot(request) + member = next(first.db_path.glob("device_*/*.db")) + with closing(sqlite3.connect(member)) as writer: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("CREATE TABLE gui_cache(value TEXT)") + writer.commit() + assert ImportService().import_snapshot(request).reused + writer.execute('UPDATE pt_snap_frame_value SET frameJson=\'{"name":"changed"}\'') + writer.commit() + assert Path(str(member) + "-wal").stat().st_size > 0 + dataset_resolver._VALIDATED.clear() + assert ImportService().import_snapshot(request).reused diff --git a/tests/core/test_latest_query.py b/tests/core/test_latest_query.py new file mode 100644 index 0000000..71a842c --- /dev/null +++ b/tests/core/test_latest_query.py @@ -0,0 +1,205 @@ +"""Latest-only coordination does not publish superseded or reset work.""" + +import sqlite3 +import time +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +from threading import Event, Lock + +import pytest + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core.dataset_sources import QueryBudget +from pt_snap_cli.core.errors import QueryCancelledError, QueryTimeoutError +from pt_snap_cli.core.latest_query import LatestQuery +from pt_snap_cli.core.query_service import QueryService +from pt_snap_cli.core.validation_budget import validation_progress +from pt_snap_cli.query.config import QueryParameter, QueryTemplate +from pt_snap_cli.query.executor import QueryExecutor + + +@pytest.mark.parametrize("supplied_budget", [False, True]) +def test_queued_tree_timeout_includes_wait_and_does_not_start_worker(monkeypatch, supplied_budget): + now = [100.0] + monkeypatch.setattr("pt_snap_cli.core.query_service.time.monotonic", lambda: now[0]) + waiting = Event() + work = Lock() + work.acquire() + + class ObservedLock: + def acquire(self, *, timeout): + waiting.set() + return work.acquire(timeout=timeout) + + def release(self): + work.release() + + def unexpected_worker(*args, **kwargs): + pytest.fail("An expired queued request must not start its query worker") + + service = QueryService() + service._frame_tree_queries._work = ObservedLock() + monkeypatch.setattr(QueryService, "_execute_query", unexpected_worker) + budget = QueryBudget(1.0, now[0]) if supplied_budget else None + try: + with ThreadPoolExecutor(max_workers=1) as pool: + request = pool.submit( + service.execute_query, + "active_memory_frame_tree_at_event", + timeout_s=1.0, + _budget=budget, + ) + try: + assert waiting.wait(3) + now[0] += 5.0 + # The caller returns while the old task still owns the lock. + with pytest.raises(QueryTimeoutError): + request.result(timeout=3) + assert work.locked() + assert service._frame_tree_queries._active is None + finally: + work.release() + finally: + service.close() + + +def test_deadline_is_checked_after_acquiring_work_lock(): + queries = LatestQuery() + checks = 0 + + def remaining(): + nonlocal checks + checks += 1 + if checks == 2: + raise QueryTimeoutError("expired while acquiring lock") + return 1.0 + + with pytest.raises(QueryTimeoutError): + with queries.run(remaining): + pytest.fail("Expired work must not run") + # Timeout releases the lock and does not poison a subsequent request. + with queries.run(): + pass + + +def test_cancel_discards_result_and_next_query_can_run(): + queries = LatestQuery() + with pytest.raises(QueryCancelledError): + with queries.run() as cancelled: + assert not cancelled() + queries.cancel() + assert cancelled() + with queries.run() as cancelled: + assert not cancelled() + + +def test_query_failure_does_not_poison_future_queries(): + queries = LatestQuery() + with pytest.raises(ValueError): + with queries.run(): + raise ValueError("query failed") + with queries.run() as cancelled: + assert not cancelled() + + +def test_queued_obsolete_request_is_skipped(): + queries = LatestQuery() + entered = [] + queries._work.acquire() + + def run(number): + with queries.run(): + entered.append(number) + + def wait_for_token(previous): + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + with queries._state: + token = queries._active + if token is not None and token is not previous: + return token + Event().wait(0.005) + pytest.fail("The request did not register its cancellation token") + + with ThreadPoolExecutor(max_workers=2) as pool: + try: + first = pool.submit(run, 1) + token = wait_for_token(None) + second = pool.submit(run, 2) + wait_for_token(token) + finally: + queries._work.release() + with pytest.raises(QueryCancelledError): + first.result(timeout=3) + second.result(timeout=3) + assert entered == [2] + + +def test_independent_services_do_not_cancel_each_other(): + first, second = LatestQuery(), LatestQuery() + with first.run() as cancelled: + with pytest.raises(QueryCancelledError): + with second.run(): + second.cancel() + assert not cancelled() + + +def test_cancellation_during_dataset_validation_preserves_error_and_cleans_handler(): + class Budget: + def __init__(self): + self.checks = 0 + + def remaining(self): + self.checks += 1 + if self.checks >= 3: + raise QueryCancelledError("cancelled during validation") + return None + + with closing(sqlite3.connect(":memory:")) as conn: + with pytest.raises(QueryCancelledError, match="during validation"): + with validation_progress(conn, Budget()): + conn.execute( + "WITH RECURSIVE n(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM n WHERE x<100000000) " + "SELECT SUM(x) FROM n" + ).fetchall() + assert conn.execute("SELECT 1").fetchone() == (1,) + + +@pytest.mark.parametrize("timeout", [0, 30]) +def test_standalone_tree_interrupts_sql_and_preserves_runtime_templates( + tmp_path, monkeypatch, timeout +): + path = tmp_path / "standalone.db" + with closing(sqlite3.connect(path)) as conn: + conn.executescript( + "CREATE TABLE dictionary (id INTEGER);" "CREATE TABLE trace_entry_0 (id INTEGER);" + ) + started = Event() + execute = QueryExecutor.execute + + def observed(self, sql, *args, **kwargs): + if "WITH RECURSIVE" in sql: + started.set() + return execute(self, sql, *args, **kwargs) + + monkeypatch.setattr(QueryExecutor, "execute", observed) + template = "active_memory_frame_tree_at_event" + with SnapshotAnalyzer(path) as analyzer, ThreadPoolExecutor(max_workers=2) as pool: + source = analyzer._query_service._get_executor(analyzer.context_cache.get(path)) + source.register_template( + QueryTemplate( + name=template, + parameters={"event_id": QueryParameter(name="event_id", type="int", required=True)}, + query="{% if event_id == 1 %}" + "WITH RECURSIVE n(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM n WHERE x<100000000) " + "SELECT SUM(x) AS value FROM n" + "{% else %}SELECT 7 AS value{% endif %}", + ) + ) + old = pool.submit(analyzer.execute_query, template, {"event_id": 1}, timeout_s=timeout) + assert started.wait(3) + new = pool.submit(analyzer.execute_query, template, {"event_id": 2}) + with pytest.raises(QueryCancelledError): + old.result(timeout=5) + assert new.result(timeout=5)["rows"] == [{"value": 7}] + assert analyzer.execute_query(template, {"event_id": 2})["rows"] == [{"value": 7}] diff --git a/tests/core/test_lightweight_compatible_import.py b/tests/core/test_lightweight_compatible_import.py new file mode 100644 index 0000000..535b9ba --- /dev/null +++ b/tests/core/test_lightweight_compatible_import.py @@ -0,0 +1,110 @@ +"""Compatible cache reuse only checks identity, inventory and schema.""" + +import dataclasses +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from pt_snap_cli.cli import app +from pt_snap_cli.core import dataset_resolver, import_service +from pt_snap_cli.core.errors import ImportExecutionError +from pt_snap_cli.core.import_service import ImportService +from tests.core.test_msinsight_export import options + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_DATASET_CACHE", raising=False) + dataset_resolver._VALIDATED.clear() + yield + dataset_resolver._VALIDATED.clear() + + +def unexpected(*args, **kwargs): + pytest.fail("default compatible reuse must not scan or hash member contents") + + +@pytest.mark.parametrize("external", [False, True]) +@pytest.mark.parametrize("focus", [False, True]) +def test_repeat_import_skips_all_content_scans(tmp_path, monkeypatch, external, focus): + request = options(tmp_path, focus=focus) + monkeypatch.setattr(import_service, "save_receipt", unexpected) + monkeypatch.setattr(dataset_resolver, "save_receipt", unexpected) + monkeypatch.setattr(dataset_resolver, "validate_dataset", unexpected) + first = ImportService().import_snapshot(request) + assert not dataset_resolver._VALIDATED + if external: + path = first.db_path / "manifest.json" + manifest = json.loads(path.read_bytes()) + del manifest["ptSnap"] + path.write_text(json.dumps(manifest), encoding="utf-8") + dataset_resolver._VALIDATED.clear() + monkeypatch.setattr(dataset_resolver, "validate_dataset", unexpected) + hash_file = dataset_resolver._hash_file + + def no_member_hash(path, **kwargs): + if path.suffix == ".db" or path.name.endswith("-wal"): + unexpected() + return hash_file(path, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", no_member_hash) + repeated = ImportService().import_snapshot(request) + assert repeated.reused + assert repeated.devices == first.devices + assert repeated.slice_count == first.slice_count + assert repeated.metadata == (None if external else first.metadata) + assert not dataset_resolver._VALIDATED # Never promote structural checks to query admission. + if focus: + assert repeated.focus_state.available_devices == first.focus_state.available_devices + + +def test_content_edits_do_not_trigger_cache_replay(tmp_path): + request = options(tmp_path, focus=False) + first = ImportService().import_snapshot(request) + member = next(first.db_path.glob("device_*/*.db")) + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute("UPDATE block_0 SET size=size+1") + assert ImportService().import_snapshot(request).reused + + +@pytest.mark.parametrize("cache_mode", ["disabled", "immutable"]) +def test_cache_configuration_does_not_enable_content_checks(tmp_path, monkeypatch, cache_mode): + request = options(tmp_path, focus=True) + ImportService().import_snapshot(request) + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", cache_mode) + monkeypatch.setattr(dataset_resolver, "validate_dataset", unexpected) + assert ImportService().import_snapshot(request).reused + + +def test_import_help_has_no_strict_cache_option(): + result = CliRunner().invoke(app, ["import", "--help"], color=False) + assert result.exit_code == 0, result.output + assert "--verify-cache-content" not in result.stdout + + +@pytest.mark.parametrize("damage", ["missing", "table", "manifest"]) +def test_default_import_still_rejects_incomplete_artifacts(tmp_path, damage): + request = options(tmp_path, focus=False) + first = ImportService().import_snapshot(request) + member = next(first.db_path.glob("device_*/*.db")) + if damage == "missing": + member.unlink() + elif damage == "table": + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute("DROP TABLE block_0") + else: + manifest_path = first.db_path / "manifest.json" + manifest = json.loads(manifest_path.read_bytes()) + manifest["status"] = "building" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ImportExecutionError): + ImportService().import_snapshot(dataclasses.replace(request, force=True)) + if damage == "missing": + assert not member.exists() diff --git a/tests/core/test_msinsight_export.py b/tests/core/test_msinsight_export.py index ef2b5e8..54764f5 100644 --- a/tests/core/test_msinsight_export.py +++ b/tests/core/test_msinsight_export.py @@ -74,7 +74,7 @@ def test_physical_schema_inline_events_streams_stable_references_and_native_pari assert manifest["sourceFile"] == str(request.snapshot_file.resolve()) assert manifest["eventsPerSlice"] == (capacity or 500000) raw = request.snapshot_file.read_bytes() - assert manifest["cacheHash"] == hashlib.sha256(b"mem_snapshot_parser_v2" + raw).hexdigest() + assert manifest["cacheHash"] == hashlib.sha256(b"mem_snapshot_parser_v3" + raw).hexdigest() assert result.metadata.source_sha256 == hashlib.sha256(raw).hexdigest() != manifest["cacheHash"] assert manifest["ptSnap"]["identity"]["targetRevision"] == MSINSIGHT_REVISION native = service.import_snapshot( @@ -139,12 +139,19 @@ def test_physical_schema_inline_events_streams_stable_references_and_native_pari assert analyzer.get_focus().callstack_layout == "v1" for device in (0, 1): for event in range(5): - assert ( - analyzer.execute_query("event", {"id": event}, device_id=device)["rows"] - == native_analyzer.execute_query("event", {"id": event}, device_id=device)[ - "rows" - ] - ) + compatible_rows = analyzer.execute_query("event", {"id": event}, device_id=device)[ + "rows" + ] + native_rows = native_analyzer.execute_query( + "event", {"id": event}, device_id=device + )["rows"] + # Generation namespaces differ across physical formats; frame + # digests and every event/frame value must remain equivalent. + for event_rows in (compatible_rows, native_rows): + for row in event_rows: + for key in ("stack_id", "source_stack_id"): + row[key] = row[key].split(":frames:", 1)[-1] + assert compatible_rows == native_rows assert not hasattr(analyzer, "import_snapshot") assert hashes(result.db_path) == before @@ -194,11 +201,10 @@ def test_consumer_tables_do_not_invalidate_compatible_cache(tmp_path, monkeypatc assert rows(member, "SELECT value FROM unrelated_consumer_cache") == [("keep",)] with closing(sqlite3.connect(member)) as conn, conn: conn.execute("UPDATE block_0 SET size=size+1") - with pytest.raises(ImportExecutionError, match="preserved"): - service.import_snapshot(request) + assert service.import_snapshot(request).reused -def test_producer_only_logical_seal_reuses_without_replaying(tmp_path, monkeypatch): +def test_producer_metadata_reuses_without_replaying(tmp_path, monkeypatch): request = options(tmp_path, focus=False) service = ImportService() first = service.import_snapshot(request) @@ -219,8 +225,7 @@ def test_producer_only_logical_seal_reuses_without_replaying(tmp_path, monkeypat assert analyzer.get_database_metadata()["status"] == "available" with closing(sqlite3.connect(first.db_path / "device_0/slice_00000.db")) as conn, conn: conn.execute("UPDATE block_0 SET size=size+1") - with pytest.raises(ImportExecutionError, match="preserved"): - service.import_snapshot(request) + assert service.import_snapshot(request).reused @pytest.mark.parametrize("cli_capacity", [None, 2]) @@ -287,7 +292,7 @@ def test_external_insight_artifact_reuses_without_claiming_metadata( "last-slice", ], ) -def test_compatible_cache_requires_semantic_validation( +def test_queries_validate_semantics_but_cache_import_does_not_scan( tmp_path, monkeypatch, sealed, external, set_focus, slice_index, mutation, diagnostic ): from pt_snap_cli.core.errors import DatabaseSchemaError @@ -309,7 +314,7 @@ def test_compatible_cache_requires_semantic_validation( if not sealed: del manifest["parserCache"] path.write_text(json.dumps(manifest)) - # The digest is self-consistent, but must not replace shared P0 validation. + # Cache admission checks structure; queries independently validate semantics. validate_cache(result.db_path, manifest["cacheHash"]) with pytest.raises(DatabaseSchemaError, match=diagnostic): DatasetResolver().inspect(result.db_path) @@ -323,11 +328,13 @@ def test_compatible_cache_requires_semantic_validation( service._msinsight_backend.replay, "stage", lambda *a, **k: pytest.fail("must preserve") ) for force in (False, True): - with pytest.raises(ImportExecutionError, match=diagnostic): - service.import_snapshot(dataclasses.replace(request, force=force, set_focus=set_focus)) + assert service.import_snapshot( + dataclasses.replace(request, force=force, set_focus=set_focus) + ).reused assert hashes(result.db_path) == before assert request.snapshot_file.read_bytes() == source_before - assert focus.read_bytes() == focus_before + if not set_focus: + assert focus.read_bytes() == focus_before @pytest.mark.parametrize("capacity", [None, 2, 3]) @@ -359,21 +366,12 @@ def test_external_cache_honors_explicit_capacity(tmp_path, monkeypatch, capacity @pytest.mark.parametrize("force", [False, True]) -@pytest.mark.parametrize("failure", ["semantic", "capacity"]) -def test_external_cache_cli_failure_preserves_artifact_and_focus( - tmp_path, monkeypatch, force, failure -): - from pt_snap_cli.core.snapshot_cache import seal_manifest - +def test_external_cache_cli_failure_preserves_artifact_and_focus(tmp_path, monkeypatch, force): request = options(tmp_path, focus=False) result = ImportService().import_snapshot(request) path = result.db_path / "manifest.json" manifest = json.loads(path.read_text()) del manifest["ptSnap"] - if failure == "semantic": - with closing(sqlite3.connect(result.db_path / "device_0/slice_00000.db")) as conn, conn: - conn.execute("UPDATE trace_entry_0 SET allocated=-1 WHERE id=0") - seal_manifest(result.db_path, manifest) path.write_text(json.dumps(manifest)) before = hashes(result.db_path) source_before = request.snapshot_file.read_bytes() @@ -394,8 +392,7 @@ def test_external_cache_cli_failure_preserves_artifact_and_focus( str(request.output_dir), "--json", ] - if failure == "capacity": - args.extend(["--events-per-slice", "3"]) + args.extend(["--events-per-slice", "3"]) if force: args.append("--force") response = CliRunner().invoke(app, args) @@ -403,8 +400,7 @@ def test_external_cache_cli_failure_preserves_artifact_and_focus( assert response.stdout == "" error = json.loads(response.stderr) assert error["ok"] is False and error["error"]["code"] == "IMPORT_FAILED" - diagnostic = "event_values" if failure == "semantic" else "events_per_slice" - assert diagnostic in error["error"]["message"] + assert "events_per_slice" in error["error"]["message"] assert "Traceback" not in response.stderr assert hashes(result.db_path) == before assert request.snapshot_file.read_bytes() == source_before @@ -412,8 +408,7 @@ def test_external_cache_cli_failure_preserves_artifact_and_focus( @pytest.mark.parametrize("external", [False, True]) -@pytest.mark.parametrize("failure", ["index", "dictionary", "deep-json"]) -def test_invalid_compatible_cache_is_preserved(tmp_path, monkeypatch, external, failure): +def test_invalid_compatible_cache_is_preserved(tmp_path, monkeypatch, external): request = options(tmp_path, focus=False) service = ImportService() result = service.import_snapshot(request) @@ -421,24 +416,7 @@ def test_invalid_compatible_cache_is_preserved(tmp_path, monkeypatch, external, manifest = json.loads(path.read_text()) if external: del manifest["ptSnap"] - member = result.db_path / "device_0/slice_00000.db" - if failure == "index": - with closing(sqlite3.connect(member)) as conn: - conn.execute("CREATE INDEX block_address ON block_0(address)") - page_size = conn.execute("PRAGMA page_size").fetchone()[0] - root = conn.execute( - "SELECT rootpage FROM sqlite_master WHERE name='block_address'" - ).fetchone()[0] - with member.open("r+b") as file: - file.seek((root - 1) * page_size) - file.write(b"\xff" * page_size) - elif failure == "dictionary": - del manifest["parserCache"] - with closing(sqlite3.connect(member)) as conn, conn: - conn.execute("DELETE FROM dictionary WHERE \"table\"='block_0'") - path.write_text( - "[" * 200000 + "0" + "]" * 200000 if failure == "deep-json" else json.dumps(manifest) - ) + path.write_text("[" * 200000 + "0" + "]" * 200000) before = hashes(result.db_path) monkeypatch.setattr( service._msinsight_backend.replay, "stage", lambda *a, **k: pytest.fail("must preserve") @@ -524,7 +502,7 @@ def test_invalid_producer_extension_reports_context(tmp_path, field, value, mess @pytest.mark.parametrize("external", [False, True]) @pytest.mark.parametrize("mutation", ["block", "trace"]) -def test_resolver_rejects_seal_mismatch_after_prior_query(tmp_path, external, mutation): +def test_resolver_accepts_valid_edits_without_digest_comparison(tmp_path, external, mutation): result = ImportService().import_snapshot(options(tmp_path, focus=False)) if external: path = result.db_path / "manifest.json" @@ -541,10 +519,7 @@ def test_resolver_rejects_seal_mismatch_after_prior_query(tmp_path, external, mu else "UPDATE trace_entry_0 SET allocated=allocated+1" ) before = hashes(result.db_path) - from pt_snap_cli.core.errors import DatabaseSchemaError - - with pytest.raises(DatabaseSchemaError, match="parserCache.*seal"): - resolver.inspect(result.db_path) + assert resolver.inspect(result.db_path) is not None assert hashes(result.db_path) == before @@ -573,11 +548,10 @@ def test_live_consumer_wal_reuses_and_queries_committed_data(tmp_path, monkeypat with SnapshotAnalyzer(result.db_path) as analyzer: assert analyzer.execute_query("event", {"id": 0}, device_id=0)["rows"] assert writer.execute("SELECT value FROM consumer_data").fetchall() == [("committed",)] - # Base-table writes cannot masquerade as consumer-only additions. + # Cache reuse no longer scans base-table contents, including committed WAL. writer.execute("UPDATE block_0 SET size=size+1") writer.commit() - with pytest.raises(ImportExecutionError, match="preserved"): - service.import_snapshot(request) + assert service.import_snapshot(request).reused def test_compatible_negative_ids_are_compact_and_boundary_ids_are_global(tmp_path): diff --git a/tests/core/test_ordered_frame_integrity.py b/tests/core/test_ordered_frame_integrity.py new file mode 100644 index 0000000..55e1b53 --- /dev/null +++ b/tests/core/test_ordered_frame_integrity.py @@ -0,0 +1,148 @@ +"""Owned frame evidence must survive publication and repeat-import checks.""" + +import dataclasses +import hashlib +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core import import_service +from pt_snap_cli.core.errors import ImportExecutionError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.models import ImportOptions +from tests.core.test_dataset_import import hashes +from tests.core.test_msinsight_export import options +from tests.snapshot.test_sharded_replay import lifecycle_data, write_source + + +@pytest.mark.parametrize("fmt", [None, "msinsight"]) +@pytest.mark.parametrize("container", [dict, tuple]) +def test_serializable_deep_optional_frames_do_not_reject_import(tmp_path, fmt, container): + data = lifecycle_data() + data["device_traces"][0] = [e for e in data["device_traces"][0] if e["action"] != "oom"] + extra = 0 + for _ in range(140): + extra = {"nested": extra} if container is dict else (extra,) + for event in data["device_traces"][0]: + # The first valid frame must not be partially inserted before rejection. + event["frames"] = [dict(event["frames"][0]), {**event["frames"][0], "extra": extra}] + source = write_source(tmp_path, data) + result = ImportService().import_snapshot( + ImportOptions(source, tmp_path / "output", set_focus=False, events_per_slice=2, format=fmt) + ) + for member in result.db_path.glob("device_*/*.db"): + with closing(sqlite3.connect(member)) as conn: + for table in ( + "pt_snap_frame_value", + "pt_snap_stack", + "pt_snap_stack_frame", + "pt_snap_event_stack", + ): + assert conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0] == 0 + with SnapshotAnalyzer(result.db_path) as analyzer: + row = analyzer.execute_query("event", {"id": 0})["rows"][0] + assert row["callstack"] + assert row["frames"] is None and row["frames_status"] == "text_only" + tree = analyzer.execute_query("active_memory_frame_tree_at_event", {"event_id": 0}) + assert any(node["node_id"] == "missing" for node in tree["rows"]) + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + + +MUTATIONS = { + "missing-table": "DROP TABLE pt_snap_stack_frame", + "bad-count": "UPDATE pt_snap_stack SET frameCount=frameCount+1", + "bad-index": "UPDATE pt_snap_stack_frame SET frameIndex=frameIndex+100", + "orphan-frame": "UPDATE pt_snap_stack_frame SET frameId=999999", + "orphan-event": "INSERT INTO pt_snap_event_stack VALUES(999999,0)", + "nonfinite-json": "UPDATE pt_snap_frame_value SET frameJson='{\"name\":NaN}'", + "duplicate-key": 'UPDATE pt_snap_frame_value SET frameJson=\'{"name":1,"name":2}\'', +} + + +@pytest.mark.parametrize("fmt", [None]) +@pytest.mark.parametrize("mutation", MUTATIONS) +def test_invalid_new_frames_never_publish(tmp_path, monkeypatch, fmt, mutation): + service = ImportService() + request = dataclasses.replace(options(tmp_path, focus=False), format=fmt) + backend = service._msinsight_backend if fmt else service._dataset_backend + stage = backend.replay.stage + + def damaged_stage(*args, **kwargs): + result = stage(*args, **kwargs) + with closing(sqlite3.connect(result.slices[0].file)) as conn, conn: + conn.execute(MUTATIONS[mutation]) + return result + + monkeypatch.setattr(backend.replay, "stage", damaged_stage) + with pytest.raises(ImportExecutionError): + service.import_snapshot(request) + target = backend.target_path(request.snapshot_file, request.output_dir) + assert not target.exists() + assert not list(request.output_dir.glob(".*stage*")) + assert not (tmp_path / ".pt-snap").exists() + + +@pytest.mark.parametrize("fmt", [None]) +def test_invalid_new_declaration_never_publishes(tmp_path, monkeypatch, fmt): + name = "build_compatible_manifest" if fmt else "build_native_manifest" + build = getattr(import_service, name) + + def missing_declaration(*args, **kwargs): + manifest = build(*args, **kwargs) + manifest.pop("extensions") + return manifest + + monkeypatch.setattr(import_service, name, missing_declaration) + service = ImportService() + request = dataclasses.replace(options(tmp_path, focus=False), format=fmt) + with pytest.raises(ImportExecutionError): + service.import_snapshot(request) + backend = service._msinsight_backend if fmt else service._dataset_backend + assert not backend.target_path(request.snapshot_file, request.output_dir).exists() + + +@pytest.mark.parametrize("fmt", [None]) +@pytest.mark.parametrize("mutation", ["missing-table", "frame-content", "declaration"]) +def test_repeat_import_rejects_changed_frames_without_overwriting(tmp_path, fmt, mutation): + service = ImportService() + request = dataclasses.replace(options(tmp_path, focus=False), format=fmt) + first = service.import_snapshot(request) + assert not first.reused + assert service.import_snapshot(request).reused + path = first.db_path / "manifest.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + member = first.db_path / "device_0/slice_00000.db" + if mutation == "declaration": + manifest.pop("extensions") + else: + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute( + MUTATIONS[mutation] + if mutation == "missing-table" + else 'UPDATE pt_snap_frame_value SET frameJson=\'{"name":"changed"}\'' + ) + if fmt is None and mutation == "missing-table": + # Resealed malformed native members must still fail schema validation. + manifest["devices"]["0"]["slices"][0]["sha256"] = hashlib.sha256( + member.read_bytes() + ).hexdigest() + path.write_text(json.dumps(manifest), encoding="utf-8") + before = hashes(first.db_path) + with pytest.raises(ImportExecutionError): + service.import_snapshot(request) + assert hashes(first.db_path) == before + assert not (tmp_path / ".pt-snap").exists() + if fmt is None: + rebuilt = service.import_snapshot(dataclasses.replace(request, force=True)) + assert not rebuilt.reused + assert service.import_snapshot(request).reused diff --git a/tests/core/test_snapshot_cache.py b/tests/core/test_snapshot_cache.py index 0cf6d01..850621e 100644 --- a/tests/core/test_snapshot_cache.py +++ b/tests/core/test_snapshot_cache.py @@ -9,7 +9,6 @@ from pathlib import Path from pt_snap_cli.core.snapshot_cache import ( - _RowEncoder, seal_manifest, source_hashes, validate_cache, @@ -17,20 +16,6 @@ class SnapshotCacheTest(unittest.TestCase): - def test_row_encoder_preserves_digest_bytes_and_bounds_cache(self): - encoder = _RowEncoder() - for stack in ("", "ascii", "\u4e2d\u6587", '"\\\n\t', None, "x" * (8 * 1024 * 1024)): - row = (1, 2, 3, 4, 0, 5, 6, 7, stack) - expected = json.dumps( - row, ensure_ascii=True, separators=(",", ":"), allow_nan=False - ).encode() - self.assertEqual(encoder.encode(row, True), expected) - self.assertEqual(encoder.encode(row, True), expected) - for index in range(3000): - encoder.encode((index, str(index)), True) - self.assertLessEqual(len(encoder.stacks), 2048) - self.assertLessEqual(encoder.size, 8 * 1024 * 1024) - def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) @@ -114,81 +99,6 @@ def open_during_write(*args, **kwargs): with patch.object(snapshot_cache.sqlite3, "connect", side_effect=open_during_write): self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - def test_base_write_after_sidecar_check_is_detected(self): - from unittest.mock import patch - - from pt_snap_cli.core import snapshot_cache - - connect = sqlite3.connect - writers = [] - self.addCleanup(lambda: [writer.close() for writer in writers]) - - def open_after_base_write(*args, **kwargs): - writer = connect(self.db) - writers.append(writer) - writer.execute("PRAGMA journal_mode=WAL") - writer.execute("UPDATE block_0 SET size=size+1") - writer.commit() - return connect(*args, **kwargs) - - with patch.object(snapshot_cache.sqlite3, "connect", side_effect=open_after_base_write): - with self.assertRaisesRegex(ValueError, "parser data"): - validate_cache(self.directory, self.hash) - - def test_consumer_commit_during_row_scan_is_allowed(self): - from unittest.mock import patch - - from pt_snap_cli.core import snapshot_cache - - with closing(sqlite3.connect(self.db)) as writer: - writer.execute("PRAGMA journal_mode=WAL") - encode = snapshot_cache._RowEncoder.encode - committed = False - - def encode_while_writing(encoder, row, inline_stack): - nonlocal committed - if not committed: - writer.execute("CREATE TABLE consumer_cache(value INTEGER)") - writer.commit() - writer.execute("PRAGMA wal_checkpoint(PASSIVE)") - committed = True - return encode(encoder, row, inline_stack) - - with patch.object(snapshot_cache._RowEncoder, "encode", encode_while_writing): - self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - self.assertTrue(committed) - - def test_parser_seal_does_not_override_producer_seal(self): - self.manifest["ptSnap"] = { - "memberHashKind": "parser-v1", - "members": {"slice.db": "0" * 64}, - } - self.save() - with self.assertRaisesRegex(ValueError, "member hash"): - validate_cache(self.directory, self.hash) - - def test_logical_producer_seal_without_parser_seal_is_checked(self): - self.manifest["ptSnap"] = { - "memberHashKind": "parser-v1", - "members": self.manifest.pop("parserCache")["members"], - } - self.save() - self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - with closing(sqlite3.connect(self.db)) as conn: - conn.execute("UPDATE block_0 SET size=size+1") - conn.commit() - with self.assertRaisesRegex(ValueError, "member hash"): - validate_cache(self.directory, self.hash) - - def test_legacy_producer_seal_is_not_overridden_by_parser_seal(self): - self.manifest["ptSnap"] = {"members": {"slice.db": source_hashes(self.db)[0]}} - self.save() - self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - with closing(sqlite3.connect(self.db)) as conn: - conn.execute("CREATE TABLE consumer_extra(value INTEGER)") - with self.assertRaisesRegex(ValueError, "member hash"): - validate_cache(self.directory, self.hash) - def test_legacy_physical_seal_accepts_closed_wal_database(self): with closing(sqlite3.connect(self.db)) as conn: conn.execute("PRAGMA journal_mode=WAL") @@ -197,22 +107,11 @@ def test_legacy_physical_seal_accepts_closed_wal_database(self): self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - def test_legacy_physical_seal_rejects_committed_wal(self): - with closing(sqlite3.connect(self.db)) as conn: - conn.execute("PRAGMA journal_mode=WAL") - self.manifest["ptSnap"] = {"members": {"slice.db": source_hashes(self.db)[0]}} - self.save() - with closing(sqlite3.connect(self.db)) as writer: - writer.execute("CREATE TABLE consumer_cache(value INTEGER)") - writer.commit() - with self.assertRaisesRegex(ValueError, "sidecars"): - validate_cache(self.directory, self.hash) - def test_source_hash_matches_shared_salt(self): raw, salted = source_hashes(self.source) self.assertEqual(raw, hashlib.sha256(self.source.read_bytes()).hexdigest()) self.assertEqual( - salted, hashlib.sha256(b"mem_snapshot_parser_v2" + self.source.read_bytes()).hexdigest() + salted, hashlib.sha256(b"mem_snapshot_parser_v3" + self.source.read_bytes()).hexdigest() ) def test_extra_tables_indexes_and_dictionary_do_not_invalidate(self): @@ -233,20 +132,6 @@ def test_extra_tables_indexes_and_dictionary_do_not_invalidate(self): conn.execute("VACUUM") self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - def test_modified_base_data_is_rejected(self): - for table, assignment in ( - ("block_0", "size=size+1"), - ("trace_entry_0", "allocated=allocated+1"), - ("dictionary", "\"value\"='changed'"), - ): - with self.subTest(table=table): - with closing(sqlite3.connect(self.db)) as conn, conn: - conn.execute(f"UPDATE {table} SET {assignment}") - with self.assertRaises(ValueError): - validate_cache(self.directory, self.hash) - seal_manifest(self.directory, self.manifest) - self.save() - def test_changed_source_is_rejected(self): self.source.write_bytes(b"changed") with self.assertRaises(ValueError): @@ -265,15 +150,6 @@ def test_rollback_journal_is_rejected(self): with self.assertRaisesRegex(ValueError, "rollback journal"): validate_cache(self.directory, self.hash) - def test_active_wal_is_read_instead_of_ignoring_committed_rows(self): - with closing(sqlite3.connect(self.db)) as conn: - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("UPDATE block_0 SET size=size+1") - conn.commit() - self.assertTrue(Path(str(self.db) + "-wal").exists()) - with self.assertRaises(ValueError): - validate_cache(self.directory, self.hash) - def test_missing_or_corrupt_db_is_rejected_without_creating_it(self): self.db.unlink() with self.assertRaises(ValueError): @@ -369,51 +245,79 @@ def change_manifest(directory, manifest, **kwargs): with self.assertRaisesRegex(ValueError, "manifest changed"): validate_cache(self.directory, self.hash) - def test_parser_salt_changes_cache_identity(self): - from unittest.mock import patch + def test_v2_cache_is_not_reused_by_v3_parser(self): + self.manifest["cacheHash"] = hashlib.sha256( + b"mem_snapshot_parser_v2" + self.source.read_bytes() + ).hexdigest() + self.save() + with self.assertRaisesRegex(ValueError, "source hash or parser salt changed"): + validate_cache(self.directory, source_hashes(self.source)[1]) - from pt_snap_cli.core import snapshot_cache + def test_deep_manifest_is_a_validation_error(self): + (self.directory / "manifest.json").write_text("[" * 200000 + "0" + "]" * 200000) + with self.assertRaisesRegex(ValueError, "nesting"): + validate_cache(self.directory, self.hash) - with patch.object(snapshot_cache, "PARSER_HASH_SALT", b"another-parser-version"): - with self.assertRaises(ValueError): - validate_cache(self.directory, source_hashes(self.source)[1]) + def test_cross_window_lifetime_references_remain_valid(self): + with closing(sqlite3.connect(self.db)) as conn, conn: + conn.execute("UPDATE block_0 SET allocEventId=-1, freeEventId=999") + seal_manifest(self.directory, self.manifest) + self.save() + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - def test_corrupt_secondary_index_is_rejected(self): + def test_default_validation_rejects_missing_required_table(self): with closing(sqlite3.connect(self.db)) as conn: - conn.execute("CREATE INDEX block_address ON block_0(address)") - page_size = conn.execute("PRAGMA page_size").fetchone()[0] - root = conn.execute( - "SELECT rootpage FROM sqlite_master WHERE name='block_address'" - ).fetchone()[0] - with self.db.open("r+b") as file: - file.seek((root - 1) * page_size) - file.write(b"\xff" * page_size) - with self.assertRaises((ValueError, sqlite3.DatabaseError)): + conn.execute("DROP TABLE block_0") + conn.commit() + with self.assertRaisesRegex(ValueError, "Missing parser table"): validate_cache(self.directory, self.hash) - def test_deep_manifest_is_a_validation_error(self): - (self.directory / "manifest.json").write_text("[" * 200000 + "0" + "]" * 200000) - with self.assertRaisesRegex(ValueError, "nesting"): + def test_default_validation_rejects_missing_member(self): + self.db.unlink() + with self.assertRaises(ValueError): validate_cache(self.directory, self.hash) + self.assertFalse(self.db.exists()) - def test_legacy_dictionary_references_are_required(self): - del self.manifest["parserCache"] - for table in ("trace_entry_0", "block_0"): - with self.subTest(table=table): - with closing(sqlite3.connect(self.db)) as conn, conn: - original = conn.execute( - 'SELECT * FROM dictionary WHERE "table"=?', (table,) - ).fetchall() - conn.execute('DELETE FROM dictionary WHERE "table"=?', (table,)) - self.save() - with self.assertRaisesRegex(ValueError, "dictionary reference"): - validate_cache(self.directory, self.hash) - with closing(sqlite3.connect(self.db)) as conn, conn: - conn.executemany("INSERT INTO dictionary VALUES(?,?,?,?)", original) + def test_default_validation_rejects_unreadable_database(self): + self.db.write_bytes(b"not a database") + with self.assertRaises(sqlite3.DatabaseError): + validate_cache(self.directory, self.hash) - def test_cross_window_lifetime_references_remain_valid(self): + def test_oom_without_dictionary_reference_is_accepted(self): with closing(sqlite3.connect(self.db)) as conn, conn: - conn.execute("UPDATE block_0 SET allocEventId=-1, freeEventId=999") + conn.execute("UPDATE trace_entry_0 SET action='oom'") seal_manifest(self.directory, self.manifest) self.save() self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) + + def test_cache_validation_never_scans_rows_or_pages(self): + from unittest.mock import patch + + statements = [] + connect = sqlite3.connect + + def traced(*args, **kwargs): + conn = connect(*args, **kwargs) + conn.set_trace_callback(statements.append) + return conn + + with patch.object(sqlite3, "connect", side_effect=traced): + seal_manifest(self.directory, self.manifest) + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) + sql = "\n".join(statements).upper() + self.assertNotIn("QUICK_CHECK", sql) + self.assertNotIn("INTEGRITY_CHECK", sql) + self.assertNotIn("DISTINCT", sql) + self.assertNotIn( + "ORDER BY", + sql.replace("ORDER BY ID DESC LIMIT 1", "").replace("ORDER BY ID LIMIT 1", ""), + ) + + def test_legacy_content_digests_are_not_compared(self): + self.manifest["parserCache"]["members"] = {"slice.db": "old-content-digest"} + self.manifest["ptSnap"] = { + "memberHashKind": "parser-v1", + "members": {"slice.db": "different-old-digest"}, + } + self.save() + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) diff --git a/tests/core/wheel_scale_acceptance.py b/tests/core/wheel_scale_acceptance.py index 6cdd38c..4313f14 100644 --- a/tests/core/wheel_scale_acceptance.py +++ b/tests/core/wheel_scale_acceptance.py @@ -111,6 +111,7 @@ def __init__(self, root, wheel, evidence_dir, wheelhouse=None): PATH=f"{bin_dir}{os.pathsep}{self.env.get('PATH', '')}", PWD=str(self.cwd), PYTHONNOUSERSITE="1", + PYTHONUTF8="1", PIP_DISABLE_PIP_VERSION_CHECK="1", NO_COLOR="1", ) @@ -142,7 +143,13 @@ def __init__(self, root, wheel, evidence_dir, wheelhouse=None): def run(self, label, command, *, expected=0): started = time.perf_counter() result = subprocess.run( - command, cwd=self.cwd, env=self.env, capture_output=True, text=True, check=False + command, + cwd=self.cwd, + env=self.env, + capture_output=True, + text=True, + encoding="utf-8", + check=False, ) self.commands.append( { diff --git a/tests/query/test_executor.py b/tests/query/test_executor.py index 35970d5..1d3c53f 100644 --- a/tests/query/test_executor.py +++ b/tests/query/test_executor.py @@ -15,6 +15,43 @@ ) +@pytest.mark.parametrize("method", ["execute", "count"]) +def test_sqlite_cancellation_without_timeout_clears_progress_handler(tmp_path, method): + path = tmp_path / "cancel.db" + with closing(sqlite3.connect(path)) as conn: + conn.execute("CREATE TABLE dictionary (id INTEGER PRIMARY KEY, value TEXT)") + conn.commit() + context = Context(path, persistent=True) + try: + executor = QueryExecutor(context) + checks = 0 + + def cancelled(): + nonlocal checks + checks += 1 + return checks >= 3 + + sql = ( + "WITH RECURSIVE n(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM n WHERE x<100000000) " + "SELECT SUM(x) AS value FROM n" + ) + with pytest.raises(QueryExecutionError, match="interrupted"): + getattr(executor, method)(sql, cancelled=cancelled) + assert checks == 3 + assert executor.execute("SELECT 42 AS value") == [{"value": 42}] + finally: + context.close() + + +def test_executor_clone_keeps_runtime_templates_without_sharing_registrations(): + first = QueryExecutor(context=None) + first.register_template(QueryTemplate(name="custom", query="SELECT 1", parameters={})) + worker = first.clone_for_context(None) + assert worker.has_runtime_template("custom") + worker.register_template(QueryTemplate(name="worker_only", query="SELECT 2", parameters={})) + assert not first.has_runtime_template("worker_only") + + class TestQueryExecutor: def test_render_simple_template(self): template = QueryTemplate( diff --git a/tests/query/test_frame_tree.py b/tests/query/test_frame_tree.py index 98e5b82..e10b4de 100644 --- a/tests/query/test_frame_tree.py +++ b/tests/query/test_frame_tree.py @@ -182,7 +182,7 @@ def test_inline_text_database_retains_queries_but_cannot_supply_original_frames( assert query(path, "event", {"id": 1}).rows[0]["callstack"] == "train.py:10" -@pytest.mark.parametrize("name", ["event_frames", "active_memory_frame_tree_at_event"]) +@pytest.mark.parametrize("name", ["event_frames"]) @pytest.mark.parametrize("slice_index", [None, 0]) def test_dataset_root_never_implicitly_selects_a_shard(tmp_path: Path, name: str, slice_index): from pt_snap_cli.core.dataset_support import DATASET_SUPPORT diff --git a/tests/test_clean_wheel_agent_acceptance.py b/tests/test_clean_wheel_agent_acceptance.py index 5dd3141..d320960 100644 --- a/tests/test_clean_wheel_agent_acceptance.py +++ b/tests/test_clean_wheel_agent_acceptance.py @@ -324,3 +324,79 @@ def cli(command, **kwargs): ) assert [command[1] for command in commands] == ["import", "import", "overview"] assert focus.read_text() == "unchanged" + + +def test_real_two_shard_cli_passes_comparator_with_additional_frame_evidence( + acceptance, tmp_path, monkeypatch +): + from typer.testing import CliRunner + + from pt_snap_cli.cli import app + from tests.snapshot.test_sharded_replay import lifecycle_data, write_source + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + data = lifecycle_data() + data["device_traces"][0] = [e for e in data["device_traces"][0] if e["action"] != "oom"] + source = write_source(tmp_path, data) + runner = CliRunner() + + def cli(command, **kwargs): + result = runner.invoke(app, command[1:]) + assert result.exit_code == 0, result.output + return json.loads(result.stdout) + + monkeypatch.setattr(acceptance, "_run_json", cli) + standalone = cli(["pt-snap", "import", str(source), "--json"])["db_path"] + overview = cli(["pt-snap", "overview", standalone, "--json"]) + peak = cli(["pt-snap", "query", standalone, "--template-use", "memory_peak", "--json"]) + evidence = acceptance._sharded_acceptance( + Path("pt-snap"), + source, + tmp_path / "datasets", + 2, + Path(standalone), + overview, + peak, + tmp_path / ".pt-snap/focus.json", + cwd=tmp_path, + env={}, + ) + assert len(evidence) == 2 + assert all(item["cross_shard_event_parity"] and item["slice_count"] >= 2 for item in evidence) + + +@pytest.mark.parametrize( + "damage", ["event-value", "missing-field", "missing-evidence", "frame-text"] +) +def test_event_comparator_does_not_hide_base_or_frame_corruption(acceptance, damage): + import copy + + event = { + "id": 0, + "action": 4, + "address": 100, + "size": 10, + "stream": 0, + "allocated": 10, + "active": 10, + "reserved": 16, + "callstack": "x.py:1 f", + } + reference = [event, {**event, "id": 1}] + crossing = [ + dict(row, frames_status="ordered", frames=[{"filename": "x.py", "line": 1, "name": "f"}]) + for row in copy.deepcopy(reference) + ] + acceptance._event_parity(crossing, reference) + if damage == "event-value": + crossing[0]["size"] += 1 + elif damage == "missing-field": + del crossing[0]["size"] + elif damage == "missing-evidence": + crossing[0]["frames"] = None + else: + crossing[0]["frames"][0]["name"] = "changed" + with pytest.raises(acceptance.AcceptanceError): + acceptance._event_parity(crossing, reference) diff --git a/tests/test_contract_cli_api.py b/tests/test_contract_cli_api.py index 4662d47..8f02699 100644 --- a/tests/test_contract_cli_api.py +++ b/tests/test_contract_cli_api.py @@ -24,7 +24,8 @@ from pt_snap_cli.cli import app from pt_snap_cli.core import InvalidDeviceError, QueryExecutionError, TemplateRenderError from pt_snap_cli.core.dataset_sources import QueryBudget -from pt_snap_cli.core.errors import QueryTimeoutError +from pt_snap_cli.core.error_codes import classify_error +from pt_snap_cli.core.errors import QueryCancelledError, QueryTimeoutError from pt_snap_cli.core.json_codec import JSON_SCHEMA_VERSION from pt_snap_cli.core.query_service import QueryService from pt_snap_cli.core.report_service import ReportService @@ -37,6 +38,36 @@ pytestmark = pytest.mark.usefixtures("owned_service_instances") +def test_cancelled_query_has_a_distinct_non_timeout_error_code(tmp_path, monkeypatch): + error = QueryCancelledError("Frame tree query cancelled by a newer request or reset.") + assert isinstance(error, QueryExecutionError) + assert classify_error(error)[0] == "QUERY_CANCELLED" + root = make_dataset(tmp_path / "cancelled", devices=(0,)) + + def cancelled_tree(*args, **kwargs): + raise error + + monkeypatch.setattr("pt_snap_cli.core.query_service.frame_tree", cancelled_tree) + with SnapshotAnalyzer(root) as analyzer: + with pytest.raises(QueryCancelledError): + analyzer.execute_query("active_memory_frame_tree_at_event", {"event_id": 1}) + result = runner.invoke( + app, + [ + "query", + str(root), + "--template-use", + "active_memory_frame_tree_at_event", + "--params", + '{"event_id":1}', + "--json", + ], + ) + assert result.exit_code == 1 + assert result.stdout == "" + assert json.loads(result.stderr)["error"]["code"] == "QUERY_CANCELLED" + + @pytest.mark.parametrize("template", ["event", "block"]) @pytest.mark.parametrize("params", [{"min_size": 2**63}, {"offset": 2**63, "limit": 1}]) def test_dataset_int64_overflow_cli_api_domain_error(tmp_path, template, params): diff --git a/tests/test_dataset_import_baseline.py b/tests/test_dataset_import_baseline.py index 2df08af..eb330e2 100644 --- a/tests/test_dataset_import_baseline.py +++ b/tests/test_dataset_import_baseline.py @@ -151,8 +151,21 @@ def test_reuse_miss_refuses_rebuild_and_preserves_existing_actual_bytes(tmp_path def test_recorded_performance_report_is_structural_not_a_speed_gate(): - from pt_snap_cli.core.dataset_support import DATASET_SUPPORT - + # This is historical evidence tied to report["source_head"], not a claim + # that queries introduced later were measured by that recorded run. + recorded_templates = { + "event", + "allocation", + "memory_peak", + "allocator_gap", + "active_blocks_at_event", + "active_memory_callstack_at_event", + "preexisting_live", + "block", + "freed_block_lifetime", + "leak_detection", + "callstack_analysis", + } repository = Path(__file__).resolve().parents[1] raw = (repository / "docs/dataset-performance-baseline.json").read_text() report = json.loads(raw) @@ -167,7 +180,7 @@ def test_recorded_performance_report_is_structural_not_a_speed_gate(): for record in report["records"]: assert record["exit"] == 0 and record["phase_rss"]["status"] == "unavailable" assert "RUSAGE_SELF" in record["rss_scope"] - assert set(record["queries"]) == set(DATASET_SUPPORT) + assert set(record["queries"]) == recorded_templates assert record["individual_shard_bytes"] and record["total_disk_bytes"] > 0 assert all(q["status"] == "passed" for q in record["queries"].values()) # Deliberately no timing/RSS/speedup threshold: these are observations. diff --git a/tests/test_interop_acceptance.py b/tests/test_interop_acceptance.py index ec050a7..6a7b204 100644 --- a/tests/test_interop_acceptance.py +++ b/tests/test_interop_acceptance.py @@ -40,10 +40,12 @@ def isolate(tmp_path, monkeypatch): @pytest.mark.parametrize("device", [0, 2]) -def test_all_eleven_contracts_against_independent_physical_reference(tmp_path, device): +def test_text_contracts_against_independent_physical_reference(tmp_path, device): root = case(tmp_path) result = compare_artifact(root, device, tmp_path / "reference.db") - assert set(result["templates"]) == set(DATASET_SUPPORT) + # This independent physical reference has text stacks only. Frame paths + # have separate real-import differential coverage against a structured DB. + assert set(result["templates"]) == set(DATASET_SUPPORT) - {"active_memory_frame_tree_at_event"} assert result["real_events"] == 16 and result["canonical_lifecycles"] == 10 assert result["raw_block_rows"] == 61 and result["repeated_block_rows"] == 51 assert result["GUI"] == "human-deferred/pending/not-run" @@ -166,9 +168,15 @@ def test_compatible_owned_cache_reuse_and_external_force_preserve_actual_bytes(t reused = service.import_snapshot(dataclasses.replace(request, force=True)) assert reused.reused and reused.metadata is None assert inventory(result.db_path) == external - # Invalid external artifacts must still be preserved, including under force. + # Lightweight reuse accepts content edits and must preserve the edited bytes. with closing(sqlite3.connect(result.db_path / "device_0/slice_00000.db")) as conn, conn: conn.execute("UPDATE block_0 SET size=size+1") + edited = inventory(result.db_path) + assert service.import_snapshot(dataclasses.replace(request, force=True)).reused + assert inventory(result.db_path) == edited + # Structural damage must still be rejected without replacing the artifact. + with closing(sqlite3.connect(result.db_path / "device_0/slice_00000.db")) as conn, conn: + conn.execute("DROP TABLE block_0") invalid = inventory(result.db_path) with pytest.raises(ImportExecutionError, match="preserv"): service.import_snapshot(dataclasses.replace(request, force=True)) diff --git a/tests/test_interop_three_shard.py b/tests/test_interop_three_shard.py index d8d6229..14a6cf6 100644 --- a/tests/test_interop_three_shard.py +++ b/tests/test_interop_three_shard.py @@ -184,7 +184,7 @@ def test_true_cli_import_keeps_one_event_per_owner_and_discovers_requested(impor if fmt == "msinsight": assert ( manifest["cacheHash"] - == hashlib.sha256(b"mem_snapshot_parser_v2" + source.read_bytes()).hexdigest() + == hashlib.sha256(b"mem_snapshot_parser_v3" + source.read_bytes()).hexdigest() ) assert {p: digest(p) for p in before} == before