diff --git a/.github/scripts/clean_wheel_agent_acceptance.py b/.github/scripts/clean_wheel_agent_acceptance.py index cd9b85d..a1fc459 100644 --- a/.github/scripts/clean_wheel_agent_acceptance.py +++ b/.github/scripts/clean_wheel_agent_acceptance.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import hashlib import json import os import subprocess @@ -197,6 +198,190 @@ def _dataset_query( return scope +EVENT_FIELDS = ( + "id", + "action", + "address", + "size", + "stream", + "allocated", + "active", + "reserved", + "callstack", +) +EVENT_SOURCE_FIELDS = ( + "source_stack_id", + "stack_id", + "stack_kind", + "text_kind", + "stack_event_id", + "frames", + "frames_status", +) +FRAME_FIELDS = ("filename", "line", "name") + + +def _standalone_rows( + query: dict[str, Any], standalone: Path, device: int, template: str +) -> list[dict[str, Any]]: + label = f"Standalone {template} reference" + _require(_path_value(query, "db_path", label) == standalone, f"{label} changed target") + _require(query.get("focus_source") == "explicit", f"{label} did not use explicit target") + _require( + type(query.get("device_id")) is int and query["device_id"] == device, + f"{label} changed device", + ) + _require(query.get("template") == template, f"{label} changed template") + rows = query.get("rows") + _require( + isinstance(rows, list) and all(isinstance(row, dict) for row in rows), + f"{label} has invalid rows", + ) + _require( + type(query.get("returned")) is int + and query["returned"] == len(rows) + and type(query.get("total")) is int + and query["total"] == len(rows) + and query.get("total_is_exact") is True + and query.get("has_more") is False + and query.get("truncated") is False, + f"{label} is incomplete", + ) + return rows + + +def _event_parity( + crossing: dict[str, Any], + reference: dict[str, Any], + frame_references: dict[int, dict[str, Any]], + *, + standalone: Path, + fingerprint: str, + device: int, + event_ids: list[int], +) -> int: + """Check base events and ordered public frame fields, not local stack IDs. + + Standalone event_frames exposes filename/line/name, not arbitrary raw extras. + The full raw-array digest below attests internal identity consistency only; + independent extra-field preservation remains covered by the core tests. + """ + expected = _standalone_rows(reference, standalone, device, "event") + rows = crossing.get("rows") + _require( + crossing.get("template") == "event" + and isinstance(rows, list) + and len(rows) == len(expected) == len(event_ids) + and set(frame_references) == set(event_ids), + "Cross-shard event/reference counts or template differ", + ) + _require( + type(crossing.get("returned")) is int + and crossing["returned"] == len(rows) + and type(crossing.get("total")) is int + and crossing["total"] == len(rows) + and crossing.get("total_is_exact") is True + and crossing.get("has_more") is False + and crossing.get("truncated") is False, + "Cross-shard event rows are incomplete", + ) + coverage = crossing.get("scope", {}).get("source_coverage", {}) + _require(isinstance(coverage, dict), "Cross-shard source coverage is invalid") + for field, value in { + "range_complete": True, + "event_position": "after", + "coverage_scope": "returned_events", + "events_resolved": len(event_ids), + "ordered_frame_events": len(event_ids), + }.items(): + _require( + type(coverage.get(field)) is type(value) and coverage[field] == value, + f"Cross-shard source coverage differs: {field}", + ) + _require( + type(coverage.get("source_queries")) is int and coverage["source_queries"] > 0, + "Cross-shard source coverage differs: source_queries", + ) + frame_count = 0 + for row, original, event_id in zip(rows, expected, event_ids, strict=True): + label = f"Event {event_id}" + _require(set(original) == set(EVENT_FIELDS), f"{label} standalone base fields differ") + _require( + isinstance(row, dict) and set(row) == set(EVENT_FIELDS + EVENT_SOURCE_FIELDS), + f"{label} dataset event fields differ", + ) + _require( + type(original["id"]) is int and original["id"] == event_id, + f"{label} standalone event order differs", + ) + for field in EVENT_FIELDS: + _require( + type(row[field]) is type(original[field]) and row[field] == original[field], + f"{label} base field differs: {field}", + ) + frames = row["frames"] + _require( + isinstance(frames, list) + and all(isinstance(frame, dict) for frame in frames) + and row["frames_status"] == "ordered" + and row["stack_kind"] == "captured", + f"{label} lost ordered frame evidence", + ) + _require( + row["text_kind"] == ("captured" if original["callstack"] else "missing") + and type(row["stack_event_id"]) is int + and row["stack_event_id"] == event_id, + f"{label} source attribution differs", + ) + try: + canonical = json.dumps( + frames, sort_keys=True, ensure_ascii=True, separators=(",", ":"), allow_nan=False + ) + except (TypeError, ValueError, RecursionError, OverflowError) as exc: + raise AcceptanceError(f"{label} has invalid raw frame JSON") from exc + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + identity = f"dataset:{fingerprint}:device:{device}:frames:sha256:{digest}" + _require( + row["source_stack_id"] == row["stack_id"] == identity, + f"{label} ordered frame identity/namespace differs", + ) + frame_rows = _standalone_rows( + frame_references[event_id], standalone, device, "event_frames" + ) + _require(len(frames) == len(frame_rows), f"{label} ordered frame count differs") + local_stack_id = None + for position, (frame, original_frame) in enumerate(zip(frames, frame_rows, strict=True)): + _require( + set(original_frame) + == set(FRAME_FIELDS + ("event_id", "stack_id", "position", "frame_id")), + f"{label} standalone frame fields differ", + ) + _require( + type(original_frame["event_id"]) is int + and original_frame["event_id"] == event_id + and type(original_frame["position"]) is int + and original_frame["position"] == position, + f"{label} standalone frame event/order differs at position {position}", + ) + _require( + all( + type(original_frame[key]) is int and original_frame[key] >= 0 + for key in ("stack_id", "frame_id") + ) + and local_stack_id in (None, original_frame["stack_id"]), + f"{label} standalone frame local identity differs at position {position}", + ) + local_stack_id = original_frame["stack_id"] + for field, field_type in (("filename", str), ("line", int), ("name", str)): + _require( + type(frame.get(field)) is type(original_frame[field]) is field_type + and frame[field] == original_frame[field], + f"{label} ordered frame field differs at position {position}: {field}", + ) + frame_count += len(frames) + return frame_count + + def _sharded_acceptance( pt_snap: Path, fixture: Path, @@ -358,10 +543,89 @@ def cli(label: str, *args: object) -> dict[str, Any]: scope.get("slice_indices") == [slices[0]["index"], slices[1]["index"]], "Event range did not cross shards", ) - _require( - crossing["rows"] == reference["rows"] and len(crossing["rows"]) == 2, - "Cross-shard real events differ from standalone", + + def check_events(actual, original, event_ids, *, name=format_name, generation=fingerprint): + frame_references = { + event_id: cli( + f"{name} standalone frames {event_id}", + "query", + standalone, + "--device", + device, + "--template-use", + "event_frames", + "--params", + json.dumps({"event_id": event_id, "frame_order": "original"}), + "--exact-total", + ) + for event_id in event_ids + } + return _event_parity( + actual, + original, + frame_references, + standalone=standalone, + fingerprint=generation, + device=device, + event_ids=event_ids, + ) + + frame_count = check_events( + crossing, reference, [slices[0]["last_event_id"], slices[1]["first_event_id"]] ) + if frame_count == 0: + # A shard boundary can legitimately fall between two empty stacks. + # Select one real captured event, then prove its frames independently; + # formatted text is only a locator, never reconstructed frame evidence. + candidate = cli( + f"{format_name} captured event locator", + "query", + standalone, + "--device", + device, + "--template-use", + "event", + "--params", + json.dumps({"min_id": 0, "order_by": "callstack", "order_dir": "DESC", "limit": 1}), + ) + candidates = candidate.get("rows") + _require( + _path_value(candidate, "db_path", "captured event locator") == standalone + and candidate.get("focus_source") == "explicit" + and type(candidate.get("device_id")) is int + and candidate["device_id"] == device + and candidate.get("template") == "event" + and type(candidate.get("returned")) is int + and candidate["returned"] == 1 + and isinstance(candidates, list) + and len(candidates) == 1 + and isinstance(candidates[0], dict), + "Invalid captured event locator", + ) + event_id = candidates[0].get("id") + _require( + type(event_id) is int + and slices[0]["first_event_id"] <= event_id <= slices[-1]["last_event_id"] + and isinstance(candidates[0].get("callstack"), str) + and bool(candidates[0]["callstack"]), + "Fixture has no real captured event for nonempty frame acceptance", + ) + point_args = ( + "--device", + device, + "--template-use", + "event", + "--params", + json.dumps({"id": event_id}), + "--exact-total", + ) + captured = cli(f"{format_name} captured event", "query", dataset, *point_args) + original = cli( + f"{format_name} standalone captured event", "query", standalone, *point_args + ) + _dataset_query(captured, dataset, dataset, fingerprint, device) + frame_count = check_events(captured, original, [event_id]) + _require(frame_count > 0, "Ordered frame acceptance has no nonempty frame evidence") _require( crossing.get("total_is_exact") is True and crossing.get("total") == 2, "Cross-shard exact total is invalid", @@ -380,6 +644,9 @@ def cli(label: str, *args: object) -> dict[str, Any]: "reused": True, "standalone_peak_parity": True, "cross_shard_event_parity": True, + "ordered_frame_field_parity": True, + "ordered_frame_identity_checked": True, + "nonempty_ordered_frames_checked": True, "focus_unchanged": True, } ) diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index ab7d111..b6277ad 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -49,12 +49,11 @@ The required installed-wheel case creates a clean venv without system packages and runs its actual `pt-snap` console script from isolated HOME/CWD. Its trusted, test-owned input has exactly 1,000,000 events; the default exporter produces two full 500,000-event shards. All installed catalog entries are classified: currently -11 dataset-supported templates succeed with installed CLI/API equivalence and -independent known-result checks, while `event_frames` and -`active_memory_frame_tree_at_event` return explicit unsupported-dataset errors. -Both report commands run: `peak-memory` succeeds and `memory-tree` preserves the -corresponding dataset rejection. The allocation/free-request/free-completion -boundary assertions remain covered, alongside static occupancy, offset pages, +12 dataset-supported templates succeed with installed CLI/API equivalence and +independent known-result checks; `event_frames` remains the explicit +unsupported-dataset query. Both `peak-memory` and `memory-tree` reports succeed. +The allocation/free-request/free-completion boundary assertions remain covered, +alongside static occupancy, offset pages, terminal survivors and unchanged source/manifest/member hashes. A second required nightly case separately imports 500,002 events and proves the original final short shard has exactly two real events, with the crossing completed lifetime diff --git a/README.md b/README.md index 147b326..97b3289 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,13 @@ byte units, allocated/active/reserved counters, expected output and partial repo - [Replayable pickle splitting](docs/en/splitting.md): one split strategy and a new output directory; separate from dataset import. - [Complete-dataset addressing](docs/en/sharded-snapshotdb.md#complete-dataset-focus-and-addressing-p1), [global built-in support](docs/en/querying.md#dataset-global-built-in-support-p3), and [recorded interoperability acceptance](docs/en/interop-acceptance.md): advanced contracts and evidence, not prerequisites for a first report. +Native reuse retains content verification and disposable validation receipts. +Compatible reuse checks source identity, complete member inventory, required +schema and event endpoints, not database bytes or arbitrary row values; queries +still perform independent admission. Parser-v2 caches require a new output +directory for parser-v3 export, never replacement. See the +[cache and preservation contract](docs/en/sharded-snapshotdb.md). + ## Built-in Query Templates & Reports Use `pt-snap query --list` or `pt-snap capabilities --json` to inspect all 13 built-in templates and their parameter/output schemas: @@ -168,8 +175,8 @@ Peak reports expose attribution completeness and same-event active-byte coverage group percentages use included bytes after filtering and ranking. See the [report guide](docs/en/querying.md#report-command) before interpreting a capped breakdown. -Standalone imports preserve ordered structured callstack frames. Use `event_frames` to -inspect them, or `report memory-tree --event-id 100 --format html > memory-tree.html` +Standalone imports and new native/compatible datasets preserve ordered frames. +Use standalone `event_frames` (dataset `event` exposes `frames`), or `report memory-tree --event-id 100 --format html > memory-tree.html` to explore event-time occupancy by caller path. See [frame memory breakdown](docs/en/querying.md#frame-memory-breakdown); older text-only databases require re-importing the original snapshot. @@ -182,6 +189,9 @@ Defaults remain 100000 work rows / 64 MiB work bytes / 512 MiB scratch pages; se [public dataset limits](docs/en/querying.md#public-dataset-limits) for precedence, deep-pagination costs and all-or-error semantics. +Concurrent frame-tree queries on one Python analyzer use the latest request; +superseded queries raise `QueryCancelledError`. See [API cancellation semantics](docs/en/snapshot-analyzer-api.md#errors-and-scope). + ## Agent Skills Install bundled diagnostic workflows with `pt-snap skill install`. Agent integration uses skills plus the CLI; there is no MCP server. See the [Agent Skills guide](docs/en/skills.md). diff --git a/README_zh.md b/README_zh.md index 3e824c1..df39282 100644 --- a/README_zh.md +++ b/README_zh.md @@ -121,6 +121,11 @@ allocated/active/reserved 指标、预期输出和报告不完整的情况。 - [可回放 pickle 拆分](docs/zh/splitting.md)：只选一种拆分策略并使用新目录，与数据集导入不同。 - [完整数据集寻址](docs/zh/sharded-snapshotdb.md#完整数据集-focus-与寻址p1)、[全局内建支持](docs/zh/querying.md#数据集全局内建支持p3)和[已记录的互操作验收](docs/zh/interop-acceptance.md)：按需查看高级合同与证据，不是第一份报告的前置条件。 +原生复用保留内容校验与可丢弃的校验记录。兼容复用检查源身份、完整成员清单、必需 +schema 和事件端点，不验证数据库字节或任意行值；查询仍独立进行准入校验。 +parser-v2 缓存须换新输出目录导出 parser-v3，不会被替换。详见 +[缓存及保留合同](docs/zh/sharded-snapshotdb.md)。 + ## 内置查询模板与报告 可执行 `pt-snap query --list` 或 `pt-snap capabilities --json` 查看全部 13 个内置查询模板及其参数与输出契约： @@ -165,7 +170,8 @@ allocated/active/reserved 指标、预期输出和报告不完整的情况。 included bytes 为分母。解读有上限的分组结果前，请参阅 [报告指南](docs/zh/querying.md#report-命令)。 -单库导入时保留结构化调用栈帧及原始顺序。用 `event_frames` 查看帧，或执行 +单库及新导入的原生/兼容分片数据集均保留去重有序帧。单库用 `event_frames`、 +数据集用 `event` 的 `frames` 查看帧，或执行 `report memory-tree --event-id 100 --format html > memory-tree.html` 按调用路径查看事件时刻的占用。详见[调用栈帧内存拆解](docs/zh/querying.md#调用栈帧内存拆解)； 旧文本数据库需要从原始快照重新导入。 @@ -177,6 +183,9 @@ included bytes 为分母。解读有上限的分组结果前，请参阅 默认仍为 100000 工作行 / 64 MiB 工作字节 / 512 MiB scratch 页；优先级、 深分页成本和整次失败语义见[公开数据集预算](docs/zh/querying.md#公开数据集预算)。 +同一 Python analyzer 的并发帧树查询以最新请求为准，旧请求抛出 `QueryCancelledError`。 +详见 [API 取消语义](docs/zh/snapshot-analyzer-api.md#错误与范围)。 + ## Agent Skills 使用 `pt-snap skill install` 安装随包诊断工作流。Agent 集成入口是 skills 与 CLI，不再提供 MCP 服务器。详见 [Agent Skills 指南](docs/zh/skills.md)。 diff --git a/docs/en/database.md b/docs/en/database.md index 212ff13..cc248a1 100644 --- a/docs/en/database.md +++ b/docs/en/database.md @@ -381,8 +381,8 @@ format and rebuilds the cache; read-only analysis never migrates a database. ### Standalone frame capability boundary Format 3 applies only to standalone imports. Native sharded imports remain format 2 -and compatible exports remain format 1; neither claims the standalone frame-tree -capability. `callstack_frame_manifest(callstackId INTEGER PRIMARY KEY, +and compatible exports remain format 1. A separate version-2 deduplicated frame +extension supports dataset trees; see [querying](querying.md#deduplicated-dataset-frame-extension-version-2). `callstack_frame_manifest(callstackId INTEGER PRIMARY KEY, frameCount INTEGER NOT NULL)` records the expected length of every captured stack, including zero. Frame fields must be text filename/name and an integer line. `event_frames` returns no rows for absent, known-empty or corrupt arrays, never a @@ -468,9 +468,8 @@ P0 compatibility-v1 table order, inline layout and manifest semantics are unchan `trace_entry_` tables with the ninth column `callstack TEXT`, not views, with the exact positional block/dictionary schema. It removes staging's unprefixed `callstack` table and writes `pt_snap_metadata.import_format_version=1`, retaining -isolated `pt_snap_block_reference` text lifecycle sources. Native standalone and -native datasets keep their interning and v2 metadata. Structured ordered frames -are not emitted or inferred from text. +isolated `pt_snap_block_reference` text lifecycle sources. Native datasets keep their text interning and v2 metadata, plus the separate +version-2 deduplicated frame extension; standalone imports use format 3. Frames are never inferred from text. Inline stack bytes scale with the sum of stack text **over every event**, not unique stacks; conversion temporarily holds native and inline tables and may leave diff --git a/docs/en/dataset-acceptance.md b/docs/en/dataset-acceptance.md index 1ca5143..443f295 100644 --- a/docs/en/dataset-acceptance.md +++ b/docs/en/dataset-acceptance.md @@ -9,10 +9,10 @@ fixture is not evidence that a producer, collector, or exporter ran. compatibility-v1 and native-v2 layouts; the nightly profile extends the same bank to 200 seeds. The original eight regression seeds and twenty query shapes remain. Each seed now runs 69–73 query cases per layout (2,896 paired comparisons for the -20-seed bank; 28,864 for 200), covering all eleven dataset-supported templates. -The two standalone-only templates, `event_frames` and -`active_memory_frame_tree_at_event`, have separate explicit rejection tests and -never count as successful differential comparisons. +20-seed bank; 28,864 for 200), covering eleven text-query contracts. Dataset frame trees have a separate +real-import single-DB differential suite in `test_dataset_frame_tree.py`, including +recursion, cross-shard sources, corruption, budgets and read-only conservation. +Only `event_frames` remains an explicit dataset rejection. Both layouts use the same independently generated 80-event history and a layout-matched standalone SQL oracle. Capacities 2/3/4/5/10/16 produce 5–40 @@ -131,10 +131,10 @@ separate empty/static/dynamic/mixed regressions across both layouts and reducers - A trusted, test-owned snapshot with 1,000,000 alloc/free-request/free-completion events goes through the clean-venv installed `pt-snap import --format msinsight` with **no capacity override**, producing two full 500,000-event members. All - installed catalog templates are classified: 11 dataset-supported templates - have CLI/API parity and independent result checks; `event_frames` and - `active_memory_frame_tree_at_event` retain explicit dataset rejection. The - `peak-memory` report succeeds; `memory-tree` retains its dataset rejection. + installed catalog templates are classified: 12 dataset-supported templates + have CLI/API parity and independent result checks; only `event_frames` + retains explicit dataset rejection. Both `peak-memory` and `memory-tree` + reports succeed, with root/self occupancy conservation checked. - A second required installed-wheel case preserves the original 500,002-event workload and its 500,000 + 2 member layout. It verifies the crossing completed lifetime, final survivor, pages and point-event results. Both installed cases diff --git a/docs/en/interop-acceptance.md b/docs/en/interop-acceptance.md index 4c25b25..d53aab9 100644 --- a/docs/en/interop-acceptance.md +++ b/docs/en/interop-acceptance.md @@ -4,7 +4,9 @@ ## Evidence layers and fixed target -The ONLY upstream target is **Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5**. +The historical parser-v2 acceptance and physical-schema reference use +**Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5**. This is not a +verified consumer target for current parser-v3 exports. Protocol/schema validation, isolated source-function checks, synthetic CI, reading an actual original-producer artifact, and actual GUI/C++ consumption are DIFFERENT evidence layers. None implies the next. **The bounded real GUI/C++ and @@ -23,7 +25,48 @@ are separately CC BY 4.0, not automatically covered by this project's MIT lineag A port needs snapshot provenance/PR decisions and retained notices. Executable pickle trust is a SEPARATE fixture review; no new committed pickle is added here. -## Forward chain: trusted source → physical export → real GUI +## Current parser-v3 status and safe retest + +Current exports use `mem_snapshot_parser_v3`. The pinned historical consumer uses +`mem_snapshot_parser_v2` and **will reparse** a v3 hash mismatch; its parse path +removes the manifest/log before invoking the producer. Do not point that old GUI +at the only source/cache copy. The legacy `ptSnap.identity.targetRevision` field +is a schema/reference identity anchor, not proof of consumer support. +The reviewed upstream master +[`5c722719e1d288106f8d18588b0a33d979a4065b`](https://api.github.com/repos/Ascend/msinsight/git/blobs/20254ee06471a4acb3757893263d042dd5df47e8) +also still computes the v2 salt and rejects a mismatched cache hash. Reuse with +these reviewed consumers is therefore incompatible, not merely untested. No +verified v3 consumer target has been selected. + +A current v3 GUI cache-reuse claim requires all of the following: + +1. Identify an immutable consumer commit and exact build; inspect its parser salt, + manifest/member/schema checks and reparse path before execution. A version + label, current branch name, or the old v2 GUI receipt is insufficient. +2. Obtain explicit authorization for that real GUI run. Use a new isolated + directory, a byte-identical copy of the separately trusted pickle, and a new + adjacent export produced by the candidate wheel. Never move or overwrite the + original cache; re-export rather than relabel v2 hashes. Record source, wheel, + manifest and member hashes plus device/capacity before opening anything; + verify `manifest.sourceFile` names the copied pickle's absolute path. +3. Open only that disposable source/cache pair with the identified build. Retain + `UP_TO_DATE` and no-producer-invocation evidence, compare curves/blocks and + first/last/cross-shard event boundaries, and exercise repeated navigation and + cancellation. Record post-run hashes and distinguish permitted consumer-derived + caches from base-data mutation or parser reruns. +4. Keep current v3 results separate from the historical receipt below. The + artifact-only runner's strict `saltedCacheHash` gate intentionally rejects + applying an old v2 receipt to v3 output; never alter that receipt or relax hashes. + +No current v3 consumer build or real GUI run is accepted by this document. + +## Historical v2 forward chain: trusted source → physical export → real GUI + +The pinned historical acceptance below used `mem_snapshot_parser_v2`. Current +exports use `mem_snapshot_parser_v3`; the historical GUI result does not verify +the new parser version. The commands below document the old run, not a safe +current-CLI procedure against the old GUI. Running them with a current CLI emits +v3; use the isolated-copy plan above, not the historical cache/consumer pair. 1. Review/trust the source separately: pickle is executable input, not a sandbox. Record exact source SHA256, size, selected device and capacity. Default import diff --git a/docs/en/querying.md b/docs/en/querying.md index 6f9bc3f..772f92c 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -99,6 +99,7 @@ across devices and there is no shared cross-device event order. | `memory_peak` | Independent allocated/active/reserved maxima over real events; earliest real ID wins ties; `start_id/end_id` may cross shards | | `allocator_gap` | The same independently selected global events, with each gap computed from counters **at that event** | | `active_blocks_at_event`, `active_memory_callstack_at_event` | Route a real event to its owner and resolve full cross-shard sources before grouping/top_n (details below) | +| `active_memory_frame_tree_at_event` | Event-owner live set with cross-shard allocation frames, grouped by complete caller paths; root/self totals conserve occupancy; no top_n pruning | | `preexisting_live` | Owner-event preexisting occupancy, not a leak count | | `block` | Merge proved allocation-event lifetimes and stable negative tokens once, then apply filters/order/page; latest containing-shard state is an observation, not state at arbitrary E | | `freed_block_lifetime` | Deduplicated captured allocations with a proved `free_completed` source; event-ID distance buckets, not time | @@ -216,7 +217,7 @@ shard. Event/allocation pages apply SQL filters and ordering per shard, merge at `offset + limit` candidates per shard, and count matches separately. Range evidence is measured before non-range filters. Ordinary text-stack statistics merge full per-shard groups before global thresholds; averages use summed sizes and counts. -Canonical ordered-frame statistics retain the bounded source-reading path because +Version-1 canonical ordered-frame statistics retain the bounded source-reading path because formatted text is not their identity. Small block/leak/lifetime reductions use read-only attached shards for proof, conflicts and latest observations before pagination; the attachment limit (normally ten) counts selected observation shards plus any @@ -372,9 +373,9 @@ those merge semantics merely by using a built-in name. ### Optional ordered-frame reader contract -This reader-only pt-snap extension is new, not an existing original-producer -feature. Current native/compatible writers emit **no** ordered-frame tables. -Declare `extensions.ptSnapOrderedFrames={"version":1}` in the manifest and use +This pt-snap extension is independent of original-producer capabilities. New +native/compatible imports write the deduplicated version-2 extension described +below. The version-1 reader remains supported. Declare `extensions.ptSnapOrderedFrames={"version":1}` in the manifest and use isolated actual tables in each covered source shard: - `pt_snap_frame_coverage(eventId INTEGER PRIMARY KEY, frameCount INTEGER)` @@ -779,7 +780,7 @@ and `semantics_version` so that lookup stays tied to the rows. ## Frame Memory Breakdown -After a standalone import with this version, inspect the original ordered frame array or +After an import with this version, inspect the original ordered frame array or decompose live memory after an event: ```bash @@ -830,15 +831,46 @@ node ID to distinguish identical labels. Lists include every match and support Tab then Enter/Space. Parent frame moves up one level; Reset zoom returns to the root without clearing the search. Clear the search field to remove the results. -Frame-tree templates and `report memory-tree` support standalone format-3 databases -only. Dataset roots are rejected explicitly; the analyzer never selects a shard -implicitly. Direct native shards have no standalone frame tables: re-import the -original snapshot as a standalone database to use these templates. Existing -native `pt_snap_frame`/coverage readers remain unchanged. `--timeout` uses the -shared query deadline. Text and HTML renderers reject incomplete or nonconserving -trees. Known-empty stacks have an `empty_callstack` synthetic leaf; -missing/corrupt stacks retain the `missing` leaf. `event_frames` returns no rows -for missing events/stacks, known-empty stacks or invalid frame arrays. +`active_memory_frame_tree_at_event` and `report memory-tree` support standalone +format-3 databases and complete native/compatible datasets. Query the dataset root: +the event owner supplies live blocks and allocation owners supply frames, including +earlier slices. Local IDs and display text never identify cross-shard caller paths. +`frame_id` is query-local; `node_id` hashes the full ordered path of +`(filename, line, name)` triples. `scope.source_coverage.ordered_frames_complete` +reports frame evidence separately from output row completeness. Missing, +unknown-version or corrupt evidence keeps its bytes in `missing`; re-import the +original snapshot to capture frames. Known-empty arrays have an `empty_callstack` +leaf. Work budgets and the shared deadline cover the full operation; failures +return no partial tree. Text and HTML renderers reject truncated/nonconserving trees. +`event_frames` remains standalone-only; use dataset `event` with `id` for its +covered original `frames`. Direct native shards lack standalone frame tables; +query the dataset root for cross-shard allocation sources. + +### Deduplicated dataset frame extension (version 2) + +New imports declare `extensions.ptSnapOrderedFrames={"version":2}` and store: + +| Table | Columns | +| --- | --- | +| `pt_snap_frame_value` | `id INTEGER PRIMARY KEY, frameJson TEXT` | +| `pt_snap_stack` | `stackId INTEGER PRIMARY KEY, frameCount INTEGER` | +| `pt_snap_stack_frame` | `stackId INTEGER, frameIndex INTEGER, frameId INTEGER, PRIMARY KEY(stackId,frameIndex)` | +| `pt_snap_event_stack` | `eventId INTEGER PRIMARY KEY, stackId INTEGER` | + +Each canonical raw frame JSON (including extra typed fields) and each distinct +ordered frame-ID sequence is stored once per shard. Repeated frames retain +positions; empty stacks have an explicit count zero. Only real events receive +references. These independent IDs preserve native text interning and compatible +inline schemas. Native extension cache identity is 2; compatible export identity +is 3, preventing reuse of old text-only exports as frame-capable artifacts. +Readers support versions 1/2 and validate exact schemas, contiguous positions, +counts, references and finite JSON with nesting depth at most 128. Writers omit +an event's optional evidence as a whole if it cannot encode within those bounds; +text import and other events remain usable. Tree frames +require text filename/name and integer line; invalid fields degrade to missing. +No text reconstruction occurs. `callstack_analysis` reduces version-2 events by +local stack and text in SQLite before resolving representative arrays and merging +global identities; repeated stacks do not require fetching every event into Python. The Python API uses the same template and contracts: diff --git a/docs/en/quickstart.md b/docs/en/quickstart.md index 0e70846..7a62dac 100644 --- a/docs/en/quickstart.md +++ b/docs/en/quickstart.md @@ -169,7 +169,9 @@ Native output is not msinsight-compatible. replaced, even with `--force`; compatibility, cache validation, GUI entrance and fixed-version evidence have their own [protocol contract](sharded-snapshotdb.md#explicit-msinsight-compatible-export-p2). Read [recorded interoperability acceptance](interop-acceptance.md) before claiming -GUI reuse. A matching hash alone is not GUI acceptance. +GUI reuse. A matching hash alone is not GUI acceptance. The historical pinned +v2 GUI reparses current v3 exports and can rewrite their caches; do not open the +only copy. Current v3 consumer acceptance remains unverified. ### Optional: Split a Snapshot diff --git a/docs/en/sharded-snapshotdb.md b/docs/en/sharded-snapshotdb.md index 0bf64ec..f6a52ad 100644 --- a/docs/en/sharded-snapshotdb.md +++ b/docs/en/sharded-snapshotdb.md @@ -155,6 +155,12 @@ conflict. Native v2 remains a separate single-DB layout, not this compatibility base. `pt_snap_metadata` and auxiliary reference tables are optional here. Unknown `extensions` declarations/tables are ignored by base validation. +Compatible cache reuse checks the ordered-frame declaration version, not frame-table +contents or stored digests. Queries independently validate the frame evidence they use. +Legacy and external text-only caches retain their read-only degradation behavior. +Unencodable optional event evidence may remain uncovered; source coverage is reported +separately. + `DatasetValidation.text_callstacks=True` means an inline text column is available, not that every event has a stack. `structured_frames=False` remains false even when a manifest claims frames: complete structured-frame capability requires a @@ -478,10 +484,18 @@ failure paths; no live upstream GUI or performance acceptance is claimed. ## Explicit msinsight-Compatible Export (P2) `pt-snap import snapshot.pkl --format msinsight --json` (alias -`--format compatibility-v1`) targets **only** -`Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5`. Its supported entrance is -opening the **same original pickle** in that target with adjacent -`.msinsight/`, not direct standalone-directory GUI import. +`--format compatibility-v1`) uses the physical schema referenced from +`Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5`. That revision is a +**historical schema/reference target, not a current parser-v3 GUI-reuse target**. +Its parser uses `mem_snapshot_parser_v2`: it treats current v3 hashes as a cache +miss and starts a reparse that can remove the manifest/log and rewrite cache data. +Do not open the only copy of a current export in that old GUI. + +The historical GUI entrance was the **same original pickle** plus adjacent +`.msinsight/`, not direct standalone-directory import. Current v3 +consumer revision/build and no-reparse GUI acceptance remain unverified. Any new +acceptance needs an explicitly identified v3 consumer and disposable, separately +approved source/cache copies; see the [safe retest plan](interop-acceptance.md#current-parser-v3-status-and-safe-retest). `--output-dir` selects another parent for pt-snap; relocation alone does not create an upstream GUI entrance. `sourceFile` is the absolute original source path. Capacity defaults to `500000` and can be set with `--events-per-slice`; default @@ -506,45 +520,53 @@ next expected ID, after verifying a unique integer primary key; negative boundar rows are excluded. It does not collect the device's event IDs in Python. This bounds additional ID-validation memory, not snapshot loading, replay, or total import RSS. -`cacheHash = SHA256(b"mem_snapshot_parser_v2" + raw_pickle_bytes)`: the salt is -**prepended**. Independent `ptSnap.identity` fields are `sourceSha256`, +`cacheHash = SHA256(b"mem_snapshot_parser_v3" + raw_pickle_bytes)`: the salt is +**prepended**. Caches using the former `mem_snapshot_parser_v2` salt do not match; +export to a new output directory instead of replacing the old cache. +Independent `ptSnap.identity` fields are `sourceSha256`, `requestedDevice`, `eventsPerSlice`, `outputFormat=compatibility-v1`, -`manifestContractVersion=1`, `exportContractVersion=2`, `referenceContractVersion=1`, +`manifestContractVersion=1`, `exportContractVersion=3`, `referenceContractVersion=1`, `metadataSchemaVersion=1`, `importFormatVersion=1`, and full `targetRevision`. -The `ptSnap` object also contains `metadata`, `omittedDevices`, and `members` -(canonical relative path → logical parser-data SHA256 with `memberHashKind=parser-v1` -for new exports). Whole validated ownership/identity -requires every member/hash and identical per-slice metadata; unknown/external +The legacy `targetRevision` name records the historical schema/reference identity; +it does not select a consumer build or certify current GUI cache reuse. +The `ptSnap` object also contains `metadata` and `omittedDevices`. Ownership/identity +requires a complete member inventory and identical per-slice metadata; unknown/external caches and unrelated files are not claimed as owned. Valid Insight caches can be reused read-only without producer metadata. CLI version alone does not invalidate. -Logical seals cover parser-owned base rows and their dictionary entries, not -consumer tables, indexes or appended columns. Both import and dataset queries -check declared seals; unsealed legacy artifacts receive structural, range, -dictionary-reference and `quick_check` validation only. Without a historical seal, -arbitrary changed values or deleted block rows cannot be proven unchanged. -Digests are not signatures. `quick_check` detects physical damage but does not -prove index/table equivalence against deliberate `writable_schema` tampering. -Default warm resolver reuse still reads all member bytes to verify the generation; -cold admission additionally scans rows. Validation time grows with dataset size. +Compatible imports check source SHA256, complete manifest/member inventory, read-only +database schema and indexed event endpoints. They do not scan base/frame rows, run +`quick_check`, or hash database bytes. Historical digest fields are accepted as member +inventories only; their values are not compared and new exports do not generate content +digests. New `parserCache.members` entries use empty strings as structural inventory values. +There is no strict-cache option. These checks do not detect arbitrary edited values, +internal event gaps or damaged secondary indexes. Ordinary dataset queries and explicit +validators retain their independent admission checks; lightweight import results never +seed their validation cache or receipts. + +Native dataset imports retain disposable validation receipts under +`~/.cache/pt-snap-cli/validation-v1/`. They hash manifest/member bytes and validate +native data independently of compatible-cache reuse. Missing or invalid receipts fall +back to full native validation; `PT_SNAP_DATASET_CACHE=off` disables receipt reuse. Compatible publication is **no-replace even with force**. Identical recognized pt-snap caches reuse without loading pickle (force does not bypass this rule). -Changed source path/content, options/contracts, corrupted parser data or legacy -byte-attested DBs changed by a consumer require a new `--output-dir`; no repair, overwrite or deletion of the old -artifact occurs. Exclusive publication and focus compensation use the existing +Changed source path/content or options/contracts require a new `--output-dir`. +The old artifact is never repaired, overwritten or deleted. Exclusive publication +and focus compensation use the existing checked transaction; source SHA256 is rechecked immediately before publish/reuse. Native force replacement and recovery semantics are untouched. Inline text costs `sum(len(callstack.encode("utf-8")) for each event)` rather than one copy per distinct stack. Existing native interning remains available. Conversion temporarily holds both layouts and can leave free SQLite pages; capacity is not an -RSS/registry/block/disk bound. No optional frame enhancement is emitted. +RSS/registry/block/disk bound. New exports also retain the optional version-2 +ordered-frame extension described above. **The artifact-only runner still reports its own GUI stage as pending/not-run.** -The separate fixed-version real GUI/C++ acceptance on 2026-10-11 is recorded in -[interoperability acceptance](interop-acceptance.md); the runner does not mark -GUI acceptance passed automatically. Fixed-source parser checks complete manifest, +The separate parser-v2 real GUI/C++ acceptance on 2026-10-11 is recorded in +[interoperability acceptance](interop-acceptance.md); neither it nor the runner +establishes current parser-v3 GUI acceptance. The historical fixed-source parser checks complete manifest, nonempty identical hash, every ready/resolved/existing member, read-only DB/device validation and allocation-cache existence **or buildability** before UP_TO_DATE. Derived allocation caches may modify DBs without invoking the producer: mtime or diff --git a/docs/en/snapshot-analyzer-api.md b/docs/en/snapshot-analyzer-api.md index e1b2a0d..fc52ea7 100644 --- a/docs/en/snapshot-analyzer-api.md +++ b/docs/en/snapshot-analyzer-api.md @@ -279,6 +279,17 @@ metadata schema version returns `status="invalid"`. ## Errors and Scope +Concurrent `active_memory_frame_tree_at_event` requests on the same analyzer +use latest-request-wins semantics. A new request interrupts the previous tree's +Python work and SQLite statements; superseded queued requests do not compute. +The old call raises `pt_snap_cli.core.QueryCancelledError` (`QUERY_CANCELLED` +in JSON), never a partial successful tree. Closing the analyzer or invalidating +its context cache also cancels outstanding tree work. Worker-owned connections +are closed in the executing thread before the next tree runs. Other templates +retain their existing concurrency contract; separate analyzers/services and +separate CLI processes do not cancel one another. This does not introduce a +persistent tree-result cache. + - `set_focus()` raises `FileNotFoundError` for a missing database and `ValueError` for an invalid SnapshotDB schema. Device validation uses the shared `InvalidDeviceError`; selecting a device without any resolvable database raises diff --git a/docs/zh/database.md b/docs/zh/database.md index e542e7d..44c6df3 100644 --- a/docs/zh/database.md +++ b/docs/zh/database.md @@ -360,7 +360,7 @@ CREATE UNIQUE INDEX idx_callstack_frame_position ### 单库帧能力边界 格式 3 仅用于单库导入。原生分片导入保持格式 2，兼容导出保持格式 1； -它们不会声称支持单库帧树。新增 `callstack_frame_manifest(callstackId INTEGER PRIMARY KEY, +它们通过独立的版本 2 去重帧扩展支持数据集帧树，详见[查询指南](querying.md#数据集去重帧扩展版本-2)。新增 `callstack_frame_manifest(callstackId INTEGER PRIMARY KEY, frameCount INTEGER NOT NULL)` 保存每个采集调用栈的预期长度，包括零。 filename/name 必须是文本，line 必须是整数。`event_frames` 对缺失、已知空数组 或损坏数组返回零行，不输出部分重建的数组。树将已知空栈与缺失/损坏证据分开。 @@ -437,7 +437,7 @@ P0 兼容 v1 表序、内联布局和 manifest 语义不变。 `trace_entry_` 表，第九列为 `callstack TEXT`，不是 VIEW；block/dictionary 位置 schema 精确匹配。转换去除暂存的未隔离 `callstack` 表，写入 `pt_snap_metadata.import_format_version=1`，保留隔离的 `pt_snap_block_reference` -文本生命周期来源。原生单库/数据集仍使用去重和 v2 metadata，不生成或从文本推断有序 frames。 +文本生命周期来源。原生数据集保留独立的版本 2 帧扩展，不从文本推断有序 frames。 内联调用栈字节按**所有事件**的文本总和增长，不按不同栈计数；转换暂时并存原生与内联表， 可能保留 SQLite free pages。容量不限制 block、加载 RSS 或总磁盘。兼容 manifest 的 diff --git a/docs/zh/dataset-acceptance.md b/docs/zh/dataset-acceptance.md index d14041b..7182ae0 100644 --- a/docs/zh/dataset-acceptance.md +++ b/docs/zh/dataset-acceptance.md @@ -7,8 +7,9 @@ `tests/core/test_dataset_realistic.py` 默认在兼容 v1 和原生 v2 两种布局上各执行 20 个确定性种子；夜间配置将同一批种子扩展至 200 个。原有八个回归种子及二十种 查询形态完整保留。每个种子每种布局运行 69–73 个查询用例（20 种子共 2,896 次 -成对比较，200 种子共 28,864 次），覆盖全部十一个 dataset 支持模板。 -仅单库支持的 `event_frames`、`active_memory_frame_tree_at_event` 另有明确拒绝测试， +成对比较，200 种子共 28,864 次），覆盖十一个文本查询契约。 +帧树由 `test_dataset_frame_tree.py` 单独验证真实导入的单库/数据集对齐、递归、跨片来源、 +损坏、预算及只读守恒；仅 `event_frames` 保留明确数据集拒绝测试， 不会计为差分成功。 两种布局使用相同的独立生成 80-event 历史，并分别与同布局单库 SQL 参考结果比较。 @@ -105,10 +106,8 @@ fallback 三条路径的这些契约。`top_n=0` 完整性另有空/static/dynam 64-MiB 上限，无需创建 8-GB 数据文件。 - 受信、测试自行构造的 1,000,000 个 alloc/free-request/free-completion 事件快照， 经干净 venv 中安装的 `pt-snap import --format msinsight`，**不覆盖容量默认值**， - 生成两个完整的 500,000-event 成员。所有已安装目录模板均被分类：11 个支持数据集的 - 模板核对 CLI/API 一致性及独立结果；`event_frames` 和 - `active_memory_frame_tree_at_event` 保留明确的数据集拒绝。`peak-memory` 报告成功， - `memory-tree` 保留数据集拒绝。 + 生成两个完整的 500,000-event 成员。所有已安装目录模板均被分类：12 个支持数据集的 + 模板核对 CLI/API 一致性及独立结果。帧树模板及 `memory-tree` 报告成功，并验证根节点/自身占用守恒；仅 `event_frames` 保留数据集拒绝。 - 第二个必需的安装包用例保留原有 500,002-event 工作负载及 500,000 + 2 成员布局， 验证跨界完成生命周期、最终存活分配、分页及定点事件结果。两个安装包用例均要求 查询前后输入、manifest 及成员 hash 不变。 diff --git a/docs/zh/interop-acceptance.md b/docs/zh/interop-acceptance.md index a391028..89a1cba 100644 --- a/docs/zh/interop-acceptance.md +++ b/docs/zh/interop-acceptance.md @@ -4,7 +4,9 @@ ## 证据分层与固定目标 -唯一上游目标是 **Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5**。 +历史 parser-v2 验收与物理 schema 参考使用 +**Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5**， +它不是当前 parser-v3 导出已验证的消费端目标。 协议/schema、隔离源函数、合成 CI、真实原版生产端产物的读取差分、GUI/C++ 实测是 不同证据层，不能互相替代。**2026-10-11 的有界真实 GUI/C++ 与原版生产端反向验收已通过**， 详见下方记录。[#205 待验收说明](https://github.com/codeasier/pt-snap-cli/issues/205#issuecomment-5987264017) @@ -19,7 +21,38 @@ 本项目 MIT lineage。真正移植须走 snapshot provenance/PR 声明并保留 notices。 可执行 pickle 的信任是独立 fixture 审核；本验收不新增已提交 pickle。 -## 正向：可信源 → 物理兼容导出 → 真实 GUI +## 当前 parser-v3 状态与安全重验 + +当前导出使用 `mem_snapshot_parser_v3`，历史固定消费端使用 `mem_snapshot_parser_v2`， +v3 hash 不匹配时**必然重解析**；该路径先删除 manifest/log，再调用生产者。 +不要让旧 GUI 打开唯一的源/缓存副本。历史字段 `ptSnap.identity.targetRevision` +是 schema/接口参考身份，不是消费端支持证明。已核上游 master +[`5c722719e1d288106f8d18588b0a33d979a4065b`](https://api.github.com/repos/Ascend/msinsight/git/blobs/20254ee06471a4acb3757893263d042dd5df47e8) +也仍计算 v2 salt 并拒绝不匹配的 cache hash。因此与这些已核消费端的复用是确定不兼容， +不只是尚未实测；尚未选择任何已验证的 v3 消费端目标。 + +当前 v3 GUI 缓存复用声明必须满足以下全部条件： + +1. 明确不可变消费端 commit 与准确 build，执行前检查 parser salt、manifest/成员/schema + 检查与重解析路径。仅版本标签、当前分支名或旧 v2 GUI receipt 都不足够。 +2. 获得该次真实 GUI 运行的明确授权。在新的隔离目录中使用另行信任的 pickle 的字节一致 + 副本，并由候选 wheel 生成新的邻接导出。不要搬移或覆盖原缓存，也不要改标签伪装 v2 hash； + 应重新导出。打开前记录源、wheel、manifest、成员 hash 以及设备/容量， + 并核对 `manifest.sourceFile` 指向 pickle 副本的绝对路径。 +3. 只用指定 build 打开该可丢弃源/缓存对，保存 `UP_TO_DATE` 与无生产者调用证据， + 对照曲线/block、首尾及跨片事件边界，覆盖重复导航与取消。记录运行后 hash，并区分 + 允许的消费端派生缓存、基础数据变更和 parser 重跑。 +4. 当前 v3 结果与下方历史 receipt 分开保存。产物 runner 的严格 `saltedCacheHash` 门禁 + 会拒绝把旧 v2 receipt 用于 v3 导出；不得修改历史 receipt 或放宽 hash 校验。 + +本文尚未接受任何当前 v3 消费端 build 或真实 GUI 运行。 + +## 历史 v2 正向：可信源 → 物理兼容导出 → 真实 GUI + +以下固定历史验收使用 `mem_snapshot_parser_v2`，当前导出已使用 +`mem_snapshot_parser_v3`；历史 GUI 结果不代表新版解析器已验收。下方命令只记录旧运行， +不是让当前 CLI 对接旧 GUI 的安全操作指引；当前 CLI 执行它们会生成 v3。 +新验收请用上方隔离副本方案，不要复用历史缓存/消费端组合。 1. 先独立审核/信任源：pickle 可执行，不是沙箱。记录 SHA256、大小、设备与容量。 默认 import 仍单库，原生/兼容模式显式区分。 diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 5faa28f..744a106 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -89,6 +89,7 @@ pt-snap query '' --device 0 --template-use event --params '{"id":123}' | `memory_peak` | allocated/active/reserved 独立真实峰值；同值选最早真实 ID；`start_id/end_id` 可跨片 | | `allocator_gap` | 使用同样独立选择的全局事件，每个 gap 由**该事件**计数器计算 | | `active_blocks_at_event`, `active_memory_callstack_at_event` | 路由实际事件到所有者片，分组/top_n 前解析完整跨片来源（见下文） | +| `active_memory_frame_tree_at_event` | 事件所属片的存活集合，跨片解析分配帧后按完整调用路径聚合；根节点与自身占用守恒，不做 top_n 剪枝 | | `preexisting_live` | 所有者事件的 preexisting 占用，不是泄漏数 | | `block` | 已证明的分配事件生命周期及稳定负 token 去重后筛选/排序/分页；最新出现片 state 是观察，不是任意 E 的状态 | | `freed_block_lifetime` | 已采集分配与已证明 `free_completed` 来源的去重生命周期；event-ID 距离桶，不是时间 | @@ -300,8 +301,8 @@ cache 仍由调用者所有。不生成合并临时数据库。`ReportService.ev ### 可选有序 frame reader 合同 -这是新增的仅 reader pt-snap 扩展，不是原版 producer 已有功能。当前原生/兼容 writer -**不生成**有序 frame 表。manifest 声明 `extensions.ptSnapOrderedFrames={"version":1}`， +此 pt-snap 扩展独立于原版 producer 能力。新原生/兼容导入生成下文的去重版本 2 +扩展；保留版本 1 原始数组读取兼容。manifest 声明 `extensions.ptSnapOrderedFrames={"version":1}`， 并在有覆盖的来源片使用隔离的实际表： - `pt_snap_frame_coverage(eventId INTEGER PRIMARY KEY, frameCount INTEGER)` @@ -699,13 +700,36 @@ HTML 图始终按字节比例绘制。即使帧过小或占用为零，也可以 列表包含全部匹配项，支持 Tab 定位后按 Enter/空格进入。Parent frame 返回父节点， Reset zoom 回到根节点且保留搜索；清空搜索框可清除搜索结果。 -帧树模板和 `report memory-tree` 仅支持单库格式 3 数据库。dataset 根目录会 -明确拒绝，分析器不会隐式选取分片。直接访问原生分片也没有这些单库帧表； -使用新模板须将原始 snapshot 重新导入为单库。已有 `pt_snap_frame`/coverage -读取能力不变。`--timeout` 使用共享查询截止时间。文本和 HTML 渲染器拒绝 -不完整或不守恒的树。已知空栈使用 `empty_callstack` 合成叶节点，缺失/损坏 -调用栈使用 `missing` 叶节点。`event_frames` 对缺失事件/栈、已知空栈以及 -无效帧数组返回零行。 +`active_memory_frame_tree_at_event` 和 `report memory-tree` 支持单库格式 3 +以及完整的原生/兼容分片数据集。查询数据集根目录：事件所属片提供存活块集合， +分配事件所属片提供原始帧，可追溯到更早分片。局部 ID 和显示文本不能作为跨片分组身份。 +`frame_id` 仅在本次查询有效，`node_id` 是完整有序 `(filename, line, name)` 调用路径的哈希。 +`scope.source_coverage.ordered_frames_complete` 单独报告帧证据完整性，与输出行完整性不同。 +缺失、未知版本或损坏证据保留在 `missing` 的占用中；重新导入原始快照可获得采集帧。 +已知空数组使用 `empty_callstack` 叶节点。工作预算和共享截止时间覆盖整个操作， +失败不返回部分树；文本和 HTML 拒绝截断或不守恒的树。 +`event_frames` 仍仅支持单库；数据集使用 `event` 的 `id` 参数读取覆盖的原始 `frames`。 +直接访问原生分片没有单库帧表，应查询数据集根目录，以跨片解析分配来源。 + +### 数据集去重帧扩展（版本 2） + +新导入声明 `extensions.ptSnapOrderedFrames={"version":2}`，使用以下表： + +| 表 | 列 | +| --- | --- | +| `pt_snap_frame_value` | `id INTEGER PRIMARY KEY, frameJson TEXT` | +| `pt_snap_stack` | `stackId INTEGER PRIMARY KEY, frameCount INTEGER` | +| `pt_snap_stack_frame` | `stackId INTEGER, frameIndex INTEGER, frameId INTEGER, PRIMARY KEY(stackId,frameIndex)` | +| `pt_snap_event_stack` | `eventId INTEGER PRIMARY KEY, stackId INTEGER` | + +每个 canonical 原始帧 JSON（包括额外类型字段）和每个不同的有序帧 ID 序列在每片只存一份。 +重复帧保留位置，空栈有明确的零 frameCount，仅真实事件保存引用。 +独立 ID 不改变原生文本去重或兼容内联 schema。原生扩展缓存身份更新为 2，兼容导出身份为 3， +避免把旧文本产物复用成帧能力产物。保留版本 1/2 读取兼容，校验精确 schema、连续位置、 +数量、引用以及深度至多 128 的有限数字 JSON。若无法在该边界内编码，写入端会整体省略 +该事件的可选帧证据，文本导入及其他事件仍可用。帧树要求 filename/name 为文本、line 为整数， +不合规帧降级为缺失叶，不从文本重建。`callstack_analysis` 对版本 2 先在 SQLite 中按 +局部栈和文本归约，再解析代表数组并合并全局身份，重复栈无需将每个事件取入 Python。 Python API 使用相同模板和契约： diff --git a/docs/zh/quickstart.md b/docs/zh/quickstart.md index 7adb359..17d09d1 100644 --- a/docs/zh/quickstart.md +++ b/docs/zh/quickstart.md @@ -153,7 +153,8 @@ RSS。输出路径、完整成员校验、force/恢复行为及支持的内建 `--format msinsight` 是显式选择。已有兼容目标即使 `--force` 也不替换；兼容性、 缓存校验、GUI 入口及固定版本证据有独立的 [协议合同](sharded-snapshotdb.md#显式-msinsight-兼容导出p2)。声称 GUI 复用前请阅读 -[已记录的互操作验收](interop-acceptance.md)，仅 hash 相同不等于 GUI 验收通过。 +[已记录的互操作验收](interop-acceptance.md)，仅 hash 相同不等于 GUI 验收通过。历史固定 +v2 GUI 会重解析当前 v3 导出并可能改写缓存，不要打开唯一副本；当前 v3 消费端验收仍未验证。 ### 可选：拆分快照 diff --git a/docs/zh/sharded-snapshotdb.md b/docs/zh/sharded-snapshotdb.md index 73ce759..e793009 100644 --- a/docs/zh/sharded-snapshotdb.md +++ b/docs/zh/sharded-snapshotdb.md @@ -124,6 +124,10 @@ dictionary 的十进制整数 key 以文本存储，使用真实带设备后缀 `callstack` 表**：现有布局检测会正确报告 v1/v2 冲突。原生 v2 仍是独立单库布局， 不是本兼容基础格式。这里不要求 `pt_snap_metadata` 或辅助引用表。 +兼容缓存复用只检查有序帧扩展声明版本，不扫描帧表内容或比较历史摘要。 +查询独立校验其使用的帧证据。旧缓存和外部纯文本缓存保留只读降级行为； +无法编码的可选事件帧仍可不覆盖，查询继续独立报告 source coverage。 + 基础校验忽略未知 `extensions` 声明/表。`DatasetValidation.text_callstacks=True` 表示有内联文本列，不保证每个事件有栈。即使 manifest 声称 frames， `structured_frames=False` 仍为假：完整结构化 frame 能力要求已识别的独立扩展版本、 @@ -382,9 +386,15 @@ pickle 仍可能整体加载。分库只限制**单库真实事件数**，不限 ## 显式 msinsight 兼容导出（P2） `pt-snap import snapshot.pkl --format msinsight --json`（别名 -`--format compatibility-v1`）**仅**对齐 -`Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5`。支持入口是在该目标版本 -打开**同一原始 pickle**及邻接 `<原始路径>.msinsight/`，不是 GUI 直接导入独立目录。 +`--format compatibility-v1`）采用参考自 +`Ascend/msinsight@101f65b877a267ffd5f66ea3834706057ba243e5` 的物理 schema。 +该版本是**历史 schema/接口参考，不是当前 parser-v3 GUI 缓存复用目标**。 +它使用 `mem_snapshot_parser_v2`，会将当前 v3 hash 视为缓存未命中并启动重解析， +可能删除 manifest/log 并改写缓存数据。不要用该旧 GUI 打开当前导出产物的唯一副本。 + +历史 GUI 入口是**同一原始 pickle**及邻接 `<原始路径>.msinsight/`，不是直接导入独立目录。 +当前 v3 消费端 revision/build 及免重解析 GUI 验收仍未验证。新验收须明确指定 v3 消费端， +并使用另行授权的可丢弃源/缓存副本，见[安全重验方案](interop-acceptance.md#当前-parser-v3-状态与安全重验)。 `--output-dir` 为 pt-snap 选择其他父目录，搬迁本身不创建上游入口。`sourceFile` 记录 绝对原始源路径。容量默认 `500000`，可用 `--events-per-slice` 调整；默认单库及显式 native-v2 输出仍独立且不变。 @@ -403,33 +413,39 @@ raw-frame 丢失与字符串 OOM action 强制转换属于已知局限，不应 边界行，不再在 Python 中收集整个设备的事件 ID。这只限制额外的 ID 校验内存， 不代表快照加载、回放或完整导入的 RSS 有界。 -`cacheHash = SHA256(b"mem_snapshot_parser_v2" + 原始_pickle_字节)`，salt **前置**。 +`cacheHash = SHA256(b"mem_snapshot_parser_v3" + 原始_pickle_字节)`，salt **前置**。 +旧 `mem_snapshot_parser_v2` 缓存不再匹配，请指定新的输出目录重新导出，不覆盖旧缓存。 独立 `ptSnap.identity` 字段为 `sourceSha256`、`requestedDevice`、`eventsPerSlice`、 -`outputFormat=compatibility-v1`、`manifestContractVersion=1`、`exportContractVersion=2`、 +`outputFormat=compatibility-v1`、`manifestContractVersion=1`、`exportContractVersion=3`、 `referenceContractVersion=1`、`metadataSchemaVersion=1`、`importFormatVersion=1`、完整 -`targetRevision`。`ptSnap` 另含 `metadata`、`omittedDevices` 和 `members`（规范相对路径 -→ 基础解析数据逻辑 SHA256，`memberHashKind=parser-v1`）。整体已校验所有权/身份要求每个成员/哈希及逐片 metadata 相同； -不接管未知/外部缓存或无关文件。CLI 版本变化本身不使缓存失效。 - -逻辑摘要覆盖解析器基础行及对应字典项，不覆盖消费者表、索引和追加列。导入和数据集查询 -均校验已声明的摘要；无摘要的旧产物仅执行结构、事件范围、字典引用及 `quick_check` -校验。缺少历史摘要时，不能追溯证明任意字段未改动或 block 行未被删除。摘要不是签名， -`quick_check` 可检测物理损坏，但不能证明刻意使用 `writable_schema` 篡改后的索引与表 -仍然一致。默认 resolver 热缓存仍读取全部成员字节核对 generation，冷准入还会扫描行， -因此校验耗时随数据量增长。 +`targetRevision`。该历史字段名记录 schema/接口参考身份，不选择消费端 build，也不证明 +当前 GUI 缓存复用已通过。`ptSnap` 另含 `metadata` 和 `omittedDevices`。所有权/身份校验 +要求成员清单完整且逐片 metadata 相同，不接管未知/外部缓存或无关文件。 +有效 Insight 缓存可以只读复用，不伪造生产者 metadata。CLI 版本变化本身不使缓存失效。 + +兼容产物导入检查源文件 SHA256、完整 manifest/成员清单、只读数据库表结构及 +索引上的事件端点；不扫描基础行或 frame 行、不执行 `quick_check`，也不读取全部 +DB 字节计算摘要。历史摘要字段仅作为成员清单兼容，不比较摘要值，新产物不再生成 +内容摘要。新的 `parserCache.members` 使用空字符串作为结构清单的值。 +不再提供缓存强校验开关。这些检查不保证发现任意字段修改、区间内部事件 +缺失或二级索引损坏。正常数据集查询和显式校验器保留独立准入检查，轻量导入结果 +不会写入其校验缓存或持久化校验记录。 + +原生数据集导入保留 `~/.cache/pt-snap-cli/validation-v1/` 下可丢弃的校验记录， +独立执行原生格式的文件摘要及内容校验，不属于兼容缓存复用逻辑。记录缺失或无效时 +回退完整原生校验，`PT_SNAP_DATASET_CACHE=off` 禁用记录复用。 兼容发布 **force 也 no-replace**。已识别、身份相同的 pt-snap 缓存不加载 pickle 即复用， -force 不绕过该规则。源路径/内容、选项/合同变化、基础解析数据损坏，或旧物理摘要产物被调用方修改后须换新 -`--output-dir`，不修复、不覆盖、不删除旧产物。排他发布与 focus 补偿使用已有受检事务， +force 不绕过该规则。源路径/内容、选项/合同变化后须换新 `--output-dir`。不修复、不覆盖、不删除旧产物。排他发布与 focus 补偿使用已有受检事务， 发布/复用前再次核对源 SHA256；原生 force 替换与恢复语义不变。 内联文本成本按每个事件的 UTF-8 调用栈字节求和，不按不同栈各存一份；原生去重仍可用。 转换暂时并存两种布局，可能留下 SQLite free pages；容量不是 RSS/registry/block/磁盘 -上限。不生成可选 frame 增强。 +上限。另保留独立的版本 2 去重 frame 扩展，详见[查询指南](querying.md#数据集去重帧扩展版本-2)。 **仅检查已有产物的 runner 仍将自身 GUI 阶段标为待验收/not-run。** -2026-10-11 独立固定版本真实 GUI/C++ 实测见[互操作验收](interop-acceptance.md)， -不表示 runner 自动判定 GUI 通过。固定源码 parser 在 UP_TO_DATE 前检查 complete manifest、 +2026-10-11 独立 parser-v2 真实 GUI/C++ 实测见[互操作验收](interop-acceptance.md)， +该历史记录和 runner 都不证明当前 parser-v3 GUI 已通过。历史固定源码 parser 在 UP_TO_DATE 前检查 complete manifest、 非空相同 hash、每个 ready/可解析/存在成员、只读 DB/设备校验，以及 allocation cache 存在**或可构建**。派生 allocation cache 可在不运行生产者时修改 DB，mtime 或新派生表 不能单独证明重跑。真实 GUI 验收须保留 UP_TO_DATE/无 script 调用日志并对照曲线、block、 diff --git a/docs/zh/snapshot-analyzer-api.md b/docs/zh/snapshot-analyzer-api.md index 2cc5bc7..847cc83 100644 --- a/docs/zh/snapshot-analyzer-api.md +++ b/docs/zh/snapshot-analyzer-api.md @@ -252,6 +252,14 @@ other_metadata = analyzer.get_database_metadata("/path/to/other.db") ## 错误与范围 +同一个 analyzer 上并发调用 `active_memory_frame_tree_at_event` 时，只执行最新请求。 +新请求会中断旧任务的 Python 计算和 SQLite 语句，排队中已过期的请求不会继续计算。 +旧调用抛出 `pt_snap_cli.core.QueryCancelledError`（JSON 错误码为 `QUERY_CANCELLED`）， +不会返回成功的部分树。关闭 analyzer 或清理其 context 缓存也会取消未结束的拆解任务。 +每个任务在执行线程中关闭自己持有的数据库连接后，下一任务才开始计算。其它模板保留 +原有并发约定；不同 analyzer/service 实例、不同 CLI 进程之间互不取消。这不会增加 +持久化的拆解结果缓存。 + - `set_focus()` 在数据库不存在时抛出 `FileNotFoundError`，在 SnapshotDB schema 无效时 抛出 `ValueError`。设备校验使用共享的 `InvalidDeviceError`；没有任何可解析数据库时只 设置设备会抛出 `RuntimeError`。 diff --git a/src/pt_snap_cli/_frame_json.py b/src/pt_snap_cli/_frame_json.py new file mode 100644 index 0000000..56509b3 --- /dev/null +++ b/src/pt_snap_cli/_frame_json.py @@ -0,0 +1,27 @@ +"""Strict JSON validation shared by optional ordered frame writers and readers.""" + +import math + + +def _invalid_constant(value: str) -> object: + raise ValueError(f"Non-JSON numeric constant: {value}") + + +def _finite_float(value: str) -> float: + result = float(value) + if not math.isfinite(result): + raise ValueError("Non-finite frame number") + return result + + +def _bounded_depth(value: object) -> bool: + pending = [(value, 0)] + while pending: + current, depth = pending.pop() + if depth > 128: + return False + if isinstance(current, dict): + pending.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + pending.extend((item, depth + 1) for item in current) + return True diff --git a/src/pt_snap_cli/api.py b/src/pt_snap_cli/api.py index fe7d9a1..7b7cf12 100644 --- a/src/pt_snap_cli/api.py +++ b/src/pt_snap_cli/api.py @@ -140,7 +140,7 @@ def invalidate_context_cache(self, db_path: Path | str | None = None) -> None: This cleanup operation remains available after close(). While open, the analyzer can create fresh contexts on its next query or overview. """ - self._context_cache.invalidate(db_path) + self._query_service.invalidate_context_cache(db_path) def get_focus(self) -> FocusState: self._ensure_open() diff --git a/src/pt_snap_cli/core/__init__.py b/src/pt_snap_cli/core/__init__.py index 807155b..c1c37b9 100644 --- a/src/pt_snap_cli/core/__init__.py +++ b/src/pt_snap_cli/core/__init__.py @@ -13,6 +13,7 @@ InvalidParameterError, InvalidSkillTargetError, PtSnapCoreError, + QueryCancelledError, QueryExecutionError, QueryTimeoutError, SkillCatalogError, @@ -65,6 +66,7 @@ from pt_snap_cli.core.split_service import SplitService __all__ = [ + "QueryCancelledError", "PtSnapCoreError", "FocusNotConfiguredError", "FocusFileInvalidError", diff --git a/src/pt_snap_cli/core/dataset_active_sql.py b/src/pt_snap_cli/core/dataset_active_sql.py index 10e035b..24768d6 100644 --- a/src/pt_snap_cli/core/dataset_active_sql.py +++ b/src/pt_snap_cli/core/dataset_active_sql.py @@ -95,7 +95,7 @@ def reduce_active( used only to obtain their owning shard's full text, never as global identity. A frame extension, even partially covered, must use ordered-frame attribution. """ - if sources.dataset.ordered_frames_version == 1: + if sources.dataset.ordered_frames_version in (1, 2): return None required = _required_slices(sources, [item], _attachment_limit()) if required is None: diff --git a/src/pt_snap_cli/core/dataset_frame_tree.py b/src/pt_snap_cli/core/dataset_frame_tree.py new file mode 100644 index 0000000..3d23247 --- /dev/null +++ b/src/pt_snap_cli/core/dataset_frame_tree.py @@ -0,0 +1,195 @@ +"""Caller-path occupancy over the event owner's live set and allocation owners.""" + +from __future__ import annotations + +import hashlib +import json +from typing import cast + +from .dataset_attribution import _rounding_cursor, _scope +from .dataset_sources import DatasetSourceResolver +from .models import QueryResult + +METRICS = ( + ("block_count", "self_block_count"), + ("size_bytes", "self_bytes"), + ("requested_bytes", "self_requested_bytes"), +) + + +def _node( + node_id: str, + parent: str | None, + depth: int, + category: str, + label: str, + frame_id: int | None = None, + frame: tuple[str, int, str] | None = None, +) -> dict[str, object]: + return { + "node_id": node_id, + "parent_id": parent, + "depth": depth, + "frame_id": frame_id, + "category": category, + "label": label, + "filename": frame[0] if frame else None, + "line": frame[1] if frame else None, + "name": frame[2] if frame else None, + **{field: 0 for pair in METRICS for field in pair}, + } + + +def frame_tree( + sources: DatasetSourceResolver, + template: str, + params: dict[str, object], + max_rows: int | None, + exact_total: bool, + slice_index: int | None, + semantics_version: int | None, +) -> QueryResult: + event_id = cast(int, params["event_id"]) + owner = sources.target(event_id, slice_index) + groups = sources.read( + sources.dataset.root / owner.file, + "SELECT allocEventId, CASE WHEN allocEventId=-1 AND freeEventId=-1 THEN 'static' " + "WHEN allocEventId=-1 THEN 'preexisting_live_at_event' ELSE 'dynamic_live_at_event' END " + "AS category, COUNT(*) AS block_count, SUM(size) AS size_bytes, " + f'SUM(COALESCE(requestedSize,0)) AS requested_bytes FROM "block_{sources.device.device_id}" ' + "WHERE size>=? AND (allocEventId=-1 OR allocEventId<=?) " + "AND (freeEventId IS NULL OR freeEventId<0 OR freeEventId>?) " + + ("AND allocEventId!=-1 " if not params["include_static"] else "") + + "GROUP BY allocEventId,category ORDER BY allocEventId,category", + [params["min_size"], event_id, event_id], + ) + references = sources.events( + cast(int, g["allocEventId"]) for g in groups if cast(int, g["allocEventId"]) >= 0 + ) + root = _node("root", None, 0, "mixed", "[all active blocks]") + nodes = {"root": root} + frame_ids: dict[tuple[str, int, str], int] = {} + ordered_blocks = dynamic_blocks = resolved_blocks = dynamic_bytes = captured_bytes = 0 + for group in groups: + sources.budget.remaining() + category = cast(str, group["category"]) + source = references.get(cast(int, group["allocEventId"])) + frames = source.frames if source is not None and source.event["action"] == 4 else None + if category == "dynamic_live_at_event": + dynamic_blocks += cast(int, group["block_count"]) + dynamic_bytes += cast(int, group["size_bytes"]) + if source is not None and source.event["action"] == 4: + resolved_blocks += cast(int, group["block_count"]) + # A covered raw array can have nonstandard dictionaries. Preserve its + # occupancy, but never invent file/line/name from the rendered text. + usable = frames is not None and all( + isinstance(f.get("filename"), str) + and type(f.get("line")) is int + and isinstance(f.get("name"), str) + for f in frames + ) + parent = root + if category == "dynamic_live_at_event" and usable: + ordered_blocks += cast(int, group["block_count"]) + captured_bytes += cast(int, group["size_bytes"]) + path = [root] + if category == "dynamic_live_at_event" and usable and frames: + for depth, value in enumerate(reversed(frames), 1): + sources.budget.remaining() + frame = ( + cast(str, value["filename"]), + cast(int, value["line"]), + cast(str, value["name"]), + ) + frame_ids.setdefault(frame, len(frame_ids)) + identity = json.dumps( + [parent["node_id"], frame], ensure_ascii=True, separators=(",", ":") + ) + node_id = "frame/" + hashlib.sha256(identity.encode("utf-8")).hexdigest() + if node_id not in nodes: + nodes[node_id] = _node( + node_id, + cast(str, parent["node_id"]), + depth, + category, + f"{frame[0]}:{frame[1]} {frame[2]}", + frame_ids[frame], + frame, + ) + sources.budget.consume([nodes[node_id]]) + parent = nodes[node_id] + path.append(parent) + else: + kind = ( + category + if category != "dynamic_live_at_event" + else ("empty_callstack" if usable and frames == [] else "missing") + ) + label = { + "static": "[static]", + "preexisting_live_at_event": "[preexisting live]", + "empty_callstack": "[empty captured callstack]", + "missing": "[missing callstack]", + }[kind] + if kind not in nodes: + nodes[kind] = _node( + kind, + "root", + 1, + "empty_callstack" if kind == "empty_callstack" else category, + label, + ) + sources.budget.consume([nodes[kind]]) + parent = nodes[kind] + path.append(parent) + for total, own in METRICS: + amount = cast(int, group[total]) + for node in path: + node[total] = cast(int, node[total]) + amount + parent[own] = cast(int, parent[own]) + amount + rows = list(nodes.values()) + with _rounding_cursor(sources) as cursor: + for row in rows: + sources.budget.remaining() + row["percent_of_total"] = cursor.execute( + "SELECT ROUND(? * 100.0 / NULLIF(?,0),4)", (row["size_bytes"], root["size_bytes"]) + ).fetchone()[0] + rows.sort( + key=lambda r: ( + cast(int, r["depth"]), + cast(str, r["parent_id"] or ""), + -cast(int, r["size_bytes"]), + cast(str, r["node_id"]), + ) + ) + selected = rows[:max_rows] if max_rows is not None and max_rows > 0 else rows + sources.budget.consume(selected) + coverage = { + "range_complete": True, + "event_position": "after", + "active_blocks": root["block_count"], + "active_bytes": root["size_bytes"], + "dynamic_blocks": dynamic_blocks, + "dynamic_bytes": dynamic_bytes, + "allocation_events_resolved": resolved_blocks, + "allocation_source_complete": resolved_blocks == dynamic_blocks, + "ordered_frame_bytes": captured_bytes, + "ordered_frame_blocks": ordered_blocks, + "ordered_frames_complete": ordered_blocks == dynamic_blocks, + "frame_degradation": None if ordered_blocks == dynamic_blocks else "text_only_or_uncovered", + "frame_identity": "filename_line_name; frame_id is query-local, node_id is caller-path hash", + "source_queries": sources.query_count, + } + return QueryResult( + total=len(rows) if exact_total else len(selected), + returned=len(selected), + device_id=sources.device.device_id, + rows=selected, + template=template, + semantics_version=semantics_version, + has_more=len(selected) < len(rows), + truncated=len(selected) < len(rows), + total_is_exact=exact_total or len(selected) == len(rows), + timeout_s=sources.budget.timeout_s, + scope=_scope(sources, owner.index, event_id, coverage), + ) diff --git a/src/pt_snap_cli/core/dataset_frames.py b/src/pt_snap_cli/core/dataset_frames.py new file mode 100644 index 0000000..30e62f3 --- /dev/null +++ b/src/pt_snap_cli/core/dataset_frames.py @@ -0,0 +1,130 @@ +"""Read the version-2 normalized frame extension without reconstructing text.""" + +from __future__ import annotations + +import hashlib +import json +from collections import defaultdict +from pathlib import Path +from typing import Protocol, cast + +from pt_snap_cli._frame_json import _bounded_depth, _finite_float, _invalid_constant + +from .dataset_contract import _unique_object + + +class FrameBudget(Protocol): + def remaining(self) -> float | None: ... + + +class FrameResolver(Protocol): + _frame_schemas: dict[Path, bool] + + @property + def budget(self) -> FrameBudget: ... + + def read( + self, path: Path, sql: str, values: list[object] | None = None + ) -> list[dict[str, object]]: ... + + +SCHEMA = { + "pt_snap_frame_value": [("id", "INTEGER", 1), ("frameJson", "TEXT", 0)], + "pt_snap_stack": [("stackId", "INTEGER", 1), ("frameCount", "INTEGER", 0)], + "pt_snap_stack_frame": [ + ("stackId", "INTEGER", 1), + ("frameIndex", "INTEGER", 2), + ("frameId", "INTEGER", 0), + ], + "pt_snap_event_stack": [("eventId", "INTEGER", 1), ("stackId", "INTEGER", 0)], +} + + +def attach_frames( + resolver: FrameResolver, path: Path, ids: list[int] +) -> dict[int, tuple[list[dict[str, object]], str]]: + result: dict[int, tuple[list[dict[str, object]], str]] = {} + batch_size = 256 + + if path not in resolver._frame_schemas: + valid = True + for table, columns in SCHEMA.items(): + kind = resolver.read(path, "SELECT type FROM sqlite_master WHERE name=?", [table]) + schema = resolver.read(path, f'PRAGMA table_info("{table}")') + if ( + kind != [{"type": "table"}] + or [(r["name"], r["type"], r["pk"]) for r in schema] != columns + ): + valid = False + break + resolver._frame_schemas[path] = valid + if not resolver._frame_schemas[path]: + return result + for start in range(0, len(ids), batch_size): + batch = ids[start : start + batch_size] + marks = ",".join("?" for _ in batch) + coverage = resolver.read( + path, + "SELECT e.eventId,e.stackId,s.frameCount FROM pt_snap_event_stack e " + f"LEFT JOIN pt_snap_stack s ON s.stackId=e.stackId WHERE e.eventId IN ({marks})", + list(batch), + ) + entries = resolver.read( + path, + "SELECT sf.stackId,sf.frameIndex,sf.frameId,f.frameJson FROM pt_snap_stack_frame sf " + "LEFT JOIN pt_snap_frame_value f ON f.id=sf.frameId " + f"WHERE sf.stackId IN (SELECT stackId FROM pt_snap_event_stack WHERE eventId IN ({marks})) " + "ORDER BY sf.stackId,sf.frameIndex", + list(batch), + ) + grouped: dict[int, list[dict[str, object]]] = defaultdict(list) + for row in entries: + grouped[cast(int, row["stackId"])].append(row) + decoded_stacks: dict[int, tuple[list[dict[str, object]], str] | None] = {} + for row in coverage: + resolver.budget.remaining() + event, stack, count = row["eventId"], row["stackId"], row["frameCount"] + if event not in ids or type(stack) is not int or type(count) is not int or count < 0: + continue + stack = cast(int, stack) + if stack not in decoded_stacks: + decoded_stacks[stack] = None + frames = grouped[stack] + if len(frames) != count or any( + type(frame["frameIndex"]) is not int + or frame["frameIndex"] != index + or type(frame["frameId"]) is not int + or not isinstance(frame["frameJson"], str) + for index, frame in enumerate(frames) + ): + continue + try: + decoded = [ + json.loads( + cast(str, frame["frameJson"]), + object_pairs_hook=_unique_object, + parse_constant=_invalid_constant, + parse_float=_finite_float, + ) + for frame in frames + ] + if not all(isinstance(frame, dict) for frame in decoded) or not _bounded_depth( + decoded + ): + continue + canonical = json.dumps( + decoded, + sort_keys=True, + ensure_ascii=True, + separators=(",", ":"), + allow_nan=False, + ) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + except (ValueError, TypeError, RecursionError, OverflowError): + continue + decoded_stacks[stack] = (decoded, digest) + value = decoded_stacks[stack] + if value is not None: + decoded, digest = value + result[cast(int, event)] = (decoded, digest) + return result diff --git a/src/pt_snap_cli/core/dataset_global.py b/src/pt_snap_cli/core/dataset_global.py index 4c6101e..fd8415c 100644 --- a/src/pt_snap_cli/core/dataset_global.py +++ b/src/pt_snap_cli/core/dataset_global.py @@ -471,7 +471,7 @@ def _finish_stacks(sources, grouped, params, scope, events, ordered): def _stacks(sources, low, high, items, params, scope): - if sources.dataset.ordered_frames_version != 1: + if sources.dataset.ordered_frames_version not in (1, 2): return _text_stacks(sources, low, high, items, params, scope) grouped = {} events = ordered = 0 @@ -485,13 +485,34 @@ def _stacks(sources, low, high, items, params, scope): join = " LEFT JOIN callstack cs ON t.callstackId=cs.id" else: projection += ", t.callstack" - batch = sources.read( - sources.dataset.root / item.file, - f"SELECT {projection} FROM {trace} t{join} " - "WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", - [low, high], - ) - refs = sources.events_from_rows(item, batch) + weights = None + if sources.dataset.ordered_frames_version == 2: + from .dataset_frames import attach_frames + + path = sources.dataset.root / item.file + attach_frames(sources, path, []) # Validate optional schema, no frame fetch. + text = "cs.callstack" if sources.dataset.callstack_layout == "v2" else "t.callstack" + stack_group = "" + if sources._frame_schemas[path]: + join += " LEFT JOIN pt_snap_event_stack es ON es.eventId=t.id" + stack_group = "es.stackId," + batch = sources.read( + path, + "SELECT MIN(t.id) AS id, COUNT(*) AS alloc_count, " + f"SUM(t.size) AS total_size, MAX(t.size) AS max_size FROM {trace} t{join} " + f"WHERE t.id>=0 AND t.id>=? AND t.id<=? GROUP BY {stack_group}{text} COLLATE BINARY", + [low, high], + ) + weights = {row["id"]: row for row in batch} + refs = sources.events(cast(int, row["id"]) for row in batch) + else: + batch = sources.read( + sources.dataset.root / item.file, + f"SELECT {projection} FROM {trace} t{join} " + "WHERE t.id>=0 AND t.id>=? AND t.id<=? ORDER BY t.id", + [low, high], + ) + refs = sources.events_from_rows(item, batch) for source in refs.values(): sources.budget.remaining() text = source.event.get("callstack") @@ -521,14 +542,23 @@ def _stacks(sources, low, high, items, params, scope): "frames_status": source.frames_status, }, ) - row["alloc_count"] += 1 - row["total_size"] += source.event["size"] - row["max_size"] = max(row["max_size"], source.event["size"]) + weight = ( + weights[source.event["id"]] + if weights is not None + else { + "alloc_count": 1, + "total_size": source.event["size"], + "max_size": source.event["size"], + } + ) + row["alloc_count"] += weight["alloc_count"] + row["total_size"] += weight["total_size"] + row["max_size"] = max(row["max_size"], weight["max_size"]) if row["text_kind"] != "captured" and source.text_kind == "captured": row["callstack"], row["stack_event_id"] = text, source.event["id"] row["text_kind"] = "captured" - events += 1 - ordered += source.frames is not None + events += weight["alloc_count"] + ordered += weight["alloc_count"] if source.frames is not None else 0 return _finish_stacks(sources, grouped, params, scope, events, ordered) @@ -636,7 +666,8 @@ def global_query( if template == "event": refs = sources.events(cast(int, r["id"]) for r in selected) detailed = ( - sources.dataset.ordered_frames_version == 1 or cast(int, params["stack_bytes"]) >= 0 + sources.dataset.ordered_frames_version in (1, 2) + or cast(int, params["stack_bytes"]) >= 0 ) selected = [dict(refs[cast(int, r["id"])].event) for r in selected] if detailed: diff --git a/src/pt_snap_cli/core/dataset_resolver.py b/src/pt_snap_cli/core/dataset_resolver.py index bf241a4..4004697 100644 --- a/src/pt_snap_cli/core/dataset_resolver.py +++ b/src/pt_snap_cli/core/dataset_resolver.py @@ -36,6 +36,7 @@ DatabaseSchemaError, InvalidDeviceError, InvalidParameterError, + QueryCancelledError, QueryTimeoutError, ) from pt_snap_cli.core.native_dataset_contract import ( @@ -47,7 +48,10 @@ validate_native_dataset, validate_native_scope, ) -from pt_snap_cli.core.snapshot_cache import validate_cache +from pt_snap_cli.core.snapshot_cache import ( + parse_cache_manifest, + validate_cache, +) from pt_snap_cli.core.validation_budget import ( ValidationBudget, check_budget, @@ -55,6 +59,12 @@ sqlite_timeout, validation_progress, ) +from pt_snap_cli.core.validation_receipts import ( + import_validation_active, + load_receipt, + receipts_enabled, + save_receipt, +) _FileStamp = tuple[int, int, int, int, int, int, int] _Generation = tuple[tuple[str, _FileStamp], ...] @@ -67,6 +77,7 @@ class ResolvedDataset: fingerprint: str ordered_frames_version: int | None = None generation: _Generation | None = None + content_verified: bool = True def require_unchanged(self, budget: ValidationBudget | None = None) -> None: """Guard a reused report/query resolution without repeating full validation.""" @@ -76,7 +87,7 @@ def require_unchanged(self, budget: ValidationBudget | None = None) -> None: current = _generation(self.root, self.validation.manifest, budget) if self.generation is None or current != self.generation: raise DatabaseSchemaError("Dataset members changed during operation; retry.") - if not _cacheable(current): + if self.content_verified and not _cacheable(current): if ( _content_fingerprint(self.root, self.validation.manifest, budget) != self.fingerprint @@ -140,7 +151,7 @@ def to_dict(self) -> dict[str, object]: "cross_slice_queries": True, "supported_queries": dict(DATASET_SUPPORT), "cross_slice_sources": True, - "ordered_frames_reader_version": 1, + "ordered_frames_reader_version": 2, "structured_frames_scope": "validation_only; query source coverage is separate", "devices": [ { @@ -303,7 +314,10 @@ class DatasetResolver: """ def inspect( - self, path: Path | str, *, budget: ValidationBudget | None = None + self, + path: Path | str, + *, + budget: ValidationBudget | None = None, ) -> ResolvedDataset | None: candidate = Path(path).expanduser().absolute() with _CACHE_LOCK: @@ -320,6 +334,10 @@ def inspect( _ = _VALIDATED.pop(cache_key, None) return None cache_key = root + if import_validation_active(): + raw = parse_cache_manifest((root / "manifest.json").read_bytes()) + if raw.get("format") != NATIVE_FORMAT: + return self._inspect_compatible_structure(root, raw, budget) strong_cache = os.environ.get("PT_SNAP_DATASET_CACHE") is None with _CACHE_LOCK: cached = _VALIDATED.get(root) @@ -380,16 +398,32 @@ def inspect( # Native validation already checks both ends against declared hashes. compatibility_before = ( _content_fingerprint(root, planned, budget, manifest_hash=before) - if strong_cache and not isinstance(planned, NativeManifest) + if strong_cache and (not isinstance(planned, NativeManifest) or receipts_enabled()) else None ) - validation = ( - validate_native_dataset(root, planned, budget=budget) - if isinstance(planned, NativeManifest) - else validate_dataset(root, budget=budget) + receipt = ( + load_receipt(root, "dataset", compatibility_before) + if compatibility_before is not None + else None ) + counts = ( + [receipt.get(key) for key in ("real", "boundary")] if receipt is not None else [] + ) + receipt_hit = len(counts) == 2 and all(type(n) is int and n >= 0 for n in counts) + if receipt_hit: + assert receipt is not None + real, boundary = receipt["real"], receipt["boundary"] + assert isinstance(real, int) and isinstance(boundary, int) + validation = DatasetValidation(planned, real, boundary) + else: + validation = ( + validate_native_dataset(root, planned, budget=budget) + if isinstance(planned, NativeManifest) + else validate_dataset(root, budget=budget) + ) if ( - not isinstance(planned, NativeManifest) + not receipt_hit + and not isinstance(planned, NativeManifest) and isinstance(raw, dict) and ("parserCache" in raw or "ptSnap" in raw) ): @@ -417,7 +451,7 @@ def inspect( ) from exc digest = hashlib.sha256() digest.update(before.encode("ascii")) - for device in checked_rows(validation.manifest.devices, budget): + for device in checked_rows(() if receipt_hit else validation.manifest.devices, budget): for item in checked_rows(device.slices, budget): digest.update(item.file.encode("utf-8")) # Native validation already verifies this digest before and @@ -432,7 +466,8 @@ def inspect( if wal.exists(): digest.update(b"-wal") digest.update(_hash_file(wal, budget=budget).encode("ascii")) - fingerprint = digest.hexdigest() + fingerprint = compatibility_before if receipt_hit else digest.hexdigest() + assert fingerprint is not None if compatibility_before is not None and compatibility_before != fingerprint: raise DatabaseSchemaError("Dataset content changed during inspection; retry.") if before != _hash_file(manifest_path, budget=budget): @@ -449,6 +484,16 @@ def inspect( version if type(version) is int else None, generation, ) + check_budget(budget) + save_receipt( + root, + "dataset", + fingerprint, + { + "real": validation.real_event_count, + "boundary": validation.boundary_event_count, + }, + ) if strong_cache or _cacheable(generation): entry = _ValidatedEntry( generation, @@ -475,11 +520,70 @@ def inspect( with _CACHE_LOCK: _ = _VALIDATED.pop(cache_key, None) raise DatabaseSchemaError("Invalid dataset: manifest nesting is too deep") from exc - except (DatabaseSchemaError, QueryTimeoutError): + except (DatabaseSchemaError, QueryTimeoutError, QueryCancelledError): with _CACHE_LOCK: _ = _VALIDATED.pop(cache_key, None) raise + @staticmethod + def _inspect_compatible_structure( + root: Path, raw: dict[str, object], budget: ValidationBudget | None + ) -> ResolvedDataset: + manifest = parse_manifest(raw, budget=budget) + _generation(root, manifest, budget) # Preflight all members before SQLite opens. + _prepare_compatibility_transport(root, manifest, budget) + generation = _generation(root, manifest, budget) + attestation = raw.get("parserCache", {}) + producer = raw.get("ptSnap", {}) + if not isinstance(attestation, dict) or not isinstance(producer, dict): + raise DatabaseSchemaError("Invalid cache extension") + identity = producer.get("identity", {}) + if not isinstance(identity, dict): + raise DatabaseSchemaError("Invalid producer identity") + selected = attestation.get("requestedDevice") + if selected is None: + selected = identity.get("requestedDevice") + cache_hash = raw.get("cacheHash") + if not isinstance(cache_hash, str): + raise DatabaseSchemaError("Invalid cacheHash") + checked = validate_cache(root, cache_hash, selected, budget=budget) + if checked != raw: + raise DatabaseSchemaError("Dataset manifest changed during inspection; retry.") + boundary = 0 + for device in manifest.devices: + for item in device.slices: + with ( + closing( + sqlite3.connect( + readonly_member_uri(root / item.file), + uri=True, + timeout=sqlite_timeout(budget), + ) + ) as conn, + validation_progress(conn, budget), + ): + # Count only synthetic IDs through the integer primary key; + # never fetch/encode callstacks or scan block contents. + boundary += conn.execute( + f'SELECT COUNT(*) FROM "trace_entry_{device.device_id}" WHERE id<0' # nosec B608 + ).fetchone()[0] + if raw != parse_cache_manifest( + (root / "manifest.json").read_bytes() + ) or generation != _generation(root, manifest, budget): + raise DatabaseSchemaError("Dataset changed during inspection; retry.") + extensions = raw.get("extensions", {}) + frames = extensions.get("ptSnapOrderedFrames", {}) if isinstance(extensions, dict) else {} + version = frames.get("version") if isinstance(frames, dict) else None + fingerprint = "structure-" + hashlib.sha256(repr(generation).encode()).hexdigest() + return ResolvedDataset( + root, + DatasetValidation(manifest, sum(d.event_count for d in manifest.devices), boundary), + fingerprint, + version if type(version) is int else None, + generation, + False, + ) + def completion_device_ids(self, path: Path | str) -> list[int] | None: """Bounded manifest-only hints, never evidence of dataset validity. diff --git a/src/pt_snap_cli/core/dataset_sources.py b/src/pt_snap_cli/core/dataset_sources.py index 550dfb8..afc41b8 100644 --- a/src/pt_snap_cli/core/dataset_sources.py +++ b/src/pt_snap_cli/core/dataset_sources.py @@ -8,16 +8,16 @@ import hashlib import json -import math import sqlite3 import time from bisect import bisect_right from collections import defaultdict -from collections.abc import Iterable +from collections.abc import Callable, Iterable from dataclasses import dataclass from pathlib import Path from typing import cast +from pt_snap_cli._frame_json import _bounded_depth, _finite_float, _invalid_constant from pt_snap_cli.context import DatabaseNotFoundError, SchemaVersionError from pt_snap_cli.core.context_cache import ContextCache from pt_snap_cli.core.dataset_contract import SliceRecord, _unique_object @@ -26,6 +26,7 @@ from pt_snap_cli.core.errors import ( DatabaseSchemaError, InvalidParameterError, + QueryCancelledError, QueryExecutionError, QueryTimeoutError, ) @@ -36,36 +37,12 @@ QueryLimits, ) from pt_snap_cli.query.executor import QueryExecutionError as ExecutorError -from pt_snap_cli.query.executor import QueryExecutor +from pt_snap_cli.query.executor import QueryExecutor, cancellation_options from pt_snap_cli.query.executor import QueryTimeoutError as ExecutorTimeout SOURCE_BATCH_SIZE = 256 -def _invalid_constant(value: str) -> object: - raise ValueError(f"Non-JSON numeric constant: {value}") - - -def _finite_float(value: str) -> float: - result = float(value) - if not math.isfinite(result): - raise ValueError("Non-finite frame number") - return result - - -def _bounded_depth(value: object) -> bool: - pending = [(value, 0)] - while pending: - current, depth = pending.pop() - if depth > 128: - return False - if isinstance(current, dict): - pending.extend((item, depth + 1) for item in current.values()) - elif isinstance(current, list): - pending.extend((item, depth + 1) for item in current) - return True - - @dataclass class QueryBudget: timeout_s: float | None @@ -77,6 +54,7 @@ class QueryBudget: # Per-operation derived database pages, not total SQLite temp space or RSS. # Zero retains the bounded in-Python fallback. max_scratch_db_bytes: int = DEFAULT_MAX_SCRATCH_DB_BYTES + cancelled: Callable[[], bool] | None = None @classmethod def from_limits( @@ -121,6 +99,8 @@ def consume(self, rows: list[dict[str, object]]) -> None: ) def remaining(self) -> float | None: + if self.cancelled is not None and self.cancelled(): + raise QueryCancelledError("Frame tree query cancelled by a newer request or reset.") if self.timeout_s is None: return None left = self.timeout_s - (time.monotonic() - self.started) @@ -228,13 +208,19 @@ def read( ) self.query_count += 1 return QueryExecutor(context).execute( - sql, values, timeout_s=self.budget.remaining(), consume=self._consume + sql, + values, + timeout_s=self.budget.remaining(), + consume=self._consume, + **cancellation_options(self.budget.cancelled), ) except ExecutorTimeout as exc: + self.budget.remaining() raise QueryTimeoutError( f"Query timed out after {self.budget.timeout_s} seconds" ) from exc except ExecutorError as exc: + self.budget.remaining() raise QueryExecutionError(str(exc)) from exc except (DatabaseNotFoundError, SchemaVersionError, sqlite3.DatabaseError) as exc: raise DatabaseSchemaError(str(exc)) from exc @@ -332,6 +318,22 @@ def _ensure_frames(self, item: SliceRecord, ids: list[int]) -> None: self._frames_resolved.update(pending) def _attach_frames(self, path: Path, ids: list[int], sources: dict[int, EventSource]) -> None: + if self.dataset.ordered_frames_version == 2: + from .dataset_frames import attach_frames + + decoded = attach_frames(self, path, ids) + for event, (frames, digest) in decoded.items(): + old = sources[event] + sources[event] = EventSource( + old.event, + old.slice_index, + f"{self.namespace}:frames:sha256:{digest}", + "captured", + frames, + "ordered", + old.local_stack_id, + ) + return # This is a NEW, explicit read contract, not a guessed original-producer # schema. Existing pt_snap_block_reference has formatted text only. if self.dataset.ordered_frames_version != 1: diff --git a/src/pt_snap_cli/core/dataset_support.py b/src/pt_snap_cli/core/dataset_support.py index 0c60053..3a712de 100644 --- a/src/pt_snap_cli/core/dataset_support.py +++ b/src/pt_snap_cli/core/dataset_support.py @@ -9,6 +9,7 @@ "allocator_gap": "global_peaks_same_event_counters", "active_blocks_at_event": "owner_event_active_set_cross_slice_sources", "active_memory_callstack_at_event": "owner_event_global_source_groups_before_top_n", + "active_memory_frame_tree_at_event": "owner_event_caller_paths_cross_slice_ordered_frames", "preexisting_live": "owner_event_preexisting_occupancy", "block": "proved_lifecycles_dedup_latest_observation", "freed_block_lifetime": "proved_completed_lifecycles_once", diff --git a/src/pt_snap_cli/core/error_codes.py b/src/pt_snap_cli/core/error_codes.py index 54b9ed4..ec4ac72 100644 --- a/src/pt_snap_cli/core/error_codes.py +++ b/src/pt_snap_cli/core/error_codes.py @@ -14,6 +14,7 @@ InvalidCategoryError, InvalidDeviceError, InvalidParameterError, + QueryCancelledError, QueryExecutionError, QueryTimeoutError, SkillError, @@ -32,6 +33,7 @@ DATABASE_SCHEMA_INVALID = "DATABASE_SCHEMA_INVALID" QUERY_FAILED = "QUERY_FAILED" QUERY_TIMEOUT = "QUERY_TIMEOUT" +QUERY_CANCELLED = "QUERY_CANCELLED" IMPORT_FAILED = "IMPORT_FAILED" IMPORT_BACKEND_MISSING = "IMPORT_BACKEND_MISSING" SNAPSHOT_INVALID = "SNAPSHOT_INVALID" @@ -87,6 +89,8 @@ def classify_error(exc: BaseException) -> tuple[str, str | None]: INVALID_PARAMETER, "Use a numeric --timeout, or set PT_SNAP_QUERY_TIMEOUT to a number of seconds.", ) + if isinstance(exc, QueryCancelledError): + return QUERY_CANCELLED, "Use the result of the latest frame-tree request." if isinstance(exc, QueryTimeoutError): return ( QUERY_TIMEOUT, diff --git a/src/pt_snap_cli/core/errors.py b/src/pt_snap_cli/core/errors.py index 7f841df..64a6dc2 100644 --- a/src/pt_snap_cli/core/errors.py +++ b/src/pt_snap_cli/core/errors.py @@ -56,6 +56,10 @@ class QueryTimeoutError(QueryExecutionError): pass +class QueryCancelledError(QueryExecutionError): + """An obsolete query was interrupted; no partial result is returned.""" + + class ImportToolMissingError(PtSnapCoreError): """Raised when the built-in snapshot import backend is unavailable. diff --git a/src/pt_snap_cli/core/import_service.py b/src/pt_snap_cli/core/import_service.py index 8f5df42..0525fbc 100644 --- a/src/pt_snap_cli/core/import_service.py +++ b/src/pt_snap_cli/core/import_service.py @@ -7,9 +7,13 @@ from dataclasses import replace from pathlib import Path -from pt_snap_cli.core.dataset_contract import parse_manifest, validate_dataset +from pt_snap_cli.core.dataset_contract import parse_manifest from pt_snap_cli.core.dataset_import_backend import DatasetImportBackend -from pt_snap_cli.core.dataset_resolver import DatasetResolver, _require_readonly_member +from pt_snap_cli.core.dataset_resolver import ( + DatasetResolver, + ResolvedDataset, + _require_readonly_member, +) from pt_snap_cli.core.errors import ( DatabaseSchemaError, FocusFileInvalidError, @@ -43,9 +47,15 @@ build_native_manifest, cache_identity, ) +from pt_snap_cli.core.ordered_frame_integrity import frame_member_digest, require_frame_declaration from pt_snap_cli.core.sharded_replay_service import ShardedReplayResult -from pt_snap_cli.core.snapshot_cache import parse_cache_manifest, source_hashes, validate_cache +from pt_snap_cli.core.snapshot_cache import ( + parse_cache_manifest, + source_hashes, + validate_cache, +) from pt_snap_cli.core.snapshot_import_backend import SnapshotImportBackend +from pt_snap_cli.core.validation_receipts import import_receipts, load_receipt, save_receipt logger = logging.getLogger(__name__) @@ -69,6 +79,10 @@ def __init__( self._msinsight_backend = MsinsightImportBackend() def import_snapshot(self, options: ImportOptions) -> ImportResult: + with import_receipts(): + return self._import_snapshot(options) + + def _import_snapshot(self, options: ImportOptions) -> ImportResult: self._validate_options(options) self._validate_snapshot_file(options.snapshot_file) if options.format in (COMPATIBILITY_FORMAT, "msinsight"): @@ -204,6 +218,20 @@ def _import_dataset(self, options: ImportOptions) -> ImportResult: and dataset is not None and manifest.identity == cache_identity(source_hash, options.device, capacity) ): + try: + require_frame_declaration( + parse_cache_manifest((target / "manifest.json").read_bytes()) + ) + if load_receipt(target, "native-frames", dataset.fingerprint) != {}: + for device in manifest.devices: + for item in device.slices: + frame_member_digest(target, target / item.file, device.device_id) + dataset.require_unchanged() + save_receipt(target, "native-frames", dataset.fingerprint, {}) + except (ValueError, OSError, sqlite3.DatabaseError, DatabaseSchemaError) as exc: + raise ImportExecutionError( + f"Existing native ordered-frame evidence is invalid; use --force to rebuild. Existing target and focus preserved: {exc}" + ) from exc self._recheck_source(options.snapshot_file, source_hash) focus_state = self._set_focus_if_requested(target, options) return ImportResult( @@ -239,7 +267,9 @@ def finalize(result: ShardedReplayResult) -> None: for item in result.slices: _require_readonly_member(result.directory, item.file) self._metadata_service.write(item.file, metadata) + frame_member_digest(result.directory, item.file, item.device) manifest = build_native_manifest(result, metadata, capacity) + require_frame_declaration(manifest) with (result.directory / "manifest.json").open("x", encoding="utf-8") as file: json.dump(manifest, file, indent=2) validated = DatasetResolver().inspect(result.directory) @@ -247,6 +277,8 @@ def finalize(result: ShardedReplayResult) -> None: raise ImportExecutionError( "Staged dataset validation did not produce a complete artifact." ) + self._remember_published_validation(target, validated) + save_receipt(target, "native-frames", validated.fingerprint, {}) completed, generated = metadata, result focus_state: FocusState | None = None @@ -310,9 +342,6 @@ def _import_compatible_dataset(self, options: ImportOptions) -> ImportResult: "Explicit events_per_slice does not match the existing compatible " "cache; choose a new output directory." ) - # A self-consistent seal is not proof of semantic validity. - # Use shared P0 without repeating the resolver's seal scan. - validate_dataset(target) self._recheck_source(options.snapshot_file, source_hash) return ImportResult( target, @@ -420,6 +449,22 @@ def focus(path: Path) -> None: generated.omitted_devices, ) + @staticmethod + def _remember_published_validation(target: Path, validated: ResolvedDataset) -> None: + if not validated.content_verified: + return + # Publication only renames the staged directory. The receipt is bound to + # all manifest/member bytes, so another artifact at target cannot hit it. + save_receipt( + target, + "dataset", + validated.fingerprint, + { + "real": validated.validation.real_event_count, + "boundary": validated.validation.boundary_event_count, + }, + ) + def _hash_source(self, source: Path) -> str: try: return self._metadata_service.calculate_sha256(source) diff --git a/src/pt_snap_cli/core/latest_query.py b/src/pt_snap_cli/core/latest_query.py new file mode 100644 index 0000000..9d002f3 --- /dev/null +++ b/src/pt_snap_cli/core/latest_query.py @@ -0,0 +1,42 @@ +"""Latest-request-wins coordination without sharing SQLite connections.""" + +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from threading import Event, Lock + +from .errors import QueryCancelledError + + +class LatestQuery: + def __init__(self) -> None: + self._state = Lock() + self._work = Lock() + self._active: Event | None = None + + def cancel(self) -> None: + with self._state: + if self._active is not None: + self._active.set() + + @contextmanager + def run(self) -> Iterator[Callable[[], bool]]: + token = Event() + with self._state: + if self._active is not None: + self._active.set() + self._active = token + try: + # The old task must unwind and close its own SQLite handles first. + with self._work: + self._check(token) + yield token.is_set + self._check(token) + finally: + with self._state: + if self._active is token: + self._active = None + + @staticmethod + def _check(token: Event) -> None: + if token.is_set(): + raise QueryCancelledError("Frame tree query cancelled by a newer request or reset.") diff --git a/src/pt_snap_cli/core/msinsight_export.py b/src/pt_snap_cli/core/msinsight_export.py index 6ff24e2..f819420 100644 --- a/src/pt_snap_cli/core/msinsight_export.py +++ b/src/pt_snap_cli/core/msinsight_export.py @@ -1,11 +1,12 @@ -"""Original interface-facts converter for fixed msinsight 101f65b8 compatibility. +"""Original converter using the historical msinsight 101f65b8 physical schema. No Ascend implementation is copied (its source is Mulan PSL v2). Physical -schema/hash facts: blobs 1fe50a5136f85365db55f4632d58f3cf7090431e, +schema and historical salt-order facts: blobs 1fe50a5136f85365db55f4632d58f3cf7090431e, 95612509ef990ac6b7034919dae4938cc6acf3ad and ca421c7b33f3a0050df9e7a01e666f558801bc2f. See sharded-snapshotdb docs. Only caller-owned finalized native staging is converted; existing caches are -never mutated, even under force. This is not GUI acceptance. +never mutated, even under force. Current exports use parser-v3 hashes; the pinned +parser-v2 consumer reparses them. No current v3 GUI acceptance is claimed. """ from __future__ import annotations @@ -23,20 +24,26 @@ TRACE_COLUMNS, DatasetContractError, DatasetManifest, - validate_dataset, + parse_manifest, ) -from .dataset_files import hash_file, readonly_member_uri, require_readonly_member +from .dataset_files import readonly_member_uri, require_readonly_member from .dataset_import_backend import DatasetImportBackend +from .dataset_resolver import DatasetResolver from .errors import ImportExecutionError from .import_metadata_contract import metadata_from_mapping from .models import ImportMetadata +from .ordered_frame_integrity import require_frame_declaration from .sharded_replay_service import ShardedReplayResult -from .snapshot_cache import parse_cache_manifest, seal_manifest, source_hashes, validate_cache +from .snapshot_cache import ( + parse_cache_manifest, + seal_manifest, + source_hashes, +) COMPATIBILITY_FORMAT = "compatibility-v1" DEFAULT_CAPACITY = 500000 # Independent semantic versions, not the installed CLI version or GUI cacheHash. -EXPORT_VERSION = 2 +EXPORT_VERSION = 3 REFERENCE_VERSION = 1 @@ -55,6 +62,7 @@ def compatible_identity(source_hash: str, device: int | None, capacity: int) -> "referenceContractVersion": REFERENCE_VERSION, "metadataSchemaVersion": 1, "importFormatVersion": 1, + # Historical schema/reference identity, not a current GUI-reuse claim. "targetRevision": MSINSIGHT_REVISION, } @@ -219,7 +227,6 @@ def build_compatible_manifest( cache_hash: str, ) -> dict[str, object]: devices: dict[str, object] = {} - files: dict[str, str] = {} for device in sorted({s.device for s in result.slices}): slices = [s for s in result.slices if s.device == device] devices[str(device)] = { @@ -244,18 +251,16 @@ def build_compatible_manifest( "cacheHash": cache_hash, "eventsPerSlice": capacity, "devices": devices, + "extensions": {"ptSnapOrderedFrames": {"version": 2}}, "ptSnap": { "identity": compatible_identity( metadata.source_sha256, metadata.requested_device, capacity ), "metadata": asdict(metadata), "omittedDevices": list(result.omitted_devices), - "members": files, }, } seal_manifest(result.directory, manifest, metadata.requested_device) - manifest["ptSnap"]["members"] = manifest["parserCache"]["members"] - manifest["ptSnap"]["memberHashKind"] = "parser-v1" return manifest @@ -270,8 +275,8 @@ class CompatibleCache: def inspect_owned_cache(target: Path) -> CompatibleCache: """Whole valid recognized identity only; salted hash alone is never ownership. - Base tables are attested logically. Consumer-owned tables and indexes may - change without invalidating parser data. Legacy exports retain byte checks. + Reuse checks source identity, manifest inventory and required DB structure. + Consumer-owned tables and indexes do not invalidate the cache. """ data = parse_cache_manifest((target / "manifest.json").read_bytes()) extension = data.get("ptSnap") @@ -280,9 +285,8 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: identity = extension.get("identity") if not isinstance(identity, dict): raise ValueError("manifest.ptSnap.identity: expected object") - for key in ("metadata", "members"): - if not isinstance(extension.get(key), dict): - raise ValueError(f"manifest.ptSnap.{key}: expected object") + if not isinstance(extension.get("metadata"), dict): + raise ValueError("manifest.ptSnap.metadata: expected object") if not isinstance(extension.get("omittedDevices"), list): raise ValueError("manifest.ptSnap.omittedDevices: expected array") version = identity.get("exportContractVersion") @@ -290,9 +294,8 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: raise ValueError( f"Legacy export contract v{version}; choose a new --output-dir to re-export as v{EXPORT_VERSION}" ) - # Logical seals establish consistency, not semantic validity. Keep shared P0 - # checks for every producer, including imports that do not register focus. - manifest = validate_dataset(target).manifest + # Reject declaration/identity failures before inspecting member structure. + manifest = parse_manifest(data) extension = data["ptSnap"] metadata = metadata_from_mapping(extension["metadata"]) identity = extension["identity"] @@ -310,31 +313,23 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: ): raise ValueError("Metadata must match physical v1 layout") paths = {s.file for d in manifest.devices for s in d.slices} + require_frame_declaration(data) + resolved = DatasetResolver().inspect(target) + if resolved is None: + raise ValueError("Compatible dataset is missing") + if parse_cache_manifest((target / "manifest.json").read_bytes()) != data: + raise ValueError("Compatible manifest changed during inspection") if any((target / (relative + "-journal")).exists() for relative in paths): raise ValueError( "Snapshot member has a rollback journal; a writer or recovery may be active, retry later" ) - if set(extension["members"]) != paths: - raise ValueError("Incomplete member attestation") expected = {"manifest.json", "parse.log", *paths, *(str(Path(p).parent) for p in paths)} expected.update(relative + suffix for relative in paths for suffix in ("-wal", "-shm")) actual = {p.relative_to(target).as_posix() for p in target.rglob("*")} if not actual <= expected or any(p.is_symlink() for p in target.rglob("*")): raise ValueError("Unrecognized files or aliases in compatible cache") - logical_members = None - if extension.get("memberHashKind") == "parser-v1": - checked = validate_cache(target, data["cacheHash"], metadata.requested_device) - if checked != data: - raise ValueError("Compatible manifest changed during inspection") - # validate_cache checks the producer seal even without parserCache. - logical_members = checked["ptSnap"]["members"] - elif "memberHashKind" in extension: - raise ValueError("Unsupported member hash contract") for relative in paths: member = target / relative - digest = logical_members[relative] if logical_members is not None else hash_file(member) - if digest != extension["members"][relative]: - raise ValueError("Compatible member hash changed") require_readonly_member(target, member, immutable=True) with closing(sqlite3.connect(readonly_member_uri(member), uri=True)) as conn: conn.row_factory = sqlite3.Row @@ -364,4 +359,5 @@ def inspect_owned_cache(target: Path) -> CompatibleCache: raise ValueError("Device selection disagrees with identity") if Path(manifest.source_file).name != metadata.source_name: raise ValueError("Source path disagrees with metadata") + resolved.require_unchanged() return CompatibleCache(manifest, identity, metadata, tuple(omitted)) diff --git a/src/pt_snap_cli/core/native_dataset_contract.py b/src/pt_snap_cli/core/native_dataset_contract.py index ffbd8bc..cc687ae 100644 --- a/src/pt_snap_cli/core/native_dataset_contract.py +++ b/src/pt_snap_cli/core/native_dataset_contract.py @@ -53,7 +53,7 @@ # Semantic contracts, deliberately independent of the installed CLI __version__. NATIVE_FORMAT_VERSION = 1 NATIVE_IMPORT_VERSION = 1 -NATIVE_EXTENSION_VERSION = 1 +NATIVE_EXTENSION_VERSION = 2 @dataclass(frozen=True) @@ -314,6 +314,11 @@ def validate_native_dataset( manifest.metadata.requested_device, manifest.events_per_slice, ) + # Read old text-only artifacts; import reuse still requires the current + # identity in ImportService. Optional frame support is declared separately. + extension = manifest.identity.get("extensionContractVersion") + if extension in (1, NATIVE_EXTENSION_VERSION): + expected_identity["extensionContractVersion"] = extension _need( manifest.identity == expected_identity, "identity", @@ -624,5 +629,6 @@ def build_native_manifest( "identity": cache_identity(metadata.source_sha256, metadata.requested_device, capacity), "metadata": asdict(metadata), "omittedDevices": list(result.omitted_devices), + "extensions": {"ptSnapOrderedFrames": {"version": 2}}, "devices": devices, } diff --git a/src/pt_snap_cli/core/ordered_frame_integrity.py b/src/pt_snap_cli/core/ordered_frame_integrity.py new file mode 100644 index 0000000..5339d97 --- /dev/null +++ b/src/pt_snap_cli/core/ordered_frame_integrity.py @@ -0,0 +1,99 @@ +"""Producer-owned frame validation, separate from consumer/base-table seals.""" + +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +from pt_snap_cli._frame_json import _bounded_depth, _finite_float, _invalid_constant + +from .dataset_contract import _unique_object +from .dataset_files import readonly_member_uri, require_readonly_member +from .dataset_frames import SCHEMA + + +def require_frame_declaration(manifest: dict[str, object]) -> None: + extensions = manifest.get("extensions") + frames = extensions.get("ptSnapOrderedFrames") if isinstance(extensions, dict) else None + if ( + not isinstance(frames, dict) + or type(frames.get("version")) is not int + or frames["version"] != 2 + ): + raise ValueError("Producer ordered-frame declaration is missing or unsupported") + + +def frame_member_digest(root: Path, member: Path, device: int) -> str: + """Validate normalized evidence and stream its exact logical content. + + Event coverage may be partial: unencodable optional evidence is intentionally + omitted by the writer. Every stored reference and frame must still be valid. + Consumer tables, columns and indexes outside this extension are not hashed. + """ + if type(device) is not int or device < 0: + raise ValueError("Invalid ordered-frame device") + require_readonly_member(root, member, immutable=True) + digest = hashlib.sha256() + with closing(sqlite3.connect(readonly_member_uri(member), uri=True)) as conn: + conn.execute("PRAGMA query_only=ON") + conn.execute("BEGIN") + for table, schema in SCHEMA.items(): + if conn.execute("SELECT type FROM sqlite_master WHERE name=?", (table,)).fetchall() != [ + ("table",) + ]: + raise ValueError(f"Ordered-frame table is missing: {table}") + actual = [(r[1], r[2], r[5]) for r in conn.execute(f'PRAGMA table_info("{table}")')] + if actual != schema: + raise ValueError(f"Invalid ordered-frame schema: {table}") + for column, kind, _ in schema: + if ( + kind == "INTEGER" + and conn.execute( + f'SELECT 1 FROM "{table}" WHERE typeof("{column}") != \'integer\' OR "{column}" < 0 LIMIT 1' + ).fetchone() + ): + raise ValueError(f"Invalid ordered-frame integer: {table}.{column}") + + checks = ( + "SELECT 1 FROM pt_snap_stack_frame f LEFT JOIN pt_snap_stack s ON s.stackId=f.stackId " + "LEFT JOIN pt_snap_frame_value v ON v.id=f.frameId WHERE s.stackId IS NULL OR v.id IS NULL LIMIT 1", + "SELECT 1 FROM pt_snap_stack s LEFT JOIN (SELECT stackId,COUNT(*) AS n,MIN(frameIndex) AS lo," + "MAX(frameIndex) AS hi FROM pt_snap_stack_frame GROUP BY stackId) f ON f.stackId=s.stackId " + "WHERE s.frameCount!=COALESCE(f.n,0) OR (s.frameCount>0 AND (f.lo!=0 OR f.hi!=s.frameCount-1)) LIMIT 1", + f"SELECT 1 FROM pt_snap_event_stack e LEFT JOIN pt_snap_stack s ON s.stackId=e.stackId " + f'LEFT JOIN "trace_entry_{device}" t ON t.id=e.eventId ' + "WHERE s.stackId IS NULL OR t.id IS NULL LIMIT 1", + ) + for sql in checks: + if conn.execute(sql).fetchone(): + raise ValueError("Invalid ordered-frame reference, frame count or sequence") + + for table, schema in SCHEMA.items(): + columns = ",".join(f'"{name}"' for name, _, _ in schema) + keys = ",".join(f'"{name}"' for name, _, pk in schema if pk) + digest.update(table.encode("ascii") + b"\n") + for row in conn.execute(f'SELECT {columns} FROM "{table}" ORDER BY {keys}'): + if table == "pt_snap_frame_value": + if not isinstance(row[1], str): + raise ValueError("Ordered frame must contain JSON text") + try: + frame = json.loads( + row[1], + object_pairs_hook=_unique_object, + parse_constant=_invalid_constant, + parse_float=_finite_float, + ) + if not isinstance(frame, dict) or not _bounded_depth([frame]): + raise ValueError("Ordered frame must be a bounded JSON object") + except (RecursionError, OverflowError) as exc: + raise ValueError("Invalid ordered-frame JSON") from exc + digest.update( + json.dumps( + row, ensure_ascii=True, separators=(",", ":"), allow_nan=False + ).encode("ascii") + ) + digest.update(b"\n") + return digest.hexdigest() diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index 83ad1c5..21deaad 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -12,6 +12,7 @@ from pt_snap_cli.core.context_cache import ContextCache from pt_snap_cli.core.dataset_attribution import event_attribution, summarize_dataset_stacks from pt_snap_cli.core.dataset_contract import QueryScope +from pt_snap_cli.core.dataset_frame_tree import frame_tree from pt_snap_cli.core.dataset_global import global_query from pt_snap_cli.core.dataset_resolver import DatasetResolver, ResolvedDataset from pt_snap_cli.core.dataset_sources import DatasetSourceResolver, QueryBudget @@ -29,6 +30,7 @@ TemplateRenderError, ) from pt_snap_cli.core.focus_service import FocusService +from pt_snap_cli.core.latest_query import LatestQuery from pt_snap_cli.core.models import ( QueryResult, ResolvedFocus, @@ -39,7 +41,7 @@ from pt_snap_cli.core.query_limits import QueryLimits, resolve_query_limits from pt_snap_cli.core.stack_summary import summarize_stacks from pt_snap_cli.query.executor import QueryExecutionError as ExecutorQueryExecutionError -from pt_snap_cli.query.executor import QueryExecutor +from pt_snap_cli.query.executor import QueryExecutor, cancellation_options from pt_snap_cli.query.executor import QueryTimeoutError as ExecutorQueryTimeoutError from pt_snap_cli.query.executor import TemplateRenderError as ExecutorTemplateRenderError from pt_snap_cli.query.registry import ( @@ -106,6 +108,8 @@ def __init__( self._owns_context_cache: bool = context_cache is None self._executor: QueryExecutor | None = None self._executor_context: Context | None = None + self._executor_source: QueryExecutor | None = None + self._frame_tree_queries = LatestQuery() @property def context_cache(self) -> ContextCache: @@ -117,13 +121,16 @@ def close(self) -> None: An injected cache is borrowed and must be closed by its owner. """ + self._frame_tree_queries.cancel() self._executor = None self._executor_context = None + self._executor_source = None if self._owns_context_cache: self._context_cache.close() def invalidate_context_cache(self, db_path: Path | str | None = None) -> None: """Drop a cached context (or all of them when ``db_path`` is None).""" + self._frame_tree_queries.cancel() self._context_cache.invalidate(db_path) def list_templates( @@ -243,6 +250,75 @@ def execute_query( query_limits: QueryLimits | None = None, _budget: QueryBudget | None = None, _resolved: _ResolvedQuery | None = None, + ) -> QueryResult: + """Frame-tree calls on this service supersede unfinished frame-tree calls. + + Separate service instances do not cancel one another. Other templates + retain their existing ownership/concurrency contract. + """ + if template != "active_memory_frame_tree_at_event": + return self._execute_query( + template, + params, + db_path, + device_id, + start_dir, + max_rows, + exact_total=exact_total, + timeout_s=timeout_s, + slice_index=slice_index, + query_limits=query_limits, + _budget=_budget, + _resolved=_resolved, + ) + with self._frame_tree_queries.run() as cancelled: + budget = _budget if _budget is not None else self._new_budget(timeout_s, query_limits) + previous = budget.cancelled + budget.cancelled = lambda: cancelled() or (previous is not None and previous()) + # Each task opens and closes its handles in its executing thread. + # The borrowed/standalone cache must never move between click workers. + worker = QueryService(self._focus_service, query_limits=self._query_limits) + # Retain override detection and custom templates, never the source connection. + worker._executor = self._executor + worker._executor_source = self._executor + try: + result = worker._execute_query( + template, + params, + db_path, + device_id, + start_dir, + max_rows, + exact_total=exact_total, + timeout_s=timeout_s, + slice_index=slice_index, + query_limits=query_limits, + _budget=budget, + _resolved=_resolved, + ) + budget.remaining() + return result + finally: + try: + worker.close() + finally: + budget.cancelled = previous + + def _execute_query( + self, + template: str, + params: dict[str, Any] | None = None, + db_path: Path | str | None = None, + device_id: int | None = None, + start_dir: Path | None = None, + max_rows: int | None = None, + *, + exact_total: bool = False, + timeout_s: float | None = None, + slice_index: int | None = None, + query_limits: QueryLimits | None = None, + _budget: QueryBudget | None = None, + _resolved: _ResolvedQuery | None = None, ) -> QueryResult: budget = _budget if _budget is not None else self._new_budget(timeout_s, query_limits) timeout_s, started = budget.timeout_s, budget.started @@ -285,9 +361,13 @@ def execute_query( if slice_index is not None: dataset.paths(QueryScope("slice", selected.device_id, slice_index)) implementation = ( - event_attribution - if template in ("active_blocks_at_event", "active_memory_callstack_at_event") - else global_query + frame_tree + if template == "active_memory_frame_tree_at_event" + else ( + event_attribution + if template in ("active_blocks_at_event", "active_memory_callstack_at_event") + else global_query + ) ) result = implementation( sources, @@ -325,6 +405,7 @@ def execute_query( device_id=target_device, max_rows=max_rows, timeout_s=budget.remaining(), + **cancellation_options(budget.cancelled), ) offset = _validated_offset(template, query_params) rank_cap = _finite_top_n(template, query_params) @@ -338,6 +419,7 @@ def execute_query( device_id=target_device, timeout_s=_remaining_timeout(timeout_s, started), ignore_row_window=True, + **cancellation_options(budget.cancelled), ) total_is_exact = True if offset == 0: @@ -349,8 +431,10 @@ def execute_query( except ExecutorTemplateRenderError as exc: raise TemplateRenderError(str(exc)) from exc except ExecutorQueryTimeoutError as exc: + budget.remaining() raise QueryTimeoutError(f"Query timed out after {timeout_s} seconds") from exc except ExecutorQueryExecutionError as exc: + budget.remaining() if get_query(template) is None: raise TemplateNotFoundError(f"Template '{template}' not found") from exc raise QueryExecutionError(str(exc)) from exc @@ -416,7 +500,12 @@ def _get_executor(self, ctx: Context) -> QueryExecutor: template directory of its own. """ if self._executor is None or self._executor_context is not ctx: - self._executor = QueryExecutor(ctx) + self._executor = ( + self._executor_source.clone_for_context(ctx) + if self._executor_source is not None + else QueryExecutor(ctx) + ) + self._executor_source = None self._executor_context = ctx return self._executor diff --git a/src/pt_snap_cli/core/report_service.py b/src/pt_snap_cli/core/report_service.py index d099b3d..6d5057e 100644 --- a/src/pt_snap_cli/core/report_service.py +++ b/src/pt_snap_cli/core/report_service.py @@ -56,7 +56,7 @@ def memory_tree_report( timeout_s: float | None = None, query_limits: QueryLimits | None = None, ) -> QueryResult: - """Return frame occupancy through the shared standalone query contract.""" + """Return standalone or dataset frame occupancy through the shared query.""" return self._query_service.execute_query( "active_memory_frame_tree_at_event", params={"event_id": event_id, "include_static": include_static, "min_size": min_size}, diff --git a/src/pt_snap_cli/core/snapshot_cache.py b/src/pt_snap_cli/core/snapshot_cache.py index 5f23341..49daccc 100644 --- a/src/pt_snap_cli/core/snapshot_cache.py +++ b/src/pt_snap_cli/core/snapshot_cache.py @@ -7,8 +7,6 @@ import os import re import sqlite3 -import sys -from collections import OrderedDict from contextlib import closing from pathlib import Path from typing import Any @@ -16,12 +14,11 @@ from .validation_budget import ( ValidationBudget, check_budget, - checked_rows, sqlite_timeout, validation_progress, ) -PARSER_HASH_SALT = b"mem_snapshot_parser_v2" +PARSER_HASH_SALT = b"mem_snapshot_parser_v3" CACHE_CONTRACT_VERSION = 1 TRACE_COLUMNS = ( "id", @@ -37,37 +34,6 @@ BLOCK_COLUMNS = ("id", "address", "size", "requestedSize", "state", "allocEventId", "freeEventId") -class _RowEncoder: - """Bound repeated callstack encoding; preserve the v1 digest bytes.""" - - def __init__(self) -> None: - self.stacks: OrderedDict[str, bytes] = OrderedDict() - self.size = 0 - - def encode(self, row: tuple[Any, ...], inline_stack: bool) -> bytes: - if not inline_stack or not isinstance(row[-1], str): - return json.dumps( - row, ensure_ascii=True, separators=(",", ":"), allow_nan=False - ).encode() - stack = row[-1] - encoded = self.stacks.get(stack) - if encoded is None: - encoded = json.dumps(stack, ensure_ascii=True, allow_nan=False).encode() - cost = sys.getsizeof(stack) + sys.getsizeof(encoded) - if cost <= 8 * 1024 * 1024: - while self.stacks and ( - self.size + cost > 8 * 1024 * 1024 or len(self.stacks) >= 2048 - ): - key, value = self.stacks.popitem(last=False) - self.size -= sys.getsizeof(key) + sys.getsizeof(value) - self.stacks[stack] = encoded - self.size += cost - else: - self.stacks.move_to_end(stack) - prefix = json.dumps(row[:-1], separators=(",", ":"), allow_nan=False).encode() - return prefix[:-1] + b"," + encoded + b"]" - - def source_hashes(source: Path) -> tuple[str, str]: """Return raw and salted SHA-256 in one streaming pass, without unpickling.""" raw = hashlib.sha256() @@ -112,13 +78,11 @@ def _member(directory: Path, relative: str) -> Path: return candidate -def _table_rows( +def _validate_table_schema( conn: sqlite3.Connection, name: str, columns: tuple[str, ...], - where: str = "", - parameters: tuple[str, ...] = (), -) -> sqlite3.Cursor: +) -> None: if conn.execute("SELECT type FROM sqlite_master WHERE name=?", (name,)).fetchone() != ( "table", ): @@ -131,75 +95,48 @@ def _table_rows( raise ValueError(f"Invalid parser column: {name}.{column}") if "id" in columns and [row[1] for row in schema if row[5]] != ["id"]: raise ValueError(f"Missing parser primary key: {name}.id") - selected = ",".join(f'"{column}"' for column in columns) - order = '"id"' if "id" in columns else selected - # Identifiers and the optional predicate are internal constants/validated device IDs. - return conn.execute( - f'SELECT {selected} FROM "{name}" {where} ORDER BY {order}', parameters - ) # nosec B608 -def member_digest( +def validate_member_structure( path: Path, device: str, start: int, end: int, *, budget: ValidationBudget | None = None -) -> str: - """Hash fixed base columns; consumer columns, tables, indexes, views and triggers are allowed.""" - if re.fullmatch(r"0|[1-9][0-9]*", str(device)) is None: - raise ValueError("Invalid device ID") - digest = hashlib.sha256() - encoder = _RowEncoder() - # Consumers may start writing after the sidecar check. Always use SQLite's - # locking/WAL protocol and one read snapshot, even for a currently closed DB. +) -> None: + """Check required schema and indexed endpoints, without scanning rows/pages.""" sidecars = [Path(str(path) + suffix) for suffix in ("-wal", "-shm", "-journal")] if any(item.is_symlink() for item in sidecars): raise ValueError("Snapshot has an aliased sidecar") if sidecars[2].exists(): - raise ValueError( - "Snapshot has a rollback journal; a writer or recovery may be active, retry later" - ) - uri = path.resolve().as_uri() + "?mode=ro" + raise ValueError("Snapshot has a rollback journal; retry later") with ( - closing(sqlite3.connect(uri, uri=True, timeout=sqlite_timeout(budget))) as conn, + closing( + sqlite3.connect( + path.resolve().as_uri() + "?mode=ro", uri=True, timeout=sqlite_timeout(budget) + ) + ) as conn, validation_progress(conn, budget), ): conn.execute("PRAGMA query_only=ON") conn.execute("BEGIN") - if conn.execute("PRAGMA quick_check").fetchall() != [("ok",)]: - raise ValueError("Snapshot member failed integrity check") - tables = ((f"trace_entry_{device}", TRACE_COLUMNS), (f"block_{device}", BLOCK_COLUMNS)) - for table, columns in tables: - digest.update(json.dumps((table, columns), separators=(",", ":")).encode()) - for row in checked_rows(_table_rows(conn, table, columns), budget): - digest.update(encoder.encode(row, columns == TRACE_COLUMNS)) - digest.update(b"\n") - count, low, high = conn.execute( - f'SELECT COUNT(*), MIN(id), MAX(id) FROM "trace_entry_{device}" WHERE id>=0' # nosec B608 - device validated above - ).fetchone() - if (count, low, high) != (end - start + 1, start, end): - raise ValueError("Snapshot event range is incomplete") - for row in _table_rows( - conn, - "dictionary", - ("table", "column", "key", "value"), - 'WHERE "table" IN (?, ?)', - tuple(table for table, _ in tables), + for name, columns in ( + (f"trace_entry_{device}", TRACE_COLUMNS), + (f"block_{device}", BLOCK_COLUMNS), + ("dictionary", ("table", "column", "key", "value")), ): - digest.update(json.dumps(row, ensure_ascii=True, separators=(",", ":")).encode()) - digest.update(b"\n") - for table, column in ((f"trace_entry_{device}", "action"), (f"block_{device}", "state")): - # Identifiers are internal constants with the device validated above. - missing = conn.execute( - f'SELECT 1 FROM (SELECT DISTINCT "{column}" AS value FROM "{table}") AS base WHERE NOT EXISTS ' # nosec B608 - '(SELECT 1 FROM dictionary AS dict WHERE dict."table"=? AND dict."column"=? ' - 'AND dict."key"=CAST(base.value AS TEXT)) LIMIT 1', - (table, column), - ).fetchone() - if missing: - raise ValueError("Snapshot dictionary reference is missing") - return digest.hexdigest() + _validate_table_schema(conn, name, columns) + low = conn.execute( + f'SELECT id FROM "trace_entry_{device}" WHERE id>=0 ORDER BY id LIMIT 1' + ).fetchone() # nosec B608 + high = conn.execute( + f'SELECT id FROM "trace_entry_{device}" WHERE id>=0 ORDER BY id DESC LIMIT 1' + ).fetchone() # nosec B608 + if low != (start,) or high != (end,): + raise ValueError("Snapshot event range is incomplete") def parser_members( - directory: Path, manifest: dict[str, Any], *, budget: ValidationBudget | None = None + directory: Path, + manifest: dict[str, Any], + *, + budget: ValidationBudget | None = None, ) -> dict[str, str]: """Validate the complete manifest and all DBs; never deserialize the source.""" if _integer(manifest.get("schemaVersion"), 1) != 1: @@ -240,13 +177,14 @@ def parser_members( relative = item["file"] if relative in members: raise ValueError("Duplicate snapshot member") - members[relative] = member_digest( - _member(directory, relative), device, start, end, budget=budget - ) + path = _member(directory, relative) + validate_member_structure(path, device, start, end, budget=budget) + members[relative] = "" return members def seal_manifest(directory: Path, manifest: dict[str, Any], device: int | None = None) -> None: + """Record structural inventory and device selection, without content digests.""" manifest["parserCache"] = { "version": CACHE_CONTRACT_VERSION, "requestedDevice": device, @@ -309,33 +247,25 @@ def validate_cache( raise ValueError("Snapshot device selection changed") members = parser_members(directory, manifest, budget=budget) attestation = manifest.get("parserCache") - # Legacy producers have no logical attestation; still validate every base table and range. + # Historical digest declarations are accepted as member inventories only. if attestation is not None and ( _integer(attestation.get("version"), 1) != CACHE_CONTRACT_VERSION - or attestation.get("members") != members + or not isinstance(attestation.get("members"), dict) + or set(attestation["members"]) != set(members) ): raise ValueError("Snapshot parser data changed") - # Both attestations must hold; parserCache must not hide an older producer seal. + # Historical producer declarations must list the same manifest members. producer = manifest.get("ptSnap", {}) if "memberHashKind" in producer and producer["memberHashKind"] != "parser-v1": raise ValueError("Unsupported member hash contract") - if producer.get("memberHashKind") == "parser-v1" and producer.get("members") != members: - raise ValueError("Snapshot producer member hash changed") + if producer.get("memberHashKind") == "parser-v1" and ( + not isinstance(producer.get("members"), dict) or set(producer["members"]) != set(members) + ): + raise ValueError("Snapshot producer member inventory changed") if "members" in producer and producer.get("memberHashKind") != "parser-v1": physical = producer["members"] if not isinstance(physical, dict) or set(physical) != set(members): raise ValueError("Invalid legacy snapshot member inventory") - from .dataset_files import hash_file - - for relative, expected in physical.items(): - member = _member(directory, relative) - wal = Path(str(member) + "-wal") - # A read-only open can create an empty WAL and SHM. They contain no - # committed rows; only the DB bytes need the legacy physical digest. - if Path(str(member) + "-journal").exists() or (wal.exists() and wal.stat().st_size): - raise ValueError("Legacy physical snapshot hash cannot validate live sidecars") - if hash_file(member, budget=budget) != expected: - raise ValueError("Legacy snapshot member hash changed") if path.read_bytes() != original: raise ValueError("Snapshot manifest changed during validation") return manifest diff --git a/src/pt_snap_cli/core/validation_budget.py b/src/pt_snap_cli/core/validation_budget.py index ec60cf0..2bb0dc3 100644 --- a/src/pt_snap_cli/core/validation_budget.py +++ b/src/pt_snap_cli/core/validation_budget.py @@ -12,7 +12,7 @@ from contextlib import contextmanager from typing import Protocol, TypeVar -from .errors import QueryTimeoutError +from .errors import QueryCancelledError, QueryTimeoutError class ValidationBudget(Protocol): @@ -50,16 +50,16 @@ def checked_rows(rows: Iterable[_T], budget: ValidationBudget | None) -> Iterato def validation_progress( conn: sqlite3.Connection, budget: ValidationBudget | None ) -> Iterator[None]: - """Interrupt SQLite and preserve QUERY_TIMEOUT rather than a schema error.""" + """Preserve timeout/cancellation instead of reporting an invalid schema.""" check_budget(budget) - timed_out: QueryTimeoutError | None = None + interrupted: QueryTimeoutError | QueryCancelledError | None = None def progress() -> int: - nonlocal timed_out + nonlocal interrupted try: check_budget(budget) - except QueryTimeoutError as exc: - timed_out = exc + except (QueryTimeoutError, QueryCancelledError) as exc: + interrupted = exc return 1 return 0 @@ -69,8 +69,8 @@ def progress() -> int: try: yield except sqlite3.DatabaseError: - if timed_out is not None: - raise timed_out from None + if interrupted is not None: + raise interrupted from None check_budget(budget) raise check_budget(budget) diff --git a/src/pt_snap_cli/core/validation_receipts.py b/src/pt_snap_cli/core/validation_receipts.py new file mode 100644 index 0000000..9c48be9 --- /dev/null +++ b/src/pt_snap_cli/core/validation_receipts.py @@ -0,0 +1,103 @@ +"""Disposable, content-addressed validation receipts for repeated imports. + +Receipts live outside datasets. They never substitute file timestamps for byte +hashes, and are only consulted inside an import operation. Bump VERSION whenever +the validation semantics change, independently of the package version. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from pathlib import Path + +VERSION = 1 +_ENABLED: ContextVar[bool] = ContextVar("import_validation_receipts", default=False) + + +@contextmanager +def import_receipts() -> Iterator[None]: + token = _ENABLED.set(True) + try: + yield + finally: + _ENABLED.reset(token) + + +def _path(root: Path, scope: str) -> Path | None: + if not receipts_enabled(): + return None + key = hashlib.sha256(str(root).encode("utf-8")).hexdigest() + try: + return ( + Path.home() + / ".cache" + / "pt-snap-cli" + / f"validation-v{VERSION}" + / f"{key}-{scope}.json" + ) + except (OSError, RuntimeError): + return None + + +def import_validation_active() -> bool: + """Import admission policy is independent of receipt/cache configuration.""" + return _ENABLED.get() + + +def receipts_enabled() -> bool: + return _ENABLED.get() and os.environ.get("PT_SNAP_DATASET_CACHE") is None + + +def _encoded(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("utf-8") + + +def load_receipt(root: Path, scope: str, fingerprint: str) -> dict[str, object] | None: + path = _path(root, scope) + if path is None: + return None + try: + if path.is_symlink() or path.stat().st_size > 4096: + return None + envelope = json.loads(path.read_bytes()) + data = envelope["data"] + if ( + envelope["sha256"] != hashlib.sha256(_encoded(data)).hexdigest() + or data["fingerprint"] != fingerprint + or not isinstance(data["payload"], dict) + ): + return None + return data["payload"] + except (OSError, ValueError, KeyError, TypeError, RecursionError): + return None + + +def save_receipt(root: Path, scope: str, fingerprint: str, payload: dict[str, object]) -> None: + path = _path(root, scope) + if path is None: + return + temporary: Path | None = None + try: + path.parent.mkdir(parents=True, exist_ok=True) + data = {"fingerprint": fingerprint, "payload": payload} + raw = _encoded({"data": data, "sha256": hashlib.sha256(_encoded(data)).hexdigest()}) + # One bounded record per dataset/scope, atomically replaced on change. + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as file: + temporary = Path(file.name) + file.write(raw) + os.replace(temporary, path) + except OSError: + # Read-only homes, full disks and stale receipts affect speed, not validity. + pass + finally: + if temporary is not None: + try: + temporary.unlink(missing_ok=True) + except OSError: + pass diff --git a/src/pt_snap_cli/query/executor.py b/src/pt_snap_cli/query/executor.py index b9ff0d7..ffc43fd 100644 --- a/src/pt_snap_cli/query/executor.py +++ b/src/pt_snap_cli/query/executor.py @@ -2,13 +2,14 @@ from __future__ import annotations +import copy import re import sqlite3 import time from collections.abc import Callable, Generator, Mapping from contextlib import contextmanager from pathlib import Path -from typing import Any +from typing import Any, TypedDict from jinja2 import Environment, StrictUndefined, Template, TemplateSyntaxError @@ -90,6 +91,14 @@ def reported_sql_limit( return injected_row_limit(validated, max_rows) +class _CancellationOptions(TypedDict, total=False): + cancelled: Callable[[], bool] + + +def cancellation_options(cancelled: Callable[[], bool] | None) -> _CancellationOptions: + return {"cancelled": cancelled} if cancelled is not None else {} + + class QueryExecutionError(Exception): """Raised when query execution fails.""" @@ -115,7 +124,9 @@ def _is_sqlite_interrupt(error: BaseException) -> bool: @contextmanager def _sqlite_timeout( - conn: sqlite3.Connection, timeout_s: float | None + conn: sqlite3.Connection, + timeout_s: float | None, + cancelled: Callable[[], bool] | None = None, ) -> Generator[None, None, None]: """Abort the current SQLite statement after ``timeout_s`` wall-clock seconds. @@ -124,13 +135,18 @@ def _sqlite_timeout( the query should run without a time bound. The handler is always cleared so persistent connections do not leak a deadline into the next statement. """ - if timeout_s is None or timeout_s <= 0: + if (timeout_s is None or timeout_s <= 0) and cancelled is None: yield return - deadline = time.monotonic() + float(timeout_s) + deadline = ( + time.monotonic() + float(timeout_s) if timeout_s is not None and timeout_s > 0 else None + ) def _handler() -> int: - return 1 if time.monotonic() >= deadline else 0 + return int( + (cancelled is not None and cancelled()) + or (deadline is not None and time.monotonic() >= deadline) + ) conn.set_progress_handler(_handler, _PROGRESS_HANDLER_INTERVAL) try: @@ -329,6 +345,12 @@ def render( return rendered_sql + def clone_for_context(self, context: Context) -> QueryExecutor: + """Copy runtime registrations to a worker without borrowing its connection.""" + worker = QueryExecutor(context) + worker._configs = copy.deepcopy(self._configs) + return worker + def execute( self, sql: str, @@ -336,6 +358,7 @@ def execute( timeout_s: float | None = None, *, consume: Callable[[list[dict[str, Any]]], None] | None = None, + cancelled: Callable[[], bool] | None = None, ) -> list[dict[str, Any]]: """Execute raw SQL query. @@ -353,7 +376,7 @@ def execute( """ try: with self._context.connect() as conn: - with _sqlite_timeout(conn, timeout_s): + with _sqlite_timeout(conn, timeout_s, cancelled): cursor = conn.cursor() try: if params: @@ -386,13 +409,19 @@ def execute( except Exception as e: raise QueryExecutionError(f"Query execution failed: {e}") from e - def count(self, sql: str, timeout_s: float | None = None) -> int: + def count( + self, + sql: str, + timeout_s: float | None = None, + *, + cancelled: Callable[[], bool] | None = None, + ) -> int: """Count the rows produced by a rendered query without materializing them.""" query = sql.strip().rstrip(";").rstrip() count_sql = f"SELECT COUNT(*) FROM ({query}) AS pt_snap_count" try: with self._context.connect() as conn: - with _sqlite_timeout(conn, timeout_s): + with _sqlite_timeout(conn, timeout_s, cancelled): cursor = conn.cursor() cursor.execute(count_sql) row = cursor.fetchone() @@ -456,6 +485,7 @@ def execute_template_page( timeout_s: float | None = None, *, probe_has_more: bool = True, + cancelled: Callable[[], bool] | None = None, ) -> tuple[list[dict[str, Any]], bool, int | None]: """Execute a named template and report whether more rows exist. @@ -480,7 +510,11 @@ def execute_template_page( fetch_sql = ( bump_trailing_limit(sql, 1) if probe_has_more and applied_limit is not None else sql ) - rows = self.execute(fetch_sql, timeout_s=timeout_s) + rows = self.execute( + fetch_sql, + timeout_s=timeout_s, + **cancellation_options(cancelled), + ) has_more = applied_limit is not None and len(rows) > applied_limit if has_more and applied_limit is not None: rows = rows[:applied_limit] @@ -495,6 +529,7 @@ def count_template( timeout_s: float | None = None, *, ignore_row_window: bool = False, + cancelled: Callable[[], bool] | None = None, ) -> int: """Count rows returned by a named template without materializing them. @@ -518,7 +553,7 @@ def count_template( if "top_n" in template.parameters: count_params["top_n"] = -1 sql = self.render(template, count_params, device_id) - return self.count(sql, timeout_s=timeout_s) + return self.count(sql, timeout_s=timeout_s, **cancellation_options(cancelled)) def execute_on_all_devices( self, diff --git a/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml b/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml index 1c8d7f1..2cb7691 100644 --- a/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml +++ b/src/pt_snap_cli/query/templates/business/active_memory_frame_tree_at_event.yaml @@ -2,9 +2,10 @@ version: "1.0" queries: active_memory_frame_tree_at_event: description: Decompose live memory after an event into a caller-first frame tree - semantics_version: 1 + semantics_version: 2 interpretation_limits: - - "Requires structured frame tables from import_format_version >= 3; old text-only databases must be re-imported." + - "Standalone databases require structured frame tables from import_format_version >= 3. Datasets resolve original allocation events across shards using ptSnapOrderedFrames version 1 or 2; absent, unknown or corrupt evidence contributes to missing stacks without reconstructing frames from text. Re-import to capture ordered frames." + - "Dataset frame_id is query-local; node_id identifies the full ordered path of filename/line/name triples. Shard-local IDs and display text are never cross-shard grouping keys. scope.source_coverage reports frame completeness separately from result row completeness." - "Uses allocation stacks of blocks live after event_id: allocEventId <= event_id and freeEventId > event_id (or absent). freeEventId is free_completed, so pending-free memory remains included." - "Traversal reverses the original innermost-first frame array. Node identity is the whole ordered caller path, preserving recursion and separating identical frames under different callers." - "size_bytes includes descendant occupancy; self_bytes counts only allocations ending at this node. size_bytes = self_bytes + sum(direct child size_bytes). Do not sum inclusive bytes across depths." diff --git a/src/pt_snap_cli/snapshot/PROVENANCE.md b/src/pt_snap_cli/snapshot/PROVENANCE.md index 40df201..57c2b48 100644 --- a/src/pt_snap_cli/snapshot/PROVENANCE.md +++ b/src/pt_snap_cli/snapshot/PROVENANCE.md @@ -299,3 +299,27 @@ Read-only queries preserve path identity, classify absent/corrupt evidence, and conserve live block, requested-byte and occupancy totals. No upstream code or new executable fixtures were incorporated; source mappings and licenses remain unchanged. This supersedes the unmerged implementation proposed in PR #207. + +## 2026-10-21: deduplicated dataset frames and cross-shard occupancy trees + +First-party ordered-frame evidence is now retained by sharded replay in an +isolated version-2 extension: raw frame JSON and ordered frame sequences are +interned independently of native text callstack IDs, with references only for +real events. Compatible conversion preserves these extension tables without +changing base inline trace columns or boundary-event normalization. Native +extension and compatible export cache identities advance; old native text-only +artifacts remain readable. Dataset trees resolve allocation owners before +caller-path grouping and preserve missing/empty/static/preexisting occupancy. +No allocator transition, event ID, source mapping, license or executable fixture +was changed, and no external implementation was incorporated. + + +## 2026-10-31: preserve text import when optional frame evidence exceeds reader bounds + +First-party sharded frame writing now applies the same bounded JSON-depth check +as readers before inserting any rows for an event's optional stack. Evidence +outside that contract is omitted as a whole, preserving native text-input +acceptance and subsequent events instead of failing publication. The small +stdlib-only validation helpers are shared outside the snapshot/core dependency +boundary. No allocator transition, event ID, base schema, license, source mapping +or executable fixture changes; no external implementation was incorporated. diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/ordered_frames.py b/src/pt_snap_cli/snapshot/tools/adaptors/ordered_frames.py new file mode 100644 index 0000000..10be117 --- /dev/null +++ b/src/pt_snap_cli/snapshot/tools/adaptors/ordered_frames.py @@ -0,0 +1,79 @@ +"""Deduplicated, optional frame evidence independent of native text stack IDs.""" + +import json +import sqlite3 + +from pt_snap_cli._frame_json import _bounded_depth + +from ...base import Frame, TraceEntry + + +class OrderedFrameWriter: + def __init__(self, connection: sqlite3.Connection): + self.connection = connection + connection.executescript( + "CREATE TABLE pt_snap_frame_value (id INTEGER PRIMARY KEY, frameJson TEXT);" + "CREATE TABLE pt_snap_stack (stackId INTEGER PRIMARY KEY, frameCount INTEGER);" + "CREATE TABLE pt_snap_stack_frame (stackId INTEGER, frameIndex INTEGER, frameId INTEGER," + " PRIMARY KEY(stackId,frameIndex));" + "CREATE TABLE pt_snap_event_stack (eventId INTEGER PRIMARY KEY, stackId INTEGER);" + ) + self.frames: dict[str, int] = {} + self.stacks: dict[tuple[int, ...], int] = {} + self.containers: dict[tuple[int, ...], tuple[object, int]] = {} + + def insert(self, event: TraceEntry) -> None: + if event.idx is None or event.idx < 0: + return + frames = event.callstack_frames() + key = tuple(map(id, frames)) + cached = self.containers.get(key) + if cached is not None: + stack_id = cached[1] + else: + try: + encoded = [ + json.dumps( + frame.to_dict() if isinstance(frame, Frame) else frame, + sort_keys=True, + ensure_ascii=True, + separators=(",", ":"), + allow_nan=False, + ) + for frame in frames + ] + # Apply the reader's depth contract to normalized JSON, including + # tuples converted to arrays, before inserting any partial stack. + decoded = [json.loads(value) for value in encoded] + if not all(isinstance(frame, dict) for frame in decoded) or not _bounded_depth( + decoded + ): + return + except (ValueError, TypeError, RecursionError, OverflowError): + # Optional evidence must not change acceptance of native text inputs. + return + ids = [] + for value in encoded: + if value not in self.frames: + frame_id = len(self.frames) + self.frames[value] = frame_id + self.connection.execute( + "INSERT INTO pt_snap_frame_value VALUES (?,?)", (frame_id, value) + ) + ids.append(self.frames[value]) + stack = tuple(ids) + if stack not in self.stacks: + stack_id = len(self.stacks) + self.stacks[stack] = stack_id + self.connection.execute( + "INSERT INTO pt_snap_stack VALUES (?,?)", (stack_id, len(stack)) + ) + self.connection.executemany( + "INSERT INTO pt_snap_stack_frame VALUES (?,?,?)", + [(stack_id, index, frame_id) for index, frame_id in enumerate(stack)], + ) + stack_id = self.stacks[stack] + self.containers[key] = (frames, stack_id) + self.connection.execute( + "INSERT INTO pt_snap_event_stack VALUES (?,?)", (event.idx, stack_id) + ) diff --git a/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py b/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py index 1d2d130..45c5d58 100644 --- a/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py +++ b/src/pt_snap_cli/snapshot/tools/adaptors/sharded_replay.py @@ -15,6 +15,7 @@ from ...base import Block, BlockState, DeviceSnapshot, TraceEntry from ...simulate import AllocatorHooker, SimulateDeviceSnapshot, SimulateHooker from .database import CallstackInterner, event2record +from .ordered_frames import OrderedFrameWriter from .snapshot2db import SnapshotDbHandler @@ -153,11 +154,13 @@ def __init__(self, item: ReplaySlice, registry: BlockRegistry): self.registry = registry self.writer = SnapshotDbHandler(str(item.file), [item.device]) self.stacks = CallstackInterner() + self.frames = OrderedFrameWriter(self.writer.db.conn) self.current_event: TraceEntry | None = None def pre_undo_event(self, wait4undo_event: TraceEntry, current_snapshot: DeviceSnapshot) -> bool: event, snapshot = wait4undo_event, current_snapshot self.current_event = event + self.frames.insert(event) self.writer.insert_event( event2record( event, diff --git a/tests/core/AGENTS.md b/tests/core/AGENTS.md index 4855dce..28d4867 100644 --- a/tests/core/AGENTS.md +++ b/tests/core/AGENTS.md @@ -57,11 +57,10 @@ The same acceptance is stored in the JUnit `dataset_acceptance` property (use The real importer exports exactly 1,000,000 events using the unchanged 500,000 default capacity. The complete installed catalog must be classified: currently -11 dataset-supported templates succeed with CLI/API result parity and independent -known-result assertions; `event_frames` and `active_memory_frame_tree_at_event` -are explicit dataset rejections. Both report commands run through the console: -`peak-memory` succeeds, while `memory-tree` preserves its documented unsupported -dataset contract. Source and every exported member remain unchanged. This is +12 dataset-supported templates succeed with CLI/API result parity and independent +known-result assertions; `event_frames` is the explicit dataset rejection. +Frame trees have additional real-import differential tests in `test_dataset_frame_tree.py`. Both report commands run through the console: +`peak-memory` and `memory-tree` succeed with shared CLI/API semantics. Source and every exported member remain unchanged. This is trusted test-owned producer/replay acceptance, not live capture or upstream GUI acceptance. Do not turn a missing wheel/skip into a successful nightly gate. diff --git a/tests/core/dataset_differential_cases.py b/tests/core/dataset_differential_cases.py index 589d1c4..65760e0 100644 --- a/tests/core/dataset_differential_cases.py +++ b/tests/core/dataset_differential_cases.py @@ -25,7 +25,8 @@ "active_memory_callstack_at_event", ) ) -UNSUPPORTED_TEMPLATES = frozenset(("event_frames", "active_memory_frame_tree_at_event")) +FRAME_TEMPLATES = frozenset(("active_memory_frame_tree_at_event",)) +UNSUPPORTED_TEMPLATES = frozenset(("event_frames",)) def configured_seeds(environ=None): diff --git a/tests/core/test_dataset_attribution.py b/tests/core/test_dataset_attribution.py index cd4da5f..34eed1f 100644 --- a/tests/core/test_dataset_attribution.py +++ b/tests/core/test_dataset_attribution.py @@ -544,6 +544,8 @@ def test_native_bare_local_stack_ids_do_not_alias_foreign_sources(tmp_path): .db_path ) raw = json.loads((root / "manifest.json").read_text()) + # Exercise legacy text-only identity independently of captured frame evidence. + raw.pop("extensions", None) for event in (17, 67): item = next( s diff --git a/tests/core/test_dataset_differential_cases.py b/tests/core/test_dataset_differential_cases.py index ae19278..f71c8de 100644 --- a/tests/core/test_dataset_differential_cases.py +++ b/tests/core/test_dataset_differential_cases.py @@ -12,6 +12,7 @@ from pt_snap_cli.query import registry from tests.core.dataset_differential_assertions import compare_case from tests.core.dataset_differential_cases import ( + FRAME_TEMPLATES, REGRESSION_SEEDS, SUPPORTED_TEMPLATES, UNSUPPORTED_TEMPLATES, @@ -61,7 +62,9 @@ def test_invalid_seed_configuration_fails_instead_of_silently_skipping(name, val def test_query_bank_preserves_original_shapes_and_all_supported_templates(monkeypatch): - assert set(DATASET_SUPPORT) == SUPPORTED_TEMPLATES + # Text-only SQL differential fixtures cannot serve the structured single-DB + # oracle; frame paths have their own real-import differential suite. + assert set(DATASET_SUPPORT) == SUPPORTED_TEMPLATES | FRAME_TEMPLATES # Other suites legitimately register custom SQL. Check packaged discovery # in a fresh registry, and restore both singleton and identity bookkeeping. with monkeypatch.context() as patch: @@ -69,7 +72,10 @@ def test_query_bank_preserves_original_shapes_and_all_supported_templates(monkey patch.setattr(registry, "_registry", registry.QueryRegistry()) patch.setattr(registry, "_packaged", {}) registry._load_all_templates() - assert set(registry.list_queries()) == SUPPORTED_TEMPLATES | UNSUPPORTED_TEMPLATES + assert ( + set(registry.list_queries()) + == SUPPORTED_TEMPLATES | FRAME_TEMPLATES | UNSUPPORTED_TEMPLATES + ) for seed in configured_seeds({"PT_SNAP_DIFFERENTIAL_SEEDS": "200"}): capacity = capacity_for_seed(seed) cases = query_cases(seed, capacity) diff --git a/tests/core/test_dataset_frame_tree.py b/tests/core/test_dataset_frame_tree.py new file mode 100644 index 0000000..0549fa8 --- /dev/null +++ b/tests/core/test_dataset_frame_tree.py @@ -0,0 +1,420 @@ +"""Full frame paths, real imports and read-only cross-shard occupancy contracts.""" + +import dataclasses +import json +import sqlite3 +import time +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +from pathlib import Path +from threading import Event + +import pytest +from typer.testing import CliRunner + +import pt_snap_cli.core.query_service as query_service_module +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.cli import app +from pt_snap_cli.core.errors import InvalidParameterError, QueryCancelledError, QueryExecutionError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.models import ImportOptions +from pt_snap_cli.core.query_limits import QueryLimits +from pt_snap_cli.core.query_service import QueryService +from pt_snap_cli.snapshot.base import TraceEntry +from pt_snap_cli.snapshot.representation import load_snapshot_representation +from pt_snap_cli.snapshot.tools.adaptors.ordered_frames import OrderedFrameWriter +from tests.core.test_dataset_attribution import case +from tests.core.test_msinsight_export import options +from tests.core.wheel_scale_acceptance import write_scale_source +from tests.snapshot.test_sharded_replay import write_source +from tests.test_dataset_focus import hashes + +TEMPLATE = "active_memory_frame_tree_at_event" +ROOT = {"filename": "train.py", "line": 1, "name": "train"} +SHARED = {"filename": "model.py", "line": 2, "name": "forward"} +LEAF = {"filename": "tensor.py", "line": 3, "name": "alloc"} +STACKS = { + 0: [LEAF, SHARED, ROOT], + 1: [LEAF, SHARED, ROOT], + 3: [LEAF, ROOT], + 4: [SHARED, SHARED, ROOT], + 5: [SHARED, ROOT], + 6: [], + 11: [LEAF, SHARED, ROOT], +} + + +@pytest.mark.parametrize("reset", [False, True]) +def test_new_frame_tree_or_close_interrupts_old_work_and_releases_connections( + tmp_path, monkeypatch, reset +): + root = framed_dataset(tmp_path) + before = hashes(root) + started = Event() + original = query_service_module.frame_tree + old_caches = [] + + def slow_tree(sources, template, params, *args): + if params["event_id"] == 8: + old_caches.append(sources.cache) + started.set() + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + sources.budget.remaining() + Event().wait(0.005) + pytest.fail("The obsolete tree was not cancelled") + return original(sources, template, params, *args) + + monkeypatch.setattr(query_service_module, "frame_tree", slow_tree) + with closing(QueryService()) as service, ThreadPoolExecutor(max_workers=2) as pool: + first = pool.submit( + service.execute_query, TEMPLATE, {"event_id": 8}, db_path=root, timeout_s=0 + ) + assert started.wait(3) + if reset: + service.close() + second = pool.submit(service.execute_query, TEMPLATE, {"event_id": 9}, db_path=root) + with pytest.raises(QueryCancelledError): + first.result(timeout=5) + result = second.result(timeout=5) + assert_conserved(result.rows) + assert all(len(cache) == 0 for cache in old_caches) + # Completion must not leave cancellation installed for the next call. + assert service.execute_query(TEMPLATE, {"event_id": 9}, db_path=root).rows == result.rows + assert hashes(root) == before + + +@pytest.mark.parametrize("reset", ["close", "invalidate_context_cache"]) +def test_analyzer_reset_cancels_its_inflight_tree(tmp_path, monkeypatch, reset): + root = framed_dataset(tmp_path) + started = Event() + original = query_service_module.frame_tree + + def slow_tree(sources, template, params, *args): + if params["event_id"] == 8: + started.set() + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + sources.budget.remaining() + Event().wait(0.005) + pytest.fail("Analyzer reset did not cancel the obsolete tree") + return original(sources, template, params, *args) + + monkeypatch.setattr(query_service_module, "frame_tree", slow_tree) + with SnapshotAnalyzer(root) as analyzer, ThreadPoolExecutor(max_workers=1) as pool: + first = pool.submit(analyzer.execute_query, TEMPLATE, {"event_id": 8}, timeout_s=0) + assert started.wait(3) + getattr(analyzer, reset)() + with pytest.raises(QueryCancelledError): + first.result(timeout=5) + if reset == "invalidate_context_cache": + assert_conserved(analyzer.execute_query(TEMPLATE, {"event_id": 9})["rows"]) + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_QUERY_TIMEOUT", raising=False) + + +def framed_dataset(tmp_path, version=2): + root = case(tmp_path) + manifest = root / "manifest.json" + raw = json.loads(manifest.read_text()) + raw["extensions"] = {"ptSnapOrderedFrames": {"version": version}} + manifest.write_text(json.dumps(raw)) + for device in (0, 2): + for index in range(8): + with ( + closing(sqlite3.connect(root / f"device_{device}/slice_{index:05d}.db")) as conn, + conn, + ): + if version == 2: + writer = OrderedFrameWriter(conn) + else: + conn.executescript( + "CREATE TABLE pt_snap_frame_coverage (eventId INTEGER PRIMARY KEY,frameCount INTEGER);" + "CREATE TABLE pt_snap_frame (eventId INTEGER,frameIndex INTEGER,frameJson TEXT, PRIMARY KEY(eventId,frameIndex));" + ) + for event in range(index * 2, index * 2 + 2): + if event not in STACKS: + continue + if version == 2: + entry = TraceEntry.from_dict( + {"id": event, "action": "alloc", "frames": STACKS[event]} + ) + writer.insert(entry) + else: + conn.execute( + "INSERT INTO pt_snap_frame_coverage VALUES (?,?)", + (event, len(STACKS[event])), + ) + conn.executemany( + "INSERT INTO pt_snap_frame VALUES (?,?,?)", + [ + (event, i, json.dumps(frame)) + for i, frame in enumerate(STACKS[event]) + ], + ) + return root + + +def assert_conserved(rows): + assert rows[0]["node_id"] == "root" + for total, own in ( + ("size_bytes", "self_bytes"), + ("requested_bytes", "self_requested_bytes"), + ("block_count", "self_block_count"), + ): + assert sum(row[own] for row in rows) == rows[0][total] + for row in rows: + assert row[total] == row[own] + sum( + child[total] for child in rows if child["parent_id"] == row["node_id"] + ) + + +@pytest.mark.parametrize("version", (1, 2)) +def test_paths_recursion_source_owners_and_devices_conserve_occupancy(tmp_path, version): + root = framed_dataset(tmp_path, version) + before = hashes(root) + with SnapshotAnalyzer(root) as analyzer: + result = analyzer.execute_query(TEMPLATE, {"event_id": 8}, exact_total=True) + rows = result["rows"] + assert_conserved(rows) + assert (rows[0]["size_bytes"], rows[0]["block_count"]) == (295, 9) + assert sorted(r["size_bytes"] for r in rows if r["name"] == "alloc") == [40, 150] + assert sorted( + (r["depth"], r["size_bytes"], r["self_bytes"]) for r in rows if r["name"] == "forward" + ) == [(2, 205, 25), (3, 30, 30)] + assert next(r for r in rows if r["node_id"] == "missing")["size_bytes"] == 15 + assert next(r for r in rows if r["node_id"] == "empty_callstack")["size_bytes"] == 20 + assert not result["scope"]["source_coverage"]["ordered_frames_complete"] + assert rows == analyzer.execute_query(TEMPLATE, {"event_id": 8}, device_id=2)["rows"] + assert analyzer.execute_query(TEMPLATE, {"event_id": 11})["rows"][0]["size_bytes"] == 285 + filtered = analyzer.execute_query( + TEMPLATE, {"event_id": 8, "include_static": False, "min_size": 30} + )["rows"] + assert_conserved(filtered) + assert filtered[0]["size_bytes"] == 220 + empty = analyzer.execute_query(TEMPLATE, {"event_id": 8, "min_size": 9999})["rows"] + assert ( + len(empty) == 1 and empty[0]["size_bytes"] == 0 and empty[0]["percent_of_total"] is None + ) + capped = analyzer.execute_query(TEMPLATE, {"event_id": 8}, max_rows=2, exact_total=True) + assert capped["rows"] == rows[:2] and capped["truncated"] and capped["total"] == len(rows) + with pytest.raises(InvalidParameterError): + analyzer.execute_query(TEMPLATE, {"event_id": 8}, slice_index=0) + assert hashes(root) == before + + +@pytest.mark.parametrize("damage", ("tail", "frame", "json", "type", "schema", "version")) +def test_bad_evidence_degrades_to_missing_without_losing_bytes(tmp_path, damage): + root = framed_dataset(tmp_path) + path = root / "device_0/slice_00000.db" + with closing(sqlite3.connect(path)) as conn, conn: + if damage == "tail": + conn.execute("DELETE FROM pt_snap_stack_frame WHERE frameIndex=2") + elif damage == "frame": + conn.execute("DELETE FROM pt_snap_frame_value") + elif damage == "json": + conn.execute("UPDATE pt_snap_frame_value SET frameJson='[]'") + elif damage == "type": + conn.execute( + "UPDATE pt_snap_frame_value SET frameJson=?", + ('{"filename":"x","line":true,"name":"n"}',), + ) + elif damage == "schema": + conn.execute("DROP TABLE pt_snap_stack_frame") + if damage == "version": + manifest = root / "manifest.json" + data = json.loads(manifest.read_text()) + data["extensions"]["ptSnapOrderedFrames"]["version"] = 999 + manifest.write_text(json.dumps(data)) + before = hashes(root) + with SnapshotAnalyzer(root) as analyzer: + rows = analyzer.execute_query(TEMPLATE, {"event_id": 8})["rows"] + assert_conserved(rows) + assert rows[0]["size_bytes"] == 295 + assert next(r for r in rows if r["node_id"] == "missing")["size_bytes"] >= 165 + assert hashes(root) == before + + +@pytest.mark.parametrize("fmt", ("pt-snap-native-v2", "msinsight")) +def test_real_import_matches_single_tree_and_cli_report(tmp_path, fmt): + request = options(tmp_path, capacity=2, focus=False) + # Same complete raw triple under two parents; preserve deep recursion too. + data = load_snapshot_representation(request.snapshot_file) # test-owned generated input + data["device_traces"][0][0]["frames"] = [LEAF, SHARED, ROOT] + data["device_traces"][0][1]["frames"] = [SHARED, SHARED, ROOT] + data["device_traces"][0][-1]["frames"] = [LEAF, ROOT] + write_source(tmp_path, data) + service = ImportService() + dataset = service.import_snapshot(dataclasses.replace(request, format=fmt, events_per_slice=2)) + single = service.import_snapshot( + ImportOptions(request.snapshot_file, tmp_path / "single", set_focus=False) + ) + before = hashes(dataset.db_path) + + def paths(rows): + by_id = {r["node_id"]: r for r in rows} + result = {} + for row in rows: + labels = [] + current = row + while current is not None: + labels.append(current["label"]) + current = by_id.get(current["parent_id"]) + result[tuple(reversed(labels))] = tuple( + row[k] + for k in ( + "size_bytes", + "self_bytes", + "requested_bytes", + "self_requested_bytes", + "block_count", + "self_block_count", + "percent_of_total", + ) + ) + return result + + with ( + SnapshotAnalyzer(dataset.db_path) as analyzer, + SnapshotAnalyzer(single.db_path) as reference, + ): + for event in range(5): + actual = analyzer.execute_query(TEMPLATE, {"event_id": event}) + assert_conserved(actual["rows"]) + assert paths(actual["rows"]) == paths( + reference.execute_query(TEMPLATE, {"event_id": event})["rows"] + ) + rows = analyzer.execute_query(TEMPLATE, {"event_id": 3})["rows"] + assert analyzer.execute_query("event", {"id": 0})["rows"][0]["frames"] == [ + LEAF, + SHARED, + ROOT, + ] + runner = CliRunner() + args = ["report", "memory-tree", str(dataset.db_path), "--event-id", "3"] + report = runner.invoke(app, [*args, "--json"]) + assert report.exit_code == 0, report.output + assert json.loads(report.output)["rows"] == rows + html = runner.invoke(app, [*args, "--format", "html"]) + assert html.exit_code == 0 and "Reset zoom" in html.output + assert hashes(dataset.db_path) == before + + +def test_dedup_storage_and_budget_failure_are_explicit(tmp_path): + root = framed_dataset(tmp_path) + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn: + assert conn.execute("SELECT COUNT(*) FROM pt_snap_frame_value").fetchone()[0] == 3 + assert conn.execute("SELECT COUNT(*) FROM pt_snap_stack").fetchone()[0] == 1 + assert conn.execute("SELECT COUNT(*) FROM pt_snap_event_stack").fetchone()[0] == 2 + service = QueryService() + try: + with pytest.raises(QueryExecutionError, match="budget"): + service.execute_query( + TEMPLATE, {"event_id": 8}, db_path=root, query_limits=QueryLimits(max_work_rows=1) + ) + assert ( + service.execute_query(TEMPLATE, {"event_id": 8}, db_path=root).rows[0]["size_bytes"] + == 295 + ) + finally: + service.close() + + +def test_same_rendered_stack_is_not_a_frame_path_identity(tmp_path): + root = framed_dataset(tmp_path) + # Both arrays render as "a:1 b\nc:2 d" when traversed caller-first. + arrays = [ + [{"filename": "a", "line": 1, "name": "b\nc:2 d"}], + [{"filename": "c", "line": 2, "name": "d"}, {"filename": "a", "line": 1, "name": "b"}], + ] + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute("DELETE FROM pt_snap_event_stack") + conn.execute("DELETE FROM pt_snap_stack_frame") + conn.execute("DELETE FROM pt_snap_stack") + conn.execute("DELETE FROM pt_snap_frame_value") + for table in ( + "pt_snap_event_stack", + "pt_snap_stack_frame", + "pt_snap_stack", + "pt_snap_frame_value", + ): + conn.execute(f"DROP TABLE {table}") + writer = OrderedFrameWriter(conn) + for event, frames in enumerate(arrays): + writer.insert(TraceEntry.from_dict({"id": event, "action": "alloc", "frames": frames})) + conn.execute("UPDATE trace_entry_0 SET callstack=? WHERE id IN (0,1)", ("a:1 b\nc:2 d",)) + with SnapshotAnalyzer(root) as analyzer: + rows = analyzer.execute_query(TEMPLATE, {"event_id": 1})["rows"] + assert_conserved(rows) + assert len([row for row in rows if row["filename"] == "a"]) == 2 + assert sorted(row["size_bytes"] for row in rows if row["filename"] == "a") == [60, 90] + + +def test_repeated_ordered_stacks_reduce_before_fetching_with_real_work_budget(tmp_path): + source = write_scale_source(tmp_path / "scale.pkl", 10_003) + result = ImportService().import_snapshot( + ImportOptions( + source, tmp_path / "output", set_focus=False, events_per_slice=5_000, format="msinsight" + ) + ) + before = hashes(result.db_path) + with SnapshotAnalyzer(result.db_path) as analyzer: + stats = analyzer.execute_query( + "callstack_analysis", {"min_count": 1}, query_limits=QueryLimits(max_work_rows=1000) + ) + assert len(stats["rows"]) == 6 + assert sum(row["alloc_count"] for row in stats["rows"]) == 10_003 + assert sum(row["total_size"] for row in stats["rows"]) == 16 * 10_003 + 64 + assert stats["scope"]["source_coverage"]["ordered_frame_events"] == 10_003 + for event, expected in ((4_999, 176), (5_000, 160), (10_002, 176)): + tree = analyzer.execute_query(TEMPLATE, {"event_id": event}) + assert tree["rows"][0]["size_bytes"] == expected + assert tree["scope"]["source_coverage"]["ordered_frames_complete"] + assert_conserved(tree["rows"]) + assert hashes(result.db_path) == before + + +def test_legacy_native_artifact_is_readable_but_frame_tree_degrades(tmp_path): + request = options(tmp_path, focus=False) + result = ImportService().import_snapshot(dataclasses.replace(request, format=None)) + manifest = result.db_path / "manifest.json" + raw = json.loads(manifest.read_text()) + raw["identity"]["extensionContractVersion"] = 1 + raw.pop("extensions") + manifest.write_text(json.dumps(raw)) + before = hashes(result.db_path) + with SnapshotAnalyzer(result.db_path) as analyzer: + result = analyzer.execute_query(TEMPLATE, {"event_id": 3}) + assert_conserved(result["rows"]) + assert {row["name"] for row in result["rows"]} == {None} + assert not result["scope"]["source_coverage"]["ordered_frames_complete"] + assert hashes(manifest.parent) == before + + +def test_statistics_representative_is_earliest_event_not_local_stack_id(tmp_path): + root = framed_dataset(tmp_path) + # Equivalent arrays may use different local IDs in optional frame evidence. + # SQLite's GROUP BY order must not select event 1 ahead of event 0. + with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: + conn.execute( + "INSERT INTO pt_snap_stack SELECT 99,frameCount FROM pt_snap_stack WHERE stackId=0" + ) + conn.execute( + "INSERT INTO pt_snap_stack_frame SELECT 99,frameIndex,frameId " + "FROM pt_snap_stack_frame WHERE stackId=0" + ) + conn.execute("UPDATE pt_snap_event_stack SET stackId=99 WHERE eventId=0") + before = hashes(root) + with SnapshotAnalyzer(root) as analyzer: + rows = analyzer.execute_query("callstack_analysis", {"min_count": 1})["rows"] + shared = next(row for row in rows if row["callstack"].startswith("shared:")) + assert shared["stack_event_id"] == 0 + assert shared["alloc_count"] == 3 # Events 0, 1 and 11 share the raw array. + assert shared["total_size"] == 24 # Trace-event sizes, not live block sizes. + assert hashes(root) == before diff --git a/tests/core/test_dataset_global.py b/tests/core/test_dataset_global.py index 7f89b6b..b267894 100644 --- a/tests/core/test_dataset_global.py +++ b/tests/core/test_dataset_global.py @@ -483,6 +483,7 @@ def test_all_dataset_row_keys_are_declared_by_shared_template_schema(tmp_path, l "stack_bytes": 0 if detailed else -1, }, "preexisting_live": {"event_id": event}, + "active_memory_frame_tree_at_event": {"event_id": event}, } with SnapshotAnalyzer(root) as analyzer: for template in DATASET_SUPPORT: diff --git a/tests/core/test_dataset_id_pagination.py b/tests/core/test_dataset_id_pagination.py index 56e4254..70d5aa8 100644 --- a/tests/core/test_dataset_id_pagination.py +++ b/tests/core/test_dataset_id_pagination.py @@ -59,6 +59,23 @@ def paired_dataset(tmp_path_factory, request): ] +def assert_shared_rows(actual, expected): + """Compare common columns while checking new dataset event frame evidence.""" + assert len(actual) == len(expected) + for row, reference in zip(actual, expected, strict=True): + assert {key: row[key] for key in reference} == reference + if "frames_status" in row: + assert row["frames_status"] == "ordered" + assert isinstance(row["frames"], list) + assert ( + "\n".join( + f"{frame['filename']}:{frame['line']} {frame['name']}" + for frame in reversed(row["frames"]) + ) + == reference["callstack"] + ) + + @pytest.mark.parametrize("template", ["event", "allocation"]) @pytest.mark.parametrize("direction", ["ASC", "DESC"]) def test_id_windows_match_standalone_with_filters_and_sparse_ids( @@ -100,7 +117,7 @@ def test_id_windows_match_standalone_with_filters_and_sparse_ids( expected = service.execute_query( template, params, single, max_rows=max_rows, exact_total=exact ) - assert actual.rows == expected.rows, params + assert_shared_rows(actual.rows, expected.rows) for field in ("total", "returned", "has_more", "truncated", "total_is_exact"): assert getattr(actual, field) == getattr(expected, field), (params, field) assert actual.scope["actual_range"] == { @@ -131,7 +148,7 @@ def test_id_slice_pages_preserve_scope_and_exact_total(paired_dataset, template, template, params, root, slice_index=item.index, exact_total=True ) expected = service.execute_query(template, params, single, exact_total=True) - assert actual.rows == expected.rows + assert_shared_rows(actual.rows, expected.rows) assert actual.total == expected.total assert actual.has_more == expected.has_more assert actual.truncated == expected.truncated diff --git a/tests/core/test_dataset_scale_acceptance.py b/tests/core/test_dataset_scale_acceptance.py index 415b8ed..d0ccbd5 100644 --- a/tests/core/test_dataset_scale_acceptance.py +++ b/tests/core/test_dataset_scale_acceptance.py @@ -375,7 +375,7 @@ def test_installed_trusted_producer_default_export_cli_and_api( assert {case["template"] for case in cases} == set(by_name) supported = {name for name, entry in by_name.items() if entry["dataset_support"]["supported"]} unsupported = set(by_name) - supported - assert unsupported == {"event_frames", "active_memory_frame_tree_at_event"} + assert unsupported == {"event_frames"} cases_file = runtime.cwd / "cases.json" cases_file.write_text(json.dumps(cases)) api = runtime.python_json("api", API_QUERIES, root, cases_file, boundary - 1) @@ -419,10 +419,12 @@ def test_installed_trusted_producer_default_export_cli_and_api( "--event-id", boundary - 1, "--json", - expected=1, + expected=0, ) - assert tree["error"]["code"] == api["tree_report"]["code"] == "QUERY_FAILED" - assert "Unsupported dataset-global" in tree["error"]["message"] + assert tree["rows"] == api["tree_report"]["rows"] + assert tree["rows"] == cli_queries["active_memory_frame_tree_at_event"]["rows"] + assert tree["rows"][0]["size_bytes"] == 176 + assert sum(row["self_bytes"] for row in tree["rows"]) == 176 assert hashes(root) == before assert sha256(source) == source_hash assert not (runtime.cwd / ".pt-snap").exists() @@ -438,7 +440,7 @@ def test_installed_trusted_producer_default_export_cli_and_api( "slices": 2, "templates_supported": sorted(supported), "templates_expected_rejections": sorted(unsupported), - "reports": {"peak-memory": "passed", "memory-tree": "expected dataset rejection"}, + "reports": {"peak-memory": "passed", "memory-tree": "passed"}, "commands": runtime.commands, "read_only_verified": True, } diff --git a/tests/core/test_dataset_sql_pushdown.py b/tests/core/test_dataset_sql_pushdown.py index 01b2ac3..b5f4ffc 100644 --- a/tests/core/test_dataset_sql_pushdown.py +++ b/tests/core/test_dataset_sql_pushdown.py @@ -390,6 +390,9 @@ def test_global_text_identity_and_candidates_ignore_source_collation( conn.execute("UPDATE trace_entry_0 SET callstackId=1 WHERE id=?", [ids[1]]) conn.execute("UPDATE trace_entry_0 SET size=1 WHERE id>=0") if layout == "v2": + # This fixture tests text collation, independently of the original raw + # frames retained by new imports. Frame identity has separate coverage. + manifest.pop("extensions", None) item["sha256"] = hashlib.sha256(member.read_bytes()).hexdigest() manifest_path.write_text(json.dumps(manifest)) expected_text = {event: texts[int(event == ids[1])] for event in ids} diff --git a/tests/core/test_dataset_validation_cache.py b/tests/core/test_dataset_validation_cache.py index 8a356ef..527c6f9 100644 --- a/tests/core/test_dataset_validation_cache.py +++ b/tests/core/test_dataset_validation_cache.py @@ -13,7 +13,7 @@ from pt_snap_cli.core.dataset_contract import validate_dataset from pt_snap_cli.core.dataset_files import hash_file from pt_snap_cli.core.dataset_resolver import DatasetResolver -from pt_snap_cli.core.errors import DatabaseSchemaError, QueryTimeoutError +from pt_snap_cli.core.errors import DatabaseSchemaError, QueryCancelledError, QueryTimeoutError from pt_snap_cli.core.validation_budget import validation_progress from tests.test_dataset_focus import make_dataset from tests.test_native_dataset import dataset as native_dataset @@ -66,7 +66,9 @@ def changed(*args, **kwargs): @pytest.mark.parametrize("cache_mode", [None, "immutable", "disabled"]) @pytest.mark.parametrize("seal", ["parserCache", "ptSnap"]) -def test_declared_seals_are_checked_in_every_cache_mode(tmp_path, monkeypatch, cache_mode, seal): +def test_historical_digests_do_not_reject_valid_content_changes( + tmp_path, monkeypatch, cache_mode, seal +): from pt_snap_cli.core.snapshot_cache import seal_manifest if cache_mode is None: @@ -87,8 +89,7 @@ def test_declared_seals_are_checked_in_every_cache_mode(tmp_path, monkeypatch, c assert DatasetResolver().inspect(root) is not None with closing(sqlite3.connect(root / "device_0/slice_00000.db")) as conn, conn: conn.execute("UPDATE trace_entry_0 SET allocated=allocated+1") - with pytest.raises(DatabaseSchemaError, match="seal"): - DatasetResolver().inspect(root) + assert DatasetResolver().inspect(root) is not None def forbidden(*args, **kwargs): @@ -462,6 +463,22 @@ def test_strong_operation_guard_hashes_even_when_filesystem_tokens_look_unchange resolved.require_unchanged() +@pytest.mark.parametrize("error", [QueryTimeoutError, QueryCancelledError]) +def test_cancelled_admission_evicts_an_existing_entry(tmp_path, monkeypatch, error): + monkeypatch.delenv("PT_SNAP_DATASET_CACHE") + root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) + DatasetResolver().inspect(root) + assert root in dataset_resolver._VALIDATED + + class Budget: + def remaining(self): + raise error("interrupted admission") + + with pytest.raises(error, match="interrupted admission"): + DatasetResolver().inspect(root, budget=Budget()) + assert root not in dataset_resolver._VALIDATED + + def test_disabling_cache_does_not_reuse_an_existing_entry(tmp_path, monkeypatch): monkeypatch.delenv("PT_SNAP_DATASET_CACHE") root = make_dataset(tmp_path / "dataset", devices=(0,), slices=1) diff --git a/tests/core/test_import_validation_receipts.py b/tests/core/test_import_validation_receipts.py new file mode 100644 index 0000000..3f65ff0 --- /dev/null +++ b/tests/core/test_import_validation_receipts.py @@ -0,0 +1,186 @@ +"""Import receipts avoid row scans across processes without trusting timestamps.""" + +import dataclasses +import json +import os +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.core import dataset_resolver, import_service +from pt_snap_cli.core.errors import ImportExecutionError, QueryTimeoutError +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.validation_receipts import import_receipts, load_receipt, save_receipt +from tests.core.test_dataset_validation_cache import CancellableBudget +from tests.core.test_msinsight_export import options + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_DATASET_CACHE", raising=False) + dataset_resolver._VALIDATED.clear() + yield + dataset_resolver._VALIDATED.clear() + + +def unexpected(*args, **kwargs): + pytest.fail("unchanged imported bytes must not require another row validation") + + +def test_first_import_seeds_cold_repeat_including_focus(tmp_path, monkeypatch): + request = dataclasses.replace(options(tmp_path), format=None) + first = ImportService().import_snapshot(request) + dataset_resolver._VALIDATED.clear() # A fresh CLI process has no in-memory entries. + for name in ("validate_native_dataset", "validate_dataset", "validate_cache"): + monkeypatch.setattr(dataset_resolver, name, unexpected) + monkeypatch.setattr(import_service, "frame_member_digest", unexpected) + hashed = [] + original = dataset_resolver._hash_file + + def hash_file(path, **kwargs): + hashed.append(path) + return original(path, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", hash_file) + second = ImportService().import_snapshot(request) + assert second.reused and second.metadata == first.metadata + assert second.focus_state.available_devices == first.focus_state.available_devices + members = list(first.db_path.glob("device_*/*.db")) + assert members and all(path in hashed for path in members) + + +def test_same_size_and_mtime_frame_edit_misses_receipt(tmp_path): + request = dataclasses.replace(options(tmp_path, focus=False), format=None) + first = ImportService().import_snapshot(request) + member = next(first.db_path.glob("device_*/*.db")) + before = member.stat() + with closing(sqlite3.connect(member)) as conn, conn: + row = conn.execute("SELECT id,frameJson FROM pt_snap_frame_value LIMIT 1").fetchone() + # Same-length valid JSON, without changing the number or size of rows. + changed = row[1].replace("name", "nAme", 1) + assert changed != row[1] + conn.execute("UPDATE pt_snap_frame_value SET frameJson=? WHERE id=?", (changed, row[0])) + assert member.stat().st_size == before.st_size + os.utime(member, ns=(before.st_atime_ns, before.st_mtime_ns)) + dataset_resolver._VALIDATED.clear() + with pytest.raises(ImportExecutionError): + ImportService().import_snapshot(request) + assert member.exists() + + +def test_compatible_consumer_table_change_reuses_without_receipt(tmp_path, monkeypatch): + request = options(tmp_path, focus=False) + first = ImportService().import_snapshot(request) + member = next(first.db_path.glob("device_*/*.db")) + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute("CREATE TABLE gui_cache(value TEXT)") + conn.execute("INSERT INTO gui_cache VALUES ('consumer')") + dataset_resolver._VALIDATED.clear() + monkeypatch.setattr(dataset_resolver, "validate_dataset", unexpected) + assert ImportService().import_snapshot(request).reused + dataset_resolver._VALIDATED.clear() + assert ImportService().import_snapshot(request).reused + + +@pytest.mark.parametrize("damage", ["missing", "malformed", "checksum", "version"]) +def test_missing_or_damaged_receipt_falls_back(tmp_path, monkeypatch, damage): + request = dataclasses.replace(options(tmp_path, focus=False), format=None) + ImportService().import_snapshot(request) + folder = tmp_path / ".cache/pt-snap-cli/validation-v1" + receipts = list(folder.glob("*.json")) + assert receipts + for path in receipts: + if damage == "missing": + path.unlink() + elif damage == "malformed": + path.write_text("{", encoding="utf-8") + elif damage == "checksum": + data = json.loads(path.read_bytes()) + data["sha256"] = "0" * 64 + path.write_text(json.dumps(data), encoding="utf-8") + else: + path.rename(path.with_suffix(".old")) + dataset_resolver._VALIDATED.clear() + calls = [] + original = dataset_resolver.validate_native_dataset + + def validate(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(dataset_resolver, "validate_native_dataset", validate) + assert ImportService().import_snapshot(request).reused + assert calls == [1] + + +def test_receipt_is_import_only_and_off_disables_it(tmp_path, monkeypatch): + request = dataclasses.replace(options(tmp_path, focus=False), format=None) + result = ImportService().import_snapshot(request) + dataset_resolver._VALIDATED.clear() + calls = [] + original = dataset_resolver.validate_native_dataset + + def validate(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(dataset_resolver, "validate_native_dataset", validate) + dataset_resolver.DatasetResolver().inspect(result.db_path) + assert calls == [1] # Ordinary analysis retains its existing admission contract. + calls.clear() + dataset_resolver._VALIDATED.clear() + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "off") + assert ImportService().import_snapshot(request).reused + assert calls == [1] + + +def test_unwritable_receipt_directory_is_only_a_cache_miss(tmp_path): + (tmp_path / ".cache").write_text("file blocks directory creation", encoding="utf-8") + with import_receipts(): + save_receipt(tmp_path, "dataset", "fingerprint", {}) + assert load_receipt(tmp_path, "dataset", "fingerprint") is None + request = dataclasses.replace(options(tmp_path, focus=False), format=None) + assert not ImportService().import_snapshot(request).reused + dataset_resolver._VALIDATED.clear() + assert ImportService().import_snapshot(request).reused + + +def test_receipt_hit_hashing_respects_cancellation(tmp_path, monkeypatch): + request = dataclasses.replace(options(tmp_path, focus=False), format=None) + first = ImportService().import_snapshot(request) + dataset_resolver._VALIDATED.clear() + budget = CancellableBudget() + original = dataset_resolver._hash_file + + def cancelled(path, **kwargs): + if path.suffix == ".db": + budget.cancelled = True + return original(path, **kwargs) + + with monkeypatch.context() as patch, import_receipts(): + patch.setattr(dataset_resolver, "_hash_file", cancelled) + with pytest.raises(QueryTimeoutError): + dataset_resolver.DatasetResolver().inspect(first.db_path, budget=budget) + assert first.db_path not in dataset_resolver._VALIDATED + assert ImportService().import_snapshot(request).reused + + +def test_compatible_committed_wal_frame_edits_do_not_trigger_content_checks(tmp_path): + request = options(tmp_path, focus=False) + first = ImportService().import_snapshot(request) + member = next(first.db_path.glob("device_*/*.db")) + with closing(sqlite3.connect(member)) as writer: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("CREATE TABLE gui_cache(value TEXT)") + writer.commit() + assert ImportService().import_snapshot(request).reused + writer.execute('UPDATE pt_snap_frame_value SET frameJson=\'{"name":"changed"}\'') + writer.commit() + assert Path(str(member) + "-wal").stat().st_size > 0 + dataset_resolver._VALIDATED.clear() + assert ImportService().import_snapshot(request).reused diff --git a/tests/core/test_latest_query.py b/tests/core/test_latest_query.py new file mode 100644 index 0000000..3160e88 --- /dev/null +++ b/tests/core/test_latest_query.py @@ -0,0 +1,139 @@ +"""Latest-only coordination does not publish superseded or reset work.""" + +import sqlite3 +import time +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +from threading import Event + +import pytest + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core.errors import QueryCancelledError +from pt_snap_cli.core.latest_query import LatestQuery +from pt_snap_cli.core.validation_budget import validation_progress +from pt_snap_cli.query.config import QueryParameter, QueryTemplate +from pt_snap_cli.query.executor import QueryExecutor + + +def test_cancel_discards_result_and_next_query_can_run(): + queries = LatestQuery() + with pytest.raises(QueryCancelledError): + with queries.run() as cancelled: + assert not cancelled() + queries.cancel() + assert cancelled() + with queries.run() as cancelled: + assert not cancelled() + + +def test_query_failure_does_not_poison_future_queries(): + queries = LatestQuery() + with pytest.raises(ValueError): + with queries.run(): + raise ValueError("query failed") + with queries.run() as cancelled: + assert not cancelled() + + +def test_queued_obsolete_request_is_skipped(): + queries = LatestQuery() + entered = [] + queries._work.acquire() + + def run(number): + with queries.run(): + entered.append(number) + + def wait_for_token(previous): + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + with queries._state: + token = queries._active + if token is not None and token is not previous: + return token + Event().wait(0.005) + pytest.fail("The request did not register its cancellation token") + + with ThreadPoolExecutor(max_workers=2) as pool: + try: + first = pool.submit(run, 1) + token = wait_for_token(None) + second = pool.submit(run, 2) + wait_for_token(token) + finally: + queries._work.release() + with pytest.raises(QueryCancelledError): + first.result(timeout=3) + second.result(timeout=3) + assert entered == [2] + + +def test_independent_services_do_not_cancel_each_other(): + first, second = LatestQuery(), LatestQuery() + with first.run() as cancelled: + with pytest.raises(QueryCancelledError): + with second.run(): + second.cancel() + assert not cancelled() + + +def test_cancellation_during_dataset_validation_preserves_error_and_cleans_handler(): + class Budget: + def __init__(self): + self.checks = 0 + + def remaining(self): + self.checks += 1 + if self.checks >= 3: + raise QueryCancelledError("cancelled during validation") + return None + + with closing(sqlite3.connect(":memory:")) as conn: + with pytest.raises(QueryCancelledError, match="during validation"): + with validation_progress(conn, Budget()): + conn.execute( + "WITH RECURSIVE n(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM n WHERE x<100000000) " + "SELECT SUM(x) FROM n" + ).fetchall() + assert conn.execute("SELECT 1").fetchone() == (1,) + + +@pytest.mark.parametrize("timeout", [0, 30]) +def test_standalone_tree_interrupts_sql_and_preserves_runtime_templates( + tmp_path, monkeypatch, timeout +): + path = tmp_path / "standalone.db" + with closing(sqlite3.connect(path)) as conn: + conn.executescript( + "CREATE TABLE dictionary (id INTEGER);" "CREATE TABLE trace_entry_0 (id INTEGER);" + ) + started = Event() + execute = QueryExecutor.execute + + def observed(self, sql, *args, **kwargs): + if "WITH RECURSIVE" in sql: + started.set() + return execute(self, sql, *args, **kwargs) + + monkeypatch.setattr(QueryExecutor, "execute", observed) + template = "active_memory_frame_tree_at_event" + with SnapshotAnalyzer(path) as analyzer, ThreadPoolExecutor(max_workers=2) as pool: + source = analyzer._query_service._get_executor(analyzer.context_cache.get(path)) + source.register_template( + QueryTemplate( + name=template, + parameters={"event_id": QueryParameter(name="event_id", type="int", required=True)}, + query="{% if event_id == 1 %}" + "WITH RECURSIVE n(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM n WHERE x<100000000) " + "SELECT SUM(x) AS value FROM n" + "{% else %}SELECT 7 AS value{% endif %}", + ) + ) + old = pool.submit(analyzer.execute_query, template, {"event_id": 1}, timeout_s=timeout) + assert started.wait(3) + new = pool.submit(analyzer.execute_query, template, {"event_id": 2}) + with pytest.raises(QueryCancelledError): + old.result(timeout=5) + assert new.result(timeout=5)["rows"] == [{"value": 7}] + assert analyzer.execute_query(template, {"event_id": 2})["rows"] == [{"value": 7}] diff --git a/tests/core/test_lightweight_compatible_import.py b/tests/core/test_lightweight_compatible_import.py new file mode 100644 index 0000000..535b9ba --- /dev/null +++ b/tests/core/test_lightweight_compatible_import.py @@ -0,0 +1,110 @@ +"""Compatible cache reuse only checks identity, inventory and schema.""" + +import dataclasses +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from pt_snap_cli.cli import app +from pt_snap_cli.core import dataset_resolver, import_service +from pt_snap_cli.core.errors import ImportExecutionError +from pt_snap_cli.core.import_service import ImportService +from tests.core.test_msinsight_export import options + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_DATASET_CACHE", raising=False) + dataset_resolver._VALIDATED.clear() + yield + dataset_resolver._VALIDATED.clear() + + +def unexpected(*args, **kwargs): + pytest.fail("default compatible reuse must not scan or hash member contents") + + +@pytest.mark.parametrize("external", [False, True]) +@pytest.mark.parametrize("focus", [False, True]) +def test_repeat_import_skips_all_content_scans(tmp_path, monkeypatch, external, focus): + request = options(tmp_path, focus=focus) + monkeypatch.setattr(import_service, "save_receipt", unexpected) + monkeypatch.setattr(dataset_resolver, "save_receipt", unexpected) + monkeypatch.setattr(dataset_resolver, "validate_dataset", unexpected) + first = ImportService().import_snapshot(request) + assert not dataset_resolver._VALIDATED + if external: + path = first.db_path / "manifest.json" + manifest = json.loads(path.read_bytes()) + del manifest["ptSnap"] + path.write_text(json.dumps(manifest), encoding="utf-8") + dataset_resolver._VALIDATED.clear() + monkeypatch.setattr(dataset_resolver, "validate_dataset", unexpected) + hash_file = dataset_resolver._hash_file + + def no_member_hash(path, **kwargs): + if path.suffix == ".db" or path.name.endswith("-wal"): + unexpected() + return hash_file(path, **kwargs) + + monkeypatch.setattr(dataset_resolver, "_hash_file", no_member_hash) + repeated = ImportService().import_snapshot(request) + assert repeated.reused + assert repeated.devices == first.devices + assert repeated.slice_count == first.slice_count + assert repeated.metadata == (None if external else first.metadata) + assert not dataset_resolver._VALIDATED # Never promote structural checks to query admission. + if focus: + assert repeated.focus_state.available_devices == first.focus_state.available_devices + + +def test_content_edits_do_not_trigger_cache_replay(tmp_path): + request = options(tmp_path, focus=False) + first = ImportService().import_snapshot(request) + member = next(first.db_path.glob("device_*/*.db")) + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute("UPDATE block_0 SET size=size+1") + assert ImportService().import_snapshot(request).reused + + +@pytest.mark.parametrize("cache_mode", ["disabled", "immutable"]) +def test_cache_configuration_does_not_enable_content_checks(tmp_path, monkeypatch, cache_mode): + request = options(tmp_path, focus=True) + ImportService().import_snapshot(request) + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", cache_mode) + monkeypatch.setattr(dataset_resolver, "validate_dataset", unexpected) + assert ImportService().import_snapshot(request).reused + + +def test_import_help_has_no_strict_cache_option(): + result = CliRunner().invoke(app, ["import", "--help"], color=False) + assert result.exit_code == 0, result.output + assert "--verify-cache-content" not in result.stdout + + +@pytest.mark.parametrize("damage", ["missing", "table", "manifest"]) +def test_default_import_still_rejects_incomplete_artifacts(tmp_path, damage): + request = options(tmp_path, focus=False) + first = ImportService().import_snapshot(request) + member = next(first.db_path.glob("device_*/*.db")) + if damage == "missing": + member.unlink() + elif damage == "table": + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute("DROP TABLE block_0") + else: + manifest_path = first.db_path / "manifest.json" + manifest = json.loads(manifest_path.read_bytes()) + manifest["status"] = "building" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ImportExecutionError): + ImportService().import_snapshot(dataclasses.replace(request, force=True)) + if damage == "missing": + assert not member.exists() diff --git a/tests/core/test_msinsight_export.py b/tests/core/test_msinsight_export.py index ef2b5e8..54764f5 100644 --- a/tests/core/test_msinsight_export.py +++ b/tests/core/test_msinsight_export.py @@ -74,7 +74,7 @@ def test_physical_schema_inline_events_streams_stable_references_and_native_pari assert manifest["sourceFile"] == str(request.snapshot_file.resolve()) assert manifest["eventsPerSlice"] == (capacity or 500000) raw = request.snapshot_file.read_bytes() - assert manifest["cacheHash"] == hashlib.sha256(b"mem_snapshot_parser_v2" + raw).hexdigest() + assert manifest["cacheHash"] == hashlib.sha256(b"mem_snapshot_parser_v3" + raw).hexdigest() assert result.metadata.source_sha256 == hashlib.sha256(raw).hexdigest() != manifest["cacheHash"] assert manifest["ptSnap"]["identity"]["targetRevision"] == MSINSIGHT_REVISION native = service.import_snapshot( @@ -139,12 +139,19 @@ def test_physical_schema_inline_events_streams_stable_references_and_native_pari assert analyzer.get_focus().callstack_layout == "v1" for device in (0, 1): for event in range(5): - assert ( - analyzer.execute_query("event", {"id": event}, device_id=device)["rows"] - == native_analyzer.execute_query("event", {"id": event}, device_id=device)[ - "rows" - ] - ) + compatible_rows = analyzer.execute_query("event", {"id": event}, device_id=device)[ + "rows" + ] + native_rows = native_analyzer.execute_query( + "event", {"id": event}, device_id=device + )["rows"] + # Generation namespaces differ across physical formats; frame + # digests and every event/frame value must remain equivalent. + for event_rows in (compatible_rows, native_rows): + for row in event_rows: + for key in ("stack_id", "source_stack_id"): + row[key] = row[key].split(":frames:", 1)[-1] + assert compatible_rows == native_rows assert not hasattr(analyzer, "import_snapshot") assert hashes(result.db_path) == before @@ -194,11 +201,10 @@ def test_consumer_tables_do_not_invalidate_compatible_cache(tmp_path, monkeypatc assert rows(member, "SELECT value FROM unrelated_consumer_cache") == [("keep",)] with closing(sqlite3.connect(member)) as conn, conn: conn.execute("UPDATE block_0 SET size=size+1") - with pytest.raises(ImportExecutionError, match="preserved"): - service.import_snapshot(request) + assert service.import_snapshot(request).reused -def test_producer_only_logical_seal_reuses_without_replaying(tmp_path, monkeypatch): +def test_producer_metadata_reuses_without_replaying(tmp_path, monkeypatch): request = options(tmp_path, focus=False) service = ImportService() first = service.import_snapshot(request) @@ -219,8 +225,7 @@ def test_producer_only_logical_seal_reuses_without_replaying(tmp_path, monkeypat assert analyzer.get_database_metadata()["status"] == "available" with closing(sqlite3.connect(first.db_path / "device_0/slice_00000.db")) as conn, conn: conn.execute("UPDATE block_0 SET size=size+1") - with pytest.raises(ImportExecutionError, match="preserved"): - service.import_snapshot(request) + assert service.import_snapshot(request).reused @pytest.mark.parametrize("cli_capacity", [None, 2]) @@ -287,7 +292,7 @@ def test_external_insight_artifact_reuses_without_claiming_metadata( "last-slice", ], ) -def test_compatible_cache_requires_semantic_validation( +def test_queries_validate_semantics_but_cache_import_does_not_scan( tmp_path, monkeypatch, sealed, external, set_focus, slice_index, mutation, diagnostic ): from pt_snap_cli.core.errors import DatabaseSchemaError @@ -309,7 +314,7 @@ def test_compatible_cache_requires_semantic_validation( if not sealed: del manifest["parserCache"] path.write_text(json.dumps(manifest)) - # The digest is self-consistent, but must not replace shared P0 validation. + # Cache admission checks structure; queries independently validate semantics. validate_cache(result.db_path, manifest["cacheHash"]) with pytest.raises(DatabaseSchemaError, match=diagnostic): DatasetResolver().inspect(result.db_path) @@ -323,11 +328,13 @@ def test_compatible_cache_requires_semantic_validation( service._msinsight_backend.replay, "stage", lambda *a, **k: pytest.fail("must preserve") ) for force in (False, True): - with pytest.raises(ImportExecutionError, match=diagnostic): - service.import_snapshot(dataclasses.replace(request, force=force, set_focus=set_focus)) + assert service.import_snapshot( + dataclasses.replace(request, force=force, set_focus=set_focus) + ).reused assert hashes(result.db_path) == before assert request.snapshot_file.read_bytes() == source_before - assert focus.read_bytes() == focus_before + if not set_focus: + assert focus.read_bytes() == focus_before @pytest.mark.parametrize("capacity", [None, 2, 3]) @@ -359,21 +366,12 @@ def test_external_cache_honors_explicit_capacity(tmp_path, monkeypatch, capacity @pytest.mark.parametrize("force", [False, True]) -@pytest.mark.parametrize("failure", ["semantic", "capacity"]) -def test_external_cache_cli_failure_preserves_artifact_and_focus( - tmp_path, monkeypatch, force, failure -): - from pt_snap_cli.core.snapshot_cache import seal_manifest - +def test_external_cache_cli_failure_preserves_artifact_and_focus(tmp_path, monkeypatch, force): request = options(tmp_path, focus=False) result = ImportService().import_snapshot(request) path = result.db_path / "manifest.json" manifest = json.loads(path.read_text()) del manifest["ptSnap"] - if failure == "semantic": - with closing(sqlite3.connect(result.db_path / "device_0/slice_00000.db")) as conn, conn: - conn.execute("UPDATE trace_entry_0 SET allocated=-1 WHERE id=0") - seal_manifest(result.db_path, manifest) path.write_text(json.dumps(manifest)) before = hashes(result.db_path) source_before = request.snapshot_file.read_bytes() @@ -394,8 +392,7 @@ def test_external_cache_cli_failure_preserves_artifact_and_focus( str(request.output_dir), "--json", ] - if failure == "capacity": - args.extend(["--events-per-slice", "3"]) + args.extend(["--events-per-slice", "3"]) if force: args.append("--force") response = CliRunner().invoke(app, args) @@ -403,8 +400,7 @@ def test_external_cache_cli_failure_preserves_artifact_and_focus( assert response.stdout == "" error = json.loads(response.stderr) assert error["ok"] is False and error["error"]["code"] == "IMPORT_FAILED" - diagnostic = "event_values" if failure == "semantic" else "events_per_slice" - assert diagnostic in error["error"]["message"] + assert "events_per_slice" in error["error"]["message"] assert "Traceback" not in response.stderr assert hashes(result.db_path) == before assert request.snapshot_file.read_bytes() == source_before @@ -412,8 +408,7 @@ def test_external_cache_cli_failure_preserves_artifact_and_focus( @pytest.mark.parametrize("external", [False, True]) -@pytest.mark.parametrize("failure", ["index", "dictionary", "deep-json"]) -def test_invalid_compatible_cache_is_preserved(tmp_path, monkeypatch, external, failure): +def test_invalid_compatible_cache_is_preserved(tmp_path, monkeypatch, external): request = options(tmp_path, focus=False) service = ImportService() result = service.import_snapshot(request) @@ -421,24 +416,7 @@ def test_invalid_compatible_cache_is_preserved(tmp_path, monkeypatch, external, manifest = json.loads(path.read_text()) if external: del manifest["ptSnap"] - member = result.db_path / "device_0/slice_00000.db" - if failure == "index": - with closing(sqlite3.connect(member)) as conn: - conn.execute("CREATE INDEX block_address ON block_0(address)") - page_size = conn.execute("PRAGMA page_size").fetchone()[0] - root = conn.execute( - "SELECT rootpage FROM sqlite_master WHERE name='block_address'" - ).fetchone()[0] - with member.open("r+b") as file: - file.seek((root - 1) * page_size) - file.write(b"\xff" * page_size) - elif failure == "dictionary": - del manifest["parserCache"] - with closing(sqlite3.connect(member)) as conn, conn: - conn.execute("DELETE FROM dictionary WHERE \"table\"='block_0'") - path.write_text( - "[" * 200000 + "0" + "]" * 200000 if failure == "deep-json" else json.dumps(manifest) - ) + path.write_text("[" * 200000 + "0" + "]" * 200000) before = hashes(result.db_path) monkeypatch.setattr( service._msinsight_backend.replay, "stage", lambda *a, **k: pytest.fail("must preserve") @@ -524,7 +502,7 @@ def test_invalid_producer_extension_reports_context(tmp_path, field, value, mess @pytest.mark.parametrize("external", [False, True]) @pytest.mark.parametrize("mutation", ["block", "trace"]) -def test_resolver_rejects_seal_mismatch_after_prior_query(tmp_path, external, mutation): +def test_resolver_accepts_valid_edits_without_digest_comparison(tmp_path, external, mutation): result = ImportService().import_snapshot(options(tmp_path, focus=False)) if external: path = result.db_path / "manifest.json" @@ -541,10 +519,7 @@ def test_resolver_rejects_seal_mismatch_after_prior_query(tmp_path, external, mu else "UPDATE trace_entry_0 SET allocated=allocated+1" ) before = hashes(result.db_path) - from pt_snap_cli.core.errors import DatabaseSchemaError - - with pytest.raises(DatabaseSchemaError, match="parserCache.*seal"): - resolver.inspect(result.db_path) + assert resolver.inspect(result.db_path) is not None assert hashes(result.db_path) == before @@ -573,11 +548,10 @@ def test_live_consumer_wal_reuses_and_queries_committed_data(tmp_path, monkeypat with SnapshotAnalyzer(result.db_path) as analyzer: assert analyzer.execute_query("event", {"id": 0}, device_id=0)["rows"] assert writer.execute("SELECT value FROM consumer_data").fetchall() == [("committed",)] - # Base-table writes cannot masquerade as consumer-only additions. + # Cache reuse no longer scans base-table contents, including committed WAL. writer.execute("UPDATE block_0 SET size=size+1") writer.commit() - with pytest.raises(ImportExecutionError, match="preserved"): - service.import_snapshot(request) + assert service.import_snapshot(request).reused def test_compatible_negative_ids_are_compact_and_boundary_ids_are_global(tmp_path): diff --git a/tests/core/test_ordered_frame_integrity.py b/tests/core/test_ordered_frame_integrity.py new file mode 100644 index 0000000..03fb8af --- /dev/null +++ b/tests/core/test_ordered_frame_integrity.py @@ -0,0 +1,152 @@ +"""Owned frame evidence must survive publication and repeat-import checks.""" + +import dataclasses +import hashlib +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +import pytest + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core import import_service +from pt_snap_cli.core.errors import ImportExecutionError +from pt_snap_cli.core.import_service import ImportService +from tests.core.test_dataset_import import hashes +from tests.core.test_msinsight_export import options +from tests.snapshot.test_sharded_replay import lifecycle_data, write_source + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + + +MUTATIONS = { + "missing-table": "DROP TABLE pt_snap_stack_frame", + "bad-count": "UPDATE pt_snap_stack SET frameCount=frameCount+1", + "bad-index": "UPDATE pt_snap_stack_frame SET frameIndex=frameIndex+100", + "orphan-frame": "UPDATE pt_snap_stack_frame SET frameId=999999", + "orphan-event": "INSERT INTO pt_snap_event_stack VALUES(999999,0)", + "nonfinite-json": "UPDATE pt_snap_frame_value SET frameJson='{\"name\":NaN}'", + "duplicate-key": 'UPDATE pt_snap_frame_value SET frameJson=\'{"name":1,"name":2}\'', +} + + +@pytest.mark.parametrize("fmt", [None]) +@pytest.mark.parametrize("mutation", MUTATIONS) +def test_invalid_new_frames_never_publish(tmp_path, monkeypatch, fmt, mutation): + service = ImportService() + request = dataclasses.replace(options(tmp_path, focus=False), format=fmt) + backend = service._msinsight_backend if fmt else service._dataset_backend + stage = backend.replay.stage + + def damaged_stage(*args, **kwargs): + result = stage(*args, **kwargs) + with closing(sqlite3.connect(result.slices[0].file)) as conn, conn: + conn.execute(MUTATIONS[mutation]) + return result + + monkeypatch.setattr(backend.replay, "stage", damaged_stage) + with pytest.raises(ImportExecutionError): + service.import_snapshot(request) + target = backend.target_path(request.snapshot_file, request.output_dir) + assert not target.exists() + assert not list(request.output_dir.glob(".*stage*")) + assert not (tmp_path / ".pt-snap").exists() + + +@pytest.mark.parametrize("fmt", [None]) +def test_invalid_new_declaration_never_publishes(tmp_path, monkeypatch, fmt): + name = "build_compatible_manifest" if fmt else "build_native_manifest" + build = getattr(import_service, name) + + def missing_declaration(*args, **kwargs): + manifest = build(*args, **kwargs) + manifest.pop("extensions") + return manifest + + monkeypatch.setattr(import_service, name, missing_declaration) + service = ImportService() + request = dataclasses.replace(options(tmp_path, focus=False), format=fmt) + with pytest.raises(ImportExecutionError): + service.import_snapshot(request) + backend = service._msinsight_backend if fmt else service._dataset_backend + assert not backend.target_path(request.snapshot_file, request.output_dir).exists() + + +@pytest.mark.parametrize("fmt", [None]) +@pytest.mark.parametrize("mutation", ["missing-table", "frame-content", "declaration"]) +def test_repeat_import_rejects_changed_frames_without_overwriting(tmp_path, fmt, mutation): + service = ImportService() + request = dataclasses.replace(options(tmp_path, focus=False), format=fmt) + first = service.import_snapshot(request) + assert not first.reused + assert service.import_snapshot(request).reused + path = first.db_path / "manifest.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + member = first.db_path / "device_0/slice_00000.db" + if mutation == "declaration": + manifest.pop("extensions") + else: + with closing(sqlite3.connect(member)) as conn, conn: + conn.execute( + MUTATIONS[mutation] + if mutation == "missing-table" + else 'UPDATE pt_snap_frame_value SET frameJson=\'{"name":"changed"}\'' + ) + if fmt is None and mutation == "missing-table": + # Resealed malformed native members must still fail schema validation. + manifest["devices"]["0"]["slices"][0]["sha256"] = hashlib.sha256( + member.read_bytes() + ).hexdigest() + path.write_text(json.dumps(manifest), encoding="utf-8") + before = hashes(first.db_path) + with pytest.raises(ImportExecutionError): + service.import_snapshot(request) + assert hashes(first.db_path) == before + assert not (tmp_path / ".pt-snap").exists() + if fmt is None: + rebuilt = service.import_snapshot(dataclasses.replace(request, force=True)) + assert not rebuilt.reused + assert service.import_snapshot(request).reused + + +@pytest.mark.parametrize("fmt", [None, "msinsight"]) +@pytest.mark.parametrize("nested_lists,covered", [(126, True), (127, False)]) +def test_optional_frame_depth_does_not_reject_text_import(tmp_path, fmt, nested_lists, covered): + request = dataclasses.replace(options(tmp_path, focus=False), format=fmt) + data = lifecycle_data() + data["device_traces"][0] = [e for e in data["device_traces"][0] if e["action"] != "oom"] + extra = 0 + for _ in range(nested_lists): + extra = [extra] + frames = [ + {"filename": "valid.py", "line": 2, "name": "valid"}, + {"filename": "edge.py", "line": 3, "name": "edge", "extra": extra}, + ] + data["device_traces"][0][0]["frames"] = frames + write_source(tmp_path, data) + service = ImportService() + first = service.import_snapshot(request) + assert not first.reused + if not covered: + member = first.db_path / "device_0/slice_00000.db" + with closing(sqlite3.connect(member.as_uri() + "?mode=ro", uri=True)) as conn: + assert not conn.execute("SELECT 1 FROM pt_snap_event_stack WHERE eventId=0").fetchall() + assert not conn.execute( + "SELECT 1 FROM pt_snap_frame_value WHERE frameJson LIKE '%valid.py%'" + ).fetchall() + before = hashes(first.db_path) + assert service.import_snapshot(request).reused + with SnapshotAnalyzer(first.db_path) as analyzer: + row = analyzer.execute_query("event", {"id": 0})["rows"][0] + assert row["callstack"] == "edge.py:3 edge\nvalid.py:2 valid" + assert row["frames_status"] == ("ordered" if covered else "text_only") + assert row["frames"] == (frames if covered else None) + following = analyzer.execute_query("event", {"id": 1})["rows"][0] + assert following["frames_status"] == "ordered" + assert hashes(first.db_path) == before diff --git a/tests/core/test_snapshot_cache.py b/tests/core/test_snapshot_cache.py index 0cf6d01..850621e 100644 --- a/tests/core/test_snapshot_cache.py +++ b/tests/core/test_snapshot_cache.py @@ -9,7 +9,6 @@ from pathlib import Path from pt_snap_cli.core.snapshot_cache import ( - _RowEncoder, seal_manifest, source_hashes, validate_cache, @@ -17,20 +16,6 @@ class SnapshotCacheTest(unittest.TestCase): - def test_row_encoder_preserves_digest_bytes_and_bounds_cache(self): - encoder = _RowEncoder() - for stack in ("", "ascii", "\u4e2d\u6587", '"\\\n\t', None, "x" * (8 * 1024 * 1024)): - row = (1, 2, 3, 4, 0, 5, 6, 7, stack) - expected = json.dumps( - row, ensure_ascii=True, separators=(",", ":"), allow_nan=False - ).encode() - self.assertEqual(encoder.encode(row, True), expected) - self.assertEqual(encoder.encode(row, True), expected) - for index in range(3000): - encoder.encode((index, str(index)), True) - self.assertLessEqual(len(encoder.stacks), 2048) - self.assertLessEqual(encoder.size, 8 * 1024 * 1024) - def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) @@ -114,81 +99,6 @@ def open_during_write(*args, **kwargs): with patch.object(snapshot_cache.sqlite3, "connect", side_effect=open_during_write): self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - def test_base_write_after_sidecar_check_is_detected(self): - from unittest.mock import patch - - from pt_snap_cli.core import snapshot_cache - - connect = sqlite3.connect - writers = [] - self.addCleanup(lambda: [writer.close() for writer in writers]) - - def open_after_base_write(*args, **kwargs): - writer = connect(self.db) - writers.append(writer) - writer.execute("PRAGMA journal_mode=WAL") - writer.execute("UPDATE block_0 SET size=size+1") - writer.commit() - return connect(*args, **kwargs) - - with patch.object(snapshot_cache.sqlite3, "connect", side_effect=open_after_base_write): - with self.assertRaisesRegex(ValueError, "parser data"): - validate_cache(self.directory, self.hash) - - def test_consumer_commit_during_row_scan_is_allowed(self): - from unittest.mock import patch - - from pt_snap_cli.core import snapshot_cache - - with closing(sqlite3.connect(self.db)) as writer: - writer.execute("PRAGMA journal_mode=WAL") - encode = snapshot_cache._RowEncoder.encode - committed = False - - def encode_while_writing(encoder, row, inline_stack): - nonlocal committed - if not committed: - writer.execute("CREATE TABLE consumer_cache(value INTEGER)") - writer.commit() - writer.execute("PRAGMA wal_checkpoint(PASSIVE)") - committed = True - return encode(encoder, row, inline_stack) - - with patch.object(snapshot_cache._RowEncoder, "encode", encode_while_writing): - self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - self.assertTrue(committed) - - def test_parser_seal_does_not_override_producer_seal(self): - self.manifest["ptSnap"] = { - "memberHashKind": "parser-v1", - "members": {"slice.db": "0" * 64}, - } - self.save() - with self.assertRaisesRegex(ValueError, "member hash"): - validate_cache(self.directory, self.hash) - - def test_logical_producer_seal_without_parser_seal_is_checked(self): - self.manifest["ptSnap"] = { - "memberHashKind": "parser-v1", - "members": self.manifest.pop("parserCache")["members"], - } - self.save() - self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - with closing(sqlite3.connect(self.db)) as conn: - conn.execute("UPDATE block_0 SET size=size+1") - conn.commit() - with self.assertRaisesRegex(ValueError, "member hash"): - validate_cache(self.directory, self.hash) - - def test_legacy_producer_seal_is_not_overridden_by_parser_seal(self): - self.manifest["ptSnap"] = {"members": {"slice.db": source_hashes(self.db)[0]}} - self.save() - self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - with closing(sqlite3.connect(self.db)) as conn: - conn.execute("CREATE TABLE consumer_extra(value INTEGER)") - with self.assertRaisesRegex(ValueError, "member hash"): - validate_cache(self.directory, self.hash) - def test_legacy_physical_seal_accepts_closed_wal_database(self): with closing(sqlite3.connect(self.db)) as conn: conn.execute("PRAGMA journal_mode=WAL") @@ -197,22 +107,11 @@ def test_legacy_physical_seal_accepts_closed_wal_database(self): self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - def test_legacy_physical_seal_rejects_committed_wal(self): - with closing(sqlite3.connect(self.db)) as conn: - conn.execute("PRAGMA journal_mode=WAL") - self.manifest["ptSnap"] = {"members": {"slice.db": source_hashes(self.db)[0]}} - self.save() - with closing(sqlite3.connect(self.db)) as writer: - writer.execute("CREATE TABLE consumer_cache(value INTEGER)") - writer.commit() - with self.assertRaisesRegex(ValueError, "sidecars"): - validate_cache(self.directory, self.hash) - def test_source_hash_matches_shared_salt(self): raw, salted = source_hashes(self.source) self.assertEqual(raw, hashlib.sha256(self.source.read_bytes()).hexdigest()) self.assertEqual( - salted, hashlib.sha256(b"mem_snapshot_parser_v2" + self.source.read_bytes()).hexdigest() + salted, hashlib.sha256(b"mem_snapshot_parser_v3" + self.source.read_bytes()).hexdigest() ) def test_extra_tables_indexes_and_dictionary_do_not_invalidate(self): @@ -233,20 +132,6 @@ def test_extra_tables_indexes_and_dictionary_do_not_invalidate(self): conn.execute("VACUUM") self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - def test_modified_base_data_is_rejected(self): - for table, assignment in ( - ("block_0", "size=size+1"), - ("trace_entry_0", "allocated=allocated+1"), - ("dictionary", "\"value\"='changed'"), - ): - with self.subTest(table=table): - with closing(sqlite3.connect(self.db)) as conn, conn: - conn.execute(f"UPDATE {table} SET {assignment}") - with self.assertRaises(ValueError): - validate_cache(self.directory, self.hash) - seal_manifest(self.directory, self.manifest) - self.save() - def test_changed_source_is_rejected(self): self.source.write_bytes(b"changed") with self.assertRaises(ValueError): @@ -265,15 +150,6 @@ def test_rollback_journal_is_rejected(self): with self.assertRaisesRegex(ValueError, "rollback journal"): validate_cache(self.directory, self.hash) - def test_active_wal_is_read_instead_of_ignoring_committed_rows(self): - with closing(sqlite3.connect(self.db)) as conn: - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("UPDATE block_0 SET size=size+1") - conn.commit() - self.assertTrue(Path(str(self.db) + "-wal").exists()) - with self.assertRaises(ValueError): - validate_cache(self.directory, self.hash) - def test_missing_or_corrupt_db_is_rejected_without_creating_it(self): self.db.unlink() with self.assertRaises(ValueError): @@ -369,51 +245,79 @@ def change_manifest(directory, manifest, **kwargs): with self.assertRaisesRegex(ValueError, "manifest changed"): validate_cache(self.directory, self.hash) - def test_parser_salt_changes_cache_identity(self): - from unittest.mock import patch + def test_v2_cache_is_not_reused_by_v3_parser(self): + self.manifest["cacheHash"] = hashlib.sha256( + b"mem_snapshot_parser_v2" + self.source.read_bytes() + ).hexdigest() + self.save() + with self.assertRaisesRegex(ValueError, "source hash or parser salt changed"): + validate_cache(self.directory, source_hashes(self.source)[1]) - from pt_snap_cli.core import snapshot_cache + def test_deep_manifest_is_a_validation_error(self): + (self.directory / "manifest.json").write_text("[" * 200000 + "0" + "]" * 200000) + with self.assertRaisesRegex(ValueError, "nesting"): + validate_cache(self.directory, self.hash) - with patch.object(snapshot_cache, "PARSER_HASH_SALT", b"another-parser-version"): - with self.assertRaises(ValueError): - validate_cache(self.directory, source_hashes(self.source)[1]) + def test_cross_window_lifetime_references_remain_valid(self): + with closing(sqlite3.connect(self.db)) as conn, conn: + conn.execute("UPDATE block_0 SET allocEventId=-1, freeEventId=999") + seal_manifest(self.directory, self.manifest) + self.save() + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) - def test_corrupt_secondary_index_is_rejected(self): + def test_default_validation_rejects_missing_required_table(self): with closing(sqlite3.connect(self.db)) as conn: - conn.execute("CREATE INDEX block_address ON block_0(address)") - page_size = conn.execute("PRAGMA page_size").fetchone()[0] - root = conn.execute( - "SELECT rootpage FROM sqlite_master WHERE name='block_address'" - ).fetchone()[0] - with self.db.open("r+b") as file: - file.seek((root - 1) * page_size) - file.write(b"\xff" * page_size) - with self.assertRaises((ValueError, sqlite3.DatabaseError)): + conn.execute("DROP TABLE block_0") + conn.commit() + with self.assertRaisesRegex(ValueError, "Missing parser table"): validate_cache(self.directory, self.hash) - def test_deep_manifest_is_a_validation_error(self): - (self.directory / "manifest.json").write_text("[" * 200000 + "0" + "]" * 200000) - with self.assertRaisesRegex(ValueError, "nesting"): + def test_default_validation_rejects_missing_member(self): + self.db.unlink() + with self.assertRaises(ValueError): validate_cache(self.directory, self.hash) + self.assertFalse(self.db.exists()) - def test_legacy_dictionary_references_are_required(self): - del self.manifest["parserCache"] - for table in ("trace_entry_0", "block_0"): - with self.subTest(table=table): - with closing(sqlite3.connect(self.db)) as conn, conn: - original = conn.execute( - 'SELECT * FROM dictionary WHERE "table"=?', (table,) - ).fetchall() - conn.execute('DELETE FROM dictionary WHERE "table"=?', (table,)) - self.save() - with self.assertRaisesRegex(ValueError, "dictionary reference"): - validate_cache(self.directory, self.hash) - with closing(sqlite3.connect(self.db)) as conn, conn: - conn.executemany("INSERT INTO dictionary VALUES(?,?,?,?)", original) + def test_default_validation_rejects_unreadable_database(self): + self.db.write_bytes(b"not a database") + with self.assertRaises(sqlite3.DatabaseError): + validate_cache(self.directory, self.hash) - def test_cross_window_lifetime_references_remain_valid(self): + def test_oom_without_dictionary_reference_is_accepted(self): with closing(sqlite3.connect(self.db)) as conn, conn: - conn.execute("UPDATE block_0 SET allocEventId=-1, freeEventId=999") + conn.execute("UPDATE trace_entry_0 SET action='oom'") seal_manifest(self.directory, self.manifest) self.save() self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) + + def test_cache_validation_never_scans_rows_or_pages(self): + from unittest.mock import patch + + statements = [] + connect = sqlite3.connect + + def traced(*args, **kwargs): + conn = connect(*args, **kwargs) + conn.set_trace_callback(statements.append) + return conn + + with patch.object(sqlite3, "connect", side_effect=traced): + seal_manifest(self.directory, self.manifest) + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) + sql = "\n".join(statements).upper() + self.assertNotIn("QUICK_CHECK", sql) + self.assertNotIn("INTEGRITY_CHECK", sql) + self.assertNotIn("DISTINCT", sql) + self.assertNotIn( + "ORDER BY", + sql.replace("ORDER BY ID DESC LIMIT 1", "").replace("ORDER BY ID LIMIT 1", ""), + ) + + def test_legacy_content_digests_are_not_compared(self): + self.manifest["parserCache"]["members"] = {"slice.db": "old-content-digest"} + self.manifest["ptSnap"] = { + "memberHashKind": "parser-v1", + "members": {"slice.db": "different-old-digest"}, + } + self.save() + self.assertEqual(validate_cache(self.directory, self.hash), self.manifest) diff --git a/tests/core/wheel_scale_acceptance.py b/tests/core/wheel_scale_acceptance.py index 6cdd38c..4313f14 100644 --- a/tests/core/wheel_scale_acceptance.py +++ b/tests/core/wheel_scale_acceptance.py @@ -111,6 +111,7 @@ def __init__(self, root, wheel, evidence_dir, wheelhouse=None): PATH=f"{bin_dir}{os.pathsep}{self.env.get('PATH', '')}", PWD=str(self.cwd), PYTHONNOUSERSITE="1", + PYTHONUTF8="1", PIP_DISABLE_PIP_VERSION_CHECK="1", NO_COLOR="1", ) @@ -142,7 +143,13 @@ def __init__(self, root, wheel, evidence_dir, wheelhouse=None): def run(self, label, command, *, expected=0): started = time.perf_counter() result = subprocess.run( - command, cwd=self.cwd, env=self.env, capture_output=True, text=True, check=False + command, + cwd=self.cwd, + env=self.env, + capture_output=True, + text=True, + encoding="utf-8", + check=False, ) self.commands.append( { diff --git a/tests/query/test_executor.py b/tests/query/test_executor.py index 35970d5..1d3c53f 100644 --- a/tests/query/test_executor.py +++ b/tests/query/test_executor.py @@ -15,6 +15,43 @@ ) +@pytest.mark.parametrize("method", ["execute", "count"]) +def test_sqlite_cancellation_without_timeout_clears_progress_handler(tmp_path, method): + path = tmp_path / "cancel.db" + with closing(sqlite3.connect(path)) as conn: + conn.execute("CREATE TABLE dictionary (id INTEGER PRIMARY KEY, value TEXT)") + conn.commit() + context = Context(path, persistent=True) + try: + executor = QueryExecutor(context) + checks = 0 + + def cancelled(): + nonlocal checks + checks += 1 + return checks >= 3 + + sql = ( + "WITH RECURSIVE n(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM n WHERE x<100000000) " + "SELECT SUM(x) AS value FROM n" + ) + with pytest.raises(QueryExecutionError, match="interrupted"): + getattr(executor, method)(sql, cancelled=cancelled) + assert checks == 3 + assert executor.execute("SELECT 42 AS value") == [{"value": 42}] + finally: + context.close() + + +def test_executor_clone_keeps_runtime_templates_without_sharing_registrations(): + first = QueryExecutor(context=None) + first.register_template(QueryTemplate(name="custom", query="SELECT 1", parameters={})) + worker = first.clone_for_context(None) + assert worker.has_runtime_template("custom") + worker.register_template(QueryTemplate(name="worker_only", query="SELECT 2", parameters={})) + assert not first.has_runtime_template("worker_only") + + class TestQueryExecutor: def test_render_simple_template(self): template = QueryTemplate( diff --git a/tests/query/test_frame_tree.py b/tests/query/test_frame_tree.py index 98e5b82..e10b4de 100644 --- a/tests/query/test_frame_tree.py +++ b/tests/query/test_frame_tree.py @@ -182,7 +182,7 @@ def test_inline_text_database_retains_queries_but_cannot_supply_original_frames( assert query(path, "event", {"id": 1}).rows[0]["callstack"] == "train.py:10" -@pytest.mark.parametrize("name", ["event_frames", "active_memory_frame_tree_at_event"]) +@pytest.mark.parametrize("name", ["event_frames"]) @pytest.mark.parametrize("slice_index", [None, 0]) def test_dataset_root_never_implicitly_selects_a_shard(tmp_path: Path, name: str, slice_index): from pt_snap_cli.core.dataset_support import DATASET_SUPPORT diff --git a/tests/test_clean_wheel_agent_acceptance.py b/tests/test_clean_wheel_agent_acceptance.py index 5dd3141..4cc5616 100644 --- a/tests/test_clean_wheel_agent_acceptance.py +++ b/tests/test_clean_wheel_agent_acceptance.py @@ -2,11 +2,13 @@ from __future__ import annotations +import hashlib import importlib.util import json import os import shlex import sys +from copy import deepcopy from pathlib import Path import pytest @@ -324,3 +326,466 @@ def cli(command, **kwargs): ) assert [command[1] for command in commands] == ["import", "import", "overview"] assert focus.read_text() == "unchanged" + + +def _frame_identity(frames, fingerprint="generation-1", device=0): + canonical = json.dumps( + frames, sort_keys=True, ensure_ascii=True, separators=(",", ":"), allow_nan=False + ) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return f"dataset:{fingerprint}:device:{device}:frames:sha256:{digest}" + + +def _standalone_query(root, template, rows): + return { + "db_path": str(root / "standalone.db"), + "focus_source": "explicit", + "device_id": 0, + "template": template, + "rows": rows, + "returned": len(rows), + "total": len(rows), + "total_is_exact": True, + "has_more": False, + "truncated": False, + } + + +def _event_parity_case(root): + # Recursive duplicates and distinct neighbors make both deduplication and + # reordering observable. Raw extras are checked by identity, not falsely + # presented as fields exposed by standalone event_frames. + leaf = {"filename": "tensor.py", "line": 1, "name": "leaf", "extra": {"flag": True}} + frames = [ + leaf, + {"filename": "model.py", "line": 20, "name": "forward"}, + deepcopy(leaf), + {"filename": "train.py", "line": 30, "name": "train"}, + ] + base = [ + { + "id": 4095, + "action": 4, + "address": 4096, + "size": 1, + "stream": 0, + "allocated": 1, + "active": 1, + "reserved": 4096, + "callstack": "full callstack", + }, + { + "id": 4096, + "action": 6, + "address": 4096, + "size": 1, + "stream": 0, + "allocated": 0, + "active": 0, + "reserved": 4096, + "callstack": "", + }, + ] + reference = _standalone_query(root, "event", base) + crossing = deepcopy(reference) + crossing["db_path"] = str(root / "dataset") + crossing["scope"] = { + "source_coverage": { + "range_complete": True, + "event_position": "after", + "coverage_scope": "returned_events", + "events_resolved": 2, + "ordered_frame_events": 2, + "source_queries": 8, + } + } + frame_references = {} + for row, raw in zip(crossing["rows"], (frames, []), strict=True): + identity = _frame_identity(raw) + row.update( + source_stack_id=identity, + stack_id=identity, + stack_kind="captured", + text_kind="captured" if row["callstack"] else "missing", + stack_event_id=row["id"], + frames=raw, + frames_status="ordered", + ) + frame_references[row["id"]] = _standalone_query( + root, + "event_frames", + [ + { + "event_id": row["id"], + "stack_id": 77, + "position": index, + "frame_id": (5, 9, 5, 11)[index], + **{key: frame[key] for key in ("filename", "line", "name")}, + } + for index, frame in enumerate(raw) + ], + ) + return crossing, reference, frame_references + + +def _check_event_parity(acceptance, root, case): + return acceptance._event_parity( + *case, + standalone=root / "standalone.db", + fingerprint="generation-1", + device=0, + event_ids=[4095, 4096], + ) + + +def test_event_parity_accepts_ordered_recursive_and_empty_stacks(acceptance, tmp_path): + case = _event_parity_case(tmp_path) + unchanged = deepcopy(case) + assert _check_event_parity(acceptance, tmp_path, case) == 4 + assert case == unchanged + + +@pytest.mark.parametrize( + "field", + ["id", "action", "address", "size", "stream", "allocated", "active", "reserved", "callstack"], +) +@pytest.mark.parametrize("damage", ["value", "missing", "type"]) +def test_event_parity_never_masks_base_event_changes(acceptance, tmp_path, field, damage): + case = _event_parity_case(tmp_path) + row = case[0]["rows"][0] + value = row[field] + if damage == "missing": + del row[field] + elif damage == "value": + row[field] = value + (" changed" if isinstance(value, str) else 1) + else: + row[field] = None if isinstance(value, str) else float(value) + with pytest.raises(acceptance.AcceptanceError, match="base field|event fields"): + _check_event_parity(acceptance, tmp_path, case) + + +@pytest.mark.parametrize( + "damage", ["event_order", "missing_event", "unknown_field", "missing_detail"] +) +def test_event_parity_rejects_changed_event_shape_or_order(acceptance, tmp_path, damage): + case = _event_parity_case(tmp_path) + rows = case[0]["rows"] + if damage == "event_order": + rows.reverse() + elif damage == "missing_event": + rows.pop() + elif damage == "unknown_field": + rows[0]["unrecognized"] = "not silently stripped" + else: + del rows[0]["frames_status"] + with pytest.raises(acceptance.AcceptanceError): + _check_event_parity(acceptance, tmp_path, case) + + +@pytest.mark.parametrize( + "damage", + [ + "drop", + "empty", + "duplicate", + "reorder", + "filename", + "line", + "name", + "missing_field", + "float_line", + "bool_line", + ], +) +def test_event_parity_rejects_frame_damage_even_with_updated_identity(acceptance, tmp_path, damage): + case = _event_parity_case(tmp_path) + row = case[0]["rows"][0] + frames = row["frames"] + if damage == "drop": + frames.pop(2) + elif damage == "empty": + frames.clear() + elif damage == "duplicate": + frames.append(deepcopy(frames[0])) + elif damage == "reorder": + frames.reverse() + elif damage == "missing_field": + del frames[0]["filename"] + elif damage == "float_line": + frames[0]["line"] = 1.0 + elif damage == "bool_line": + frames[0]["line"] = True + else: + frames[0][damage] = 999 if damage == "line" else "corrupt" + row["source_stack_id"] = row["stack_id"] = _frame_identity(frames) + with pytest.raises(acceptance.AcceptanceError, match="ordered frame count|ordered frame field"): + _check_event_parity(acceptance, tmp_path, case) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("frames", None), + ("frames", ["not a frame"]), + ("frames_status", "text_only"), + ("stack_kind", "missing"), + ("text_kind", "missing"), + ("stack_event_id", 4096), + ("stack_event_id", 4095.0), + ("stack_id", "local:77"), + ("source_stack_id", "local:77"), + ], +) +def test_event_parity_requires_full_ordered_source_evidence(acceptance, tmp_path, field, value): + case = _event_parity_case(tmp_path) + case[0]["rows"][0][field] = value + with pytest.raises(acceptance.AcceptanceError): + _check_event_parity(acceptance, tmp_path, case) + + +@pytest.mark.parametrize("damage", ["fingerprint", "device", "digest", "extra_raw_field"]) +def test_event_parity_checks_full_raw_digest_and_namespace(acceptance, tmp_path, damage): + case = _event_parity_case(tmp_path) + row = case[0]["rows"][0] + if damage == "extra_raw_field": + row["frames"][0]["extra"]["flag"] = False + else: + identity = _frame_identity( + row["frames"], + "generation-2" if damage == "fingerprint" else "generation-1", + 1 if damage == "device" else 0, + ) + if damage == "digest": + identity = identity[:-1] + ("0" if identity[-1] != "0" else "1") + row["source_stack_id"] = row["stack_id"] = identity + with pytest.raises(acceptance.AcceptanceError, match="identity/namespace"): + _check_event_parity(acceptance, tmp_path, case) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("range_complete", False), + ("event_position", "before"), + ("coverage_scope", "all_events"), + ("events_resolved", 1), + ("events_resolved", 2.0), + ("ordered_frame_events", 1), + ("ordered_frame_events", 2.0), + ("source_queries", 0), + ("source_queries", True), + ], +) +def test_event_parity_rejects_incomplete_or_misleading_source_coverage( + acceptance, tmp_path, field, value +): + case = _event_parity_case(tmp_path) + case[0]["scope"]["source_coverage"][field] = value + with pytest.raises(acceptance.AcceptanceError, match="source coverage"): + _check_event_parity(acceptance, tmp_path, case) + + +@pytest.mark.parametrize("template", ["event", "event_frames"]) +@pytest.mark.parametrize( + ("field", "value"), + [ + ("db_path", "/wrong"), + ("focus_source", "project"), + ("device_id", 1), + ("device_id", False), + ("template", "wrong"), + ("returned", -1), + ("total", -1), + ("total_is_exact", False), + ("has_more", True), + ("truncated", True), + ], +) +def test_event_parity_rejects_incomplete_or_wrong_standalone_reference( + acceptance, tmp_path, template, field, value +): + case = _event_parity_case(tmp_path) + query = case[1] if template == "event" else case[2][4095] + query[field] = value + with pytest.raises(acceptance.AcceptanceError, match="Standalone"): + _check_event_parity(acceptance, tmp_path, case) + + +@pytest.mark.parametrize( + ("field", "value"), + [("event_id", 4096), ("position", 1), ("position", False), ("frame_id", "5"), ("stack_id", -1)], +) +def test_event_parity_validates_frame_reference_locators(acceptance, tmp_path, field, value): + case = _event_parity_case(tmp_path) + case[2][4095]["rows"][0][field] = value + with pytest.raises(acceptance.AcceptanceError, match="standalone frame"): + _check_event_parity(acceptance, tmp_path, case) + + +@pytest.mark.parametrize( + ("empty_boundary", "supplement_empty"), [(False, False), (True, False), (True, True)] +) +def test_sharded_cli_uses_independent_frame_references_for_both_formats( + acceptance, tmp_path, monkeypatch, empty_boundary, supplement_empty +): + focus = tmp_path / "focus.json" + focus.write_text("unchanged") + standalone = tmp_path / "standalone.db" + devices = [{"device_id": 0, "first_event_id": 0, "last_event_id": 8191}] + standalone_peak = {"device_id": 0, "rows": [{"peak_allocated": 1}]} + payloads = {} + for format_name in ("pt-snap-native-v2", "compatibility-v1"): + root = tmp_path / format_name + root.mkdir() + members = [] + for index in (0, 1): + member = root / f"slice_{index:05d}.db" + member.write_bytes(b"not opened by mocked CLI") + members.append( + { + "index": index, + "db_path": str(member), + "first_event_id": index * 4096, + "last_event_id": (index + 1) * 4096 - 1, + } + ) + overview = _overview(root) + overview["devices"] = devices + overview["dataset"]["format"] = format_name + overview["dataset"]["devices"] = [{"device_id": 0, "event_count": 8192, "slices": members}] + payloads[format_name] = overview + imports = {} + calls = [] + crossing, reference, frame_references = _event_parity_case(tmp_path) + captured = deepcopy(crossing["rows"][0]) + captured["id"] = captured["stack_event_id"] = 1 + captured_reference = deepcopy(reference["rows"][0]) + captured_reference["id"] = 1 + captured_frames = deepcopy(frame_references[4095]) + for row in captured_frames["rows"]: + row["event_id"] = 1 + if supplement_empty: + captured["frames"] = [] + captured["stack_id"] = captured["source_stack_id"] = _frame_identity([]) + captured_frames = _standalone_query(tmp_path, "event_frames", []) + if empty_boundary: + crossing["rows"][0]["frames"] = [] + crossing["rows"][0]["callstack"] = reference["rows"][0]["callstack"] = "" + crossing["rows"][0]["text_kind"] = "missing" + crossing["rows"][0]["stack_id"] = crossing["rows"][0]["source_stack_id"] = _frame_identity( + [] + ) + frame_references[4095] = _standalone_query(tmp_path, "event_frames", []) + + def cli(command, **kwargs): + calls.append(command) + assert command[0] == "installed-pt-snap" and command[-1] == "--json" + action = command[1] + if action == "import": + assert "--no-focus" in command + name = command[command.index("--format") + 1] + imports[name] = imports.get(name, 0) + 1 + return { + "db_path": str(tmp_path / name), + "dataset_path": str(tmp_path / name), + "format": name, + "rebuilt": imports[name] == 1, + "reused": imports[name] == 2, + "focus_state": None, + "slice_count": 2, + } + target = Path(command[2]) + root = target.parent if target.name == "manifest.json" else target + if action == "overview": + result = deepcopy(payloads[root.name]) + result["db_path"] = str(target) + return result + assert action == "query" + assert command[command.index("--device") + 1] == "0" + template = command[command.index("--template-use") + 1] + params = json.loads(command[command.index("--params") + 1]) if "--params" in command else {} + if template == "event_frames": + assert target == standalone and params["frame_order"] == "original" + assert "--exact-total" in command + return deepcopy( + captured_frames if params["event_id"] == 1 else frame_references[params["event_id"]] + ) + if target == standalone: + assert template == "event" + if params.get("limit") == 1: + assert params == { + "min_id": 0, + "order_by": "callstack", + "order_dir": "DESC", + "limit": 1, + } + # This locator is deliberately paginated, unlike the proof queries. + return { + **_standalone_query(tmp_path, "event", [captured_reference]), + "has_more": True, + "truncated": True, + "total_is_exact": False, + } + assert "--exact-total" in command + return deepcopy( + _standalone_query(tmp_path, "event", [captured_reference]) + if params.get("id") == 1 + else reference + ) + query = _query(root) + query["db_path"] = str(target) + query["template"] = template + query["scope"]["slice_indices"] = [0, 1] + if template == "memory_peak": + query["rows"] = standalone_peak["rows"] + query["scope"]["actual_range"] = {"real_event_count": 8192} + elif "id" in params: + event_id = params["id"] + query["scope"]["slice_indices"] = [event_id // 4096] + query["scope"]["source_coverage"] = {"range_complete": True} + query["rows"] = [{"id": event_id}] + if event_id == 1: + query["rows"] = [captured] + query["scope"]["source_coverage"] = { + **crossing["scope"]["source_coverage"], + "events_resolved": 1, + "ordered_frame_events": 1, + } + else: + assert params == {"min_id": 4095, "max_id": 4096} + assert "--exact-total" in command + query["rows"] = crossing["rows"] + query["scope"].update(crossing["scope"]) + query.update(returned=len(query["rows"]), total=len(query["rows"]), total_is_exact=True) + return deepcopy(query) + + monkeypatch.setattr(acceptance, "_run_json", cli) + + def run(): + return acceptance._sharded_acceptance( + Path("installed-pt-snap"), + tmp_path / "unused.pkl", + tmp_path, + 4096, + standalone, + {"devices": devices}, + standalone_peak, + focus, + cwd=tmp_path, + env={}, + ) + + if supplement_empty: + with pytest.raises(acceptance.AcceptanceError, match="no nonempty frame evidence"): + run() + assert focus.read_text() == "unchanged" + return + evidence = run() + assert [item["format"] for item in evidence] == ["pt-snap-native-v2", "compatibility-v1"] + assert all( + item["ordered_frame_field_parity"] and item["nonempty_ordered_frames_checked"] + for item in evidence + ) + frame_calls = [command for command in calls if "event_frames" in command] + assert len(frame_calls) == (6 if empty_boundary else 4) + assert focus.read_text() == "unchanged" diff --git a/tests/test_contract_cli_api.py b/tests/test_contract_cli_api.py index 4662d47..8f02699 100644 --- a/tests/test_contract_cli_api.py +++ b/tests/test_contract_cli_api.py @@ -24,7 +24,8 @@ from pt_snap_cli.cli import app from pt_snap_cli.core import InvalidDeviceError, QueryExecutionError, TemplateRenderError from pt_snap_cli.core.dataset_sources import QueryBudget -from pt_snap_cli.core.errors import QueryTimeoutError +from pt_snap_cli.core.error_codes import classify_error +from pt_snap_cli.core.errors import QueryCancelledError, QueryTimeoutError from pt_snap_cli.core.json_codec import JSON_SCHEMA_VERSION from pt_snap_cli.core.query_service import QueryService from pt_snap_cli.core.report_service import ReportService @@ -37,6 +38,36 @@ pytestmark = pytest.mark.usefixtures("owned_service_instances") +def test_cancelled_query_has_a_distinct_non_timeout_error_code(tmp_path, monkeypatch): + error = QueryCancelledError("Frame tree query cancelled by a newer request or reset.") + assert isinstance(error, QueryExecutionError) + assert classify_error(error)[0] == "QUERY_CANCELLED" + root = make_dataset(tmp_path / "cancelled", devices=(0,)) + + def cancelled_tree(*args, **kwargs): + raise error + + monkeypatch.setattr("pt_snap_cli.core.query_service.frame_tree", cancelled_tree) + with SnapshotAnalyzer(root) as analyzer: + with pytest.raises(QueryCancelledError): + analyzer.execute_query("active_memory_frame_tree_at_event", {"event_id": 1}) + result = runner.invoke( + app, + [ + "query", + str(root), + "--template-use", + "active_memory_frame_tree_at_event", + "--params", + '{"event_id":1}', + "--json", + ], + ) + assert result.exit_code == 1 + assert result.stdout == "" + assert json.loads(result.stderr)["error"]["code"] == "QUERY_CANCELLED" + + @pytest.mark.parametrize("template", ["event", "block"]) @pytest.mark.parametrize("params", [{"min_size": 2**63}, {"offset": 2**63, "limit": 1}]) def test_dataset_int64_overflow_cli_api_domain_error(tmp_path, template, params): diff --git a/tests/test_dataset_import_baseline.py b/tests/test_dataset_import_baseline.py index 2df08af..eb330e2 100644 --- a/tests/test_dataset_import_baseline.py +++ b/tests/test_dataset_import_baseline.py @@ -151,8 +151,21 @@ def test_reuse_miss_refuses_rebuild_and_preserves_existing_actual_bytes(tmp_path def test_recorded_performance_report_is_structural_not_a_speed_gate(): - from pt_snap_cli.core.dataset_support import DATASET_SUPPORT - + # This is historical evidence tied to report["source_head"], not a claim + # that queries introduced later were measured by that recorded run. + recorded_templates = { + "event", + "allocation", + "memory_peak", + "allocator_gap", + "active_blocks_at_event", + "active_memory_callstack_at_event", + "preexisting_live", + "block", + "freed_block_lifetime", + "leak_detection", + "callstack_analysis", + } repository = Path(__file__).resolve().parents[1] raw = (repository / "docs/dataset-performance-baseline.json").read_text() report = json.loads(raw) @@ -167,7 +180,7 @@ def test_recorded_performance_report_is_structural_not_a_speed_gate(): for record in report["records"]: assert record["exit"] == 0 and record["phase_rss"]["status"] == "unavailable" assert "RUSAGE_SELF" in record["rss_scope"] - assert set(record["queries"]) == set(DATASET_SUPPORT) + assert set(record["queries"]) == recorded_templates assert record["individual_shard_bytes"] and record["total_disk_bytes"] > 0 assert all(q["status"] == "passed" for q in record["queries"].values()) # Deliberately no timing/RSS/speedup threshold: these are observations. diff --git a/tests/test_interop_acceptance.py b/tests/test_interop_acceptance.py index ec050a7..719f62a 100644 --- a/tests/test_interop_acceptance.py +++ b/tests/test_interop_acceptance.py @@ -40,10 +40,12 @@ def isolate(tmp_path, monkeypatch): @pytest.mark.parametrize("device", [0, 2]) -def test_all_eleven_contracts_against_independent_physical_reference(tmp_path, device): +def test_text_contracts_against_independent_physical_reference(tmp_path, device): root = case(tmp_path) result = compare_artifact(root, device, tmp_path / "reference.db") - assert set(result["templates"]) == set(DATASET_SUPPORT) + # This independent physical reference has text stacks only. Frame paths + # have separate real-import differential coverage against a structured DB. + assert set(result["templates"]) == set(DATASET_SUPPORT) - {"active_memory_frame_tree_at_event"} assert result["real_events"] == 16 and result["canonical_lifecycles"] == 10 assert result["raw_block_rows"] == 61 and result["repeated_block_rows"] == 51 assert result["GUI"] == "human-deferred/pending/not-run" @@ -166,16 +168,49 @@ def test_compatible_owned_cache_reuse_and_external_force_preserve_actual_bytes(t reused = service.import_snapshot(dataclasses.replace(request, force=True)) assert reused.reused and reused.metadata is None assert inventory(result.db_path) == external - # Invalid external artifacts must still be preserved, including under force. + # Compatible reuse is structural, not content validation. Arbitrary row + # edits are deliberately not detected or repaired, even under force. with closing(sqlite3.connect(result.db_path / "device_0/slice_00000.db")) as conn, conn: conn.execute("UPDATE block_0 SET size=size+1") - invalid = inventory(result.db_path) - with pytest.raises(ImportExecutionError, match="preserv"): - service.import_snapshot(dataclasses.replace(request, force=True)) - assert inventory(result.db_path) == invalid + edited = inventory(result.db_path) + assert edited != external + for force in (False, True): + reused = service.import_snapshot(dataclasses.replace(request, force=force)) + assert reused.reused and reused.metadata is None + assert inventory(result.db_path) == edited assert before != external +@pytest.mark.parametrize("external", [False, True]) +@pytest.mark.parametrize("damage", ["member", "schema", "endpoint"]) +def test_compatible_structural_rejection_preserves_every_remaining_byte(tmp_path, external, damage): + request = options(tmp_path, focus=False) + service = ImportService() + result = service.import_snapshot(request) + manifest = result.db_path / "manifest.json" + if external: + raw = json.loads(manifest.read_bytes()) + raw.pop("ptSnap") + manifest.write_text(json.dumps(raw), encoding="utf-8") + member = result.db_path / "device_0/slice_00000.db" + if damage == "member": + member.unlink() + else: + with closing(sqlite3.connect(member)) as conn, conn: + if damage == "schema": + conn.execute("DROP TABLE block_0") + else: + # Indexed endpoints remain mandatory even without row scans. + changed = conn.execute("DELETE FROM trace_entry_0 WHERE id=0") + assert changed.rowcount == 1 + before = inventory(result.db_path) + for force in (False, True): + with pytest.raises(ImportExecutionError, match="preserv"): + service.import_snapshot(dataclasses.replace(request, force=force)) + assert inventory(result.db_path) == before + assert not (tmp_path / ".pt-snap").exists() + + @pytest.mark.parametrize("kind", ["trace", "block", "dictionary"]) def test_physical_row_bounds_include_negative_trace_and_dictionary(tmp_path, kind): root = case(tmp_path) diff --git a/tests/test_interop_three_shard.py b/tests/test_interop_three_shard.py index d8d6229..14a6cf6 100644 --- a/tests/test_interop_three_shard.py +++ b/tests/test_interop_three_shard.py @@ -184,7 +184,7 @@ def test_true_cli_import_keeps_one_event_per_owner_and_discovers_requested(impor if fmt == "msinsight": assert ( manifest["cacheHash"] - == hashlib.sha256(b"mem_snapshot_parser_v2" + source.read_bytes()).hexdigest() + == hashlib.sha256(b"mem_snapshot_parser_v3" + source.read_bytes()).hexdigest() ) assert {p: digest(p) for p in before} == before