diff --git a/CHANGELOG.md b/CHANGELOG.md index 838ce9c..9a21a3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - 新增 `mutation.schema_version: 1` 题目级人工排除：按稳定 mutation ID 记录有界非空理由，lint 拒绝未知字段、错误版本、无效/重复 ID、超量记录和非 standard Judge 配置；排除在 `--max-mutants` 前应用，失效旧 ID 保留并报告 warning。 - mutation evidence 升至 v2，并在 JSON、终端和 Markdown report 中并列 raw、excluded、effective、selected、out-of-scope 与 unmatched exclusion 计数；部分算子运行不会把其他算子的有效排除误报为失效。排除理由进入 source/计划哈希并逐项展示，Schema 类型、当前 Core/编译器身份、配置变化和输入变化均参与 current 判定；发布故障保留旧 evidence。人工排除仍不构成正确性证明或 build 硬门禁。 +- mutation 候选定位改为固定 Tree-sitter C++ 语法树，只接受函数/lambda 复合语句体中的真实表达式，排除模板尖括号、运算符声明、`<=>`、宏、concept/requires、`case` 标签、`static_assert` 与未求值上下文；保留仍有效的 `cpp-token-v1` ID，旧 evidence 变 stale，解析失败不回退 Token 扫描且不覆盖最后成功 evidence。新增 UTF-8、CRLF/LF 和复杂 C++ canonical plan Fixture，并把解析器版本纳入 builder fingerprint、Doctor smoke 与依赖审计。 ## [0.6.6] - 2026-08-09 diff --git a/README.md b/README.md index f1c727c..7163ffb 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ Agent 完成题目后，应明确报告下列结果： 对 `judge.type: custom` 或 `judge.type: interactive` 的新题，或修改 Checker/Interactor 后，先在 `judge.qa` 中登记题目级 fixture，再运行 `probhub judge-qa --no-cache`。fixture 每次都会执行，只有编译结果可以缓存；`judge-qa-evidence-v1.json` 是本地有界证据，不会进入 ZIP、PDF 或 Manifest。lint/status 中的 evidence 缺失或过期是体检 warning，但 `seal` 和正式 `build` 不允许已配置题目绕过通过的 Judge QA。 -标准题需要补充检查数据是否能区分常见边界错误时，可以运行 `probhub mutation --no-cache`。它只对首个 C++ accepted 生成少量稳定变异，成功后写入本地有界 evidence；变异体不会进入题目源码、PDF、ZIP 或 Manifest。确认某个变异等价或不适用后，可在 `probhub.yaml` 的 `mutation.exclusions` 中按稳定 ID 记录理由；`report` 会并列显示原始、排除、有效和实际选择数量，并区分本轮未选择对应算子的记录与已经失效的旧 ID。人工排除和 `survived` 都不是正确性证明，也不会替代证明、独立标程、期望矩阵和 stress。完整限制见 [std 变异测试](references/mutation-testing.md)。 +标准题需要补充检查数据是否能区分常见边界错误时，可以运行 `probhub mutation --no-cache`。它使用固定 C++ 语法树，只在首个 accepted 的函数/lambda 复合语句体中生成少量稳定变异，跳过模板、宏、`case` 标签和未求值语法；成功后写入本地有界 evidence，变异体不会进入题目源码、PDF、ZIP 或 Manifest。确认某个变异等价或不适用后，可在 `probhub.yaml` 的 `mutation.exclusions` 中按稳定 ID 记录理由；`report` 会并列显示原始、排除、有效和实际选择数量，并区分本轮未选择对应算子的记录与已经失效的旧 ID。人工排除和 `survived` 都不是正确性证明，也不会替代证明、独立标程、期望矩阵和 stress。完整限制见 [std 变异测试](references/mutation-testing.md)。 ## 并行出题时怎么做 diff --git a/SKILL.md b/SKILL.md index f4b836c..a3754a9 100644 --- a/SKILL.md +++ b/SKILL.md @@ -155,7 +155,7 @@ Judge QA 每次都会重新执行 fixture 和内建探针；`--no-cache` 只额 probhub mutation L01 --operator comparison-boundary --no-cache ``` -变异测试是开放世界探测的补充证据，不替代算法证明、独立标程、期望矩阵或 stress。`survived` 只说明当前测试数据没有区分该变异；编译失败和 Judge/Validator/资源故障不得计为击杀。只有主 Agent 审查源码并确认变异等价、不适用或无法表达真实错误后，才可在 `probhub.yaml` 的 `mutation.exclusions` 中按稳定 ID 写入非空理由；不得为了提高分数批量排除幸存变异。排除后必须重跑 mutation，并检查 `report` 的 raw/excluded/effective/selected、out-of-scope、unmatched exclusion 与理由。成功 evidence 原子写入题目 `.probhub`，失败或输入变化不覆盖上一份成功 evidence；该能力不作为 build 硬门禁，也不接入 Legacy、WebUI 或 Checker/Interactor 执行。 +变异测试是开放世界探测的补充证据，不替代算法证明、独立标程、期望矩阵或 stress。`survived` 只说明当前测试数据没有区分该变异；编译失败和 Judge/Validator/资源故障不得计为击杀。Core 使用固定 Tree-sitter C++ 语法树，只定位函数/lambda 复合语句体；解析失败时不得绕过错误或退回正则替换。只有主 Agent 审查源码并确认变异等价、不适用或无法表达真实错误后，才可在 `probhub.yaml` 的 `mutation.exclusions` 中按稳定 ID 写入非空理由；不得为了提高分数批量排除幸存变异。排除后必须重跑 mutation，并检查 `report` 的 raw/excluded/effective/selected、out-of-scope、unmatched exclusion 与理由。成功 evidence 原子写入题目 `.probhub`，失败或输入变化不覆盖上一份成功 evidence；该能力不作为 build 硬门禁，也不接入 Legacy、WebUI 或 Checker/Interactor 执行。 完整语法、产物、退出码和故障处理见 `references/cli.md`。配置或执行差分测试前读取 `references/stress.md`；修改资源限制、解释 OLE 或排查残留进程时读取 `references/process-control.md`。 diff --git a/probhub/doctor.py b/probhub/doctor.py index 9f3f182..2ec5ac9 100644 --- a/probhub/doctor.py +++ b/probhub/doctor.py @@ -9,9 +9,19 @@ from pathlib import Path from .builder_fingerprint import builder_toolchain_status +from .mutation_syntax import TREE_SITTER_CPP_VERSION, TREE_SITTER_VERSION from .process_control import run_managed_to_files +_MUTATION_PARSER_SMOKE = ( + "import sys; " + "sys.path.insert(0, sys.argv[1]); " + "from probhub.mutation_syntax import locate_cpp_mutation_syntax; " + "locations = locate_cpp_mutation_syntax('int f(int x) { return x < 3; }\\n'); " + "assert len(locations.comparisons) == 1" +) + + def _command_probe(command, args=("--version",)): path = shutil.which(command) if not path: @@ -119,6 +129,44 @@ def _version_tuple(text, *, prefix=""): return tuple(map(int, match.groups())) if match else None +def _mutation_parser_probe(): + env = os.environ.copy() + for variable in ("PYTHONHOME", "PYTHONPATH", "PYTHONSTARTUP"): + env.pop(variable, None) + package_root = Path(__file__).resolve().parent.parent + try: + with tempfile.TemporaryDirectory(prefix="probhub-doctor-parser-") as temp: + temp_path = Path(temp) + stdout_path = temp_path / "stdout" + stderr_path = temp_path / "stderr" + result = run_managed_to_files( + [sys.executable, "-c", _MUTATION_PARSER_SMOKE, str(package_root)], + stdout_path=stdout_path, + stderr_path=stderr_path, + timeout=10, + memory_limit_mb=None, + output_limit_bytes=1024 * 1024, + process_limit=8, + cwd=temp_path, + env=env, + ) + stdout = stdout_path.read_text(encoding="utf-8", errors="replace") + stderr = stderr_path.read_text(encoding="utf-8", errors="replace") + except (OSError, ValueError) as exc: + return {"ok": False, "diagnostic": str(exc)} + if result["reason"] != "completed": + diagnostic = ( + "timed out after 10s" + if result["reason"] == "time_limit" + else (result.get("message") or result["reason"]) + ) + return {"ok": False, "diagnostic": diagnostic} + if result["returncode"] != 0: + diagnostic = (stderr or stdout or "parser smoke test failed").strip()[-4000:] + return {"ok": False, "diagnostic": diagnostic} + return {"ok": True, "diagnostic": None} + + def run_doctor(): node = _command_version("node") node_version = _version_tuple(node["version"], prefix="v") @@ -168,12 +216,18 @@ def run_doctor(): } modules = { name: importlib.util.find_spec(name) is not None - for name in ("flask", "yaml", "pypdf") + for name in ("flask", "yaml", "pypdf", "tree_sitter", "tree_sitter_cpp") } modules["pypdf"] = modules["pypdf"] and bool( builder_toolchain["pypdf_version"] ) - distributions = {"flask": "Flask", "yaml": "PyYAML", "pypdf": "pypdf"} + distributions = { + "flask": "Flask", + "yaml": "PyYAML", + "pypdf": "pypdf", + "tree_sitter": "tree-sitter", + "tree_sitter_cpp": "tree-sitter-cpp", + } module_versions = {} for name, distribution in distributions.items(): if not modules[name]: @@ -187,9 +241,24 @@ def run_doctor(): ) except importlib_metadata.PackageNotFoundError: module_versions[name] = "unknown" + modules["tree_sitter"] = ( + modules["tree_sitter"] + and module_versions["tree_sitter"] == TREE_SITTER_VERSION + ) + modules["tree_sitter_cpp"] = ( + modules["tree_sitter_cpp"] + and module_versions["tree_sitter_cpp"] == TREE_SITTER_CPP_VERSION + ) + parser_probe = {"ok": False, "diagnostic": "parser dependencies are unavailable"} + if modules["tree_sitter"] and modules["tree_sitter_cpp"]: + parser_probe = _mutation_parser_probe() + if not parser_probe["ok"]: + modules["tree_sitter"] = False + modules["tree_sitter_cpp"] = False return { "ok": all(item["ok"] for item in tools.values()) and all(modules.values()), "tools": tools, "python_modules": modules, "python_module_versions": module_versions, + "mutation_parser": parser_probe, } diff --git a/probhub/install_deps.py b/probhub/install_deps.py index a168cf7..4492203 100644 --- a/probhub/install_deps.py +++ b/probhub/install_deps.py @@ -44,6 +44,10 @@ def _pip_install_command(requirements, *, user_install): ] if user_install: command.append("--user") + command.extend([ + "--only-binary", + "tree-sitter,tree-sitter-cpp", + ]) command.extend(["-r", str(requirements)]) if os.name != "nt": return command diff --git a/probhub/mutation.py b/probhub/mutation.py index b899504..6ea319a 100644 --- a/probhub/mutation.py +++ b/probhub/mutation.py @@ -36,9 +36,13 @@ load_mutation_exclusions, normalize_mutation_exclusions, ) +from .mutation_syntax import ( + MUTATION_LOCATOR_VERSION, + locate_cpp_mutation_syntax, + mutation_parser_identity, +) from .process_control import run_managed_to_files from .solutions import ( - _mask_cpp_non_code, normalize_solution_entries, resolve_solution_source, ) @@ -60,12 +64,6 @@ "infrastructure-failed", }) _MUTATION_EXCLUSION_STATUSES = frozenset({"matched", "out-of-scope", "unmatched"}) -@dataclass(frozen=True) -class CppToken: - start: int - end: int - text: str - kind: str @dataclass(frozen=True) @@ -92,18 +90,10 @@ def as_dict(self): } -_TOKEN_RE = re.compile( - r"(?:" - r"0[xX][0-9A-Fa-f]+[uUlL]*|" - r"0[bB][01]+[uUlL]*|" - r"0[0-7]*[uUlL]*|" - r"[1-9][0-9]*[uUlL]*|" - r"[A-Za-z_]\w*|" - r"==|!=|<=|>=|&&|\|\||\+\+|--|<<|>>|\+=|-=|\*=|/=|%=|->|::|" - r"[^\s]" - r")" +_INTEGER_RE = re.compile( + r"(?P0|[1-9][0-9]*)" + r"(?P(?:[uU](?:[lL]|ll|LL)?|(?:[lL]|ll|LL)[uU]?))?$" ) -_INTEGER_RE = re.compile(r"(?:0|[1-9][0-9]*)[uUlL]*$") _COMPARISON_REPLACEMENTS = { "<": "<=", "<=": "<", @@ -118,79 +108,25 @@ def mutation_evidence_path(problem_dir): return Path(problem_dir) / ".probhub" / MUTATION_EVIDENCE_FILENAME -def _is_directive(source, token): - offset = token.start() if callable(getattr(token, "start", None)) else token.start - line_start = source.rfind("\n", 0, offset) + 1 - return source[line_start:offset].lstrip().startswith("#") - - -def tokenize_cpp(source): - """Tokenize code while masking comments and literals byte-for-byte. - - The mask comes from the existing source verifier and preserves positions, - including newlines. This deliberately is not a C++ parser: unsupported - constructs simply produce no mutation instead of guessing. - """ - masked = _mask_cpp_non_code(source) - tokens = [] - for match in _TOKEN_RE.finditer(masked): - text = source[match.start():match.end()] - if not text or text.isspace() or _is_directive(source, match): - continue - kind = "identifier" - if _INTEGER_RE.fullmatch(text): - kind = "integer" - elif text in _COMPARISON_REPLACEMENTS or text in {"!", "(", ")", ";"}: - kind = "operator" if text in _COMPARISON_REPLACEMENTS or text == "!" else "punct" - tokens.append(CppToken(match.start(), match.end(), text, kind)) - return tokens - - def _line_column(source, offset): line = source.count("\n", 0, offset) + 1 previous = source.rfind("\n", 0, offset) return line, offset - previous -def _matching_parenthesis(tokens, opening_index): - depth = 0 - for index in range(opening_index, len(tokens)): - text = tokens[index].text - if text == "(": - depth += 1 - elif text == ")": - depth -= 1 - if depth == 0: - return index - if depth < 0: - return None - return None - - -def _condition_is_mutable(tokens, opening_index, closing_index): - inner_depth = 0 - for token in tokens[opening_index + 1:closing_index]: - if token.text in {"(", "[", "{"}: - inner_depth += 1 - elif token.text in {")", "]", "}" - }: - inner_depth = max(0, inner_depth - 1) - elif inner_depth == 0 and token.text in {";", "="}: - # Declarations in if-init statements and malformed fragments are - # intentionally excluded from the first syntax-level slice. - return False - return opening_index + 1 < closing_index - - -def _mutation_candidates(source, operators): - tokens = tokenize_cpp(source) +def _normalized_mutation_text(value): + return value.replace("\r\n", "\n").replace("\r", "\n") + + +def _mutation_candidates(source, operators, *, locations=None): + locations = locations or locate_cpp_mutation_syntax(source) candidates = [] selected = set(operators) if "comparison-boundary" in selected: - for index, token in enumerate(tokens): + for comparison in locations.comparisons: + token = comparison.operator replacement = _COMPARISON_REPLACEMENTS.get(token.text) - previous = tokens[index - 1].text if index else None - if replacement is not None and previous not in {"operator", "template"}: + if replacement is not None: candidates.append(( token, "comparison-boundary", @@ -199,63 +135,55 @@ def _mutation_candidates(source, operators): )) if "integer-boundary" in selected: - for index, token in enumerate(tokens): - if token.kind != "integer": - continue - adjacent_identifier = ( - index + 1 < len(tokens) - and tokens[index + 1].start == token.end - and tokens[index + 1].kind == "identifier" - ) - if adjacent_identifier: - continue - previous = tokens[index - 1].text if index else None - following = tokens[index + 1].text if index + 1 < len(tokens) else None - if previous not in _COMPARISON_REPLACEMENTS and following not in _COMPARISON_REPLACEMENTS: + for comparison in locations.comparisons: + if comparison.operator.text not in _COMPARISON_REPLACEMENTS: continue - for delta in (1, -1): - replacement = f"({token.text} {'+' if delta > 0 else '-'} 1)" - candidates.append(( - token, - "integer-boundary", - replacement, - f"change integer boundary by {delta:+d}", - )) + for token in (comparison.left, comparison.right): + if token.kind != "number_literal": + continue + literal = _INTEGER_RE.fullmatch(token.text) + if literal is None: + continue + for delta in (1, -1): + replacement = f"({token.text} {'+' if delta > 0 else '-'} 1)" + candidates.append(( + token, + "integer-boundary", + replacement, + f"change integer boundary by {delta:+d}", + )) if "boolean-negation" in selected: - for token in tokens: - if token.text == "!": - candidates.append(( - token, - "boolean-negation", - "", - "remove boolean negation", - )) - for index, token in enumerate(tokens[:-1]): - if token.text not in {"if", "while"} or tokens[index + 1].text != "(": - continue - closing_index = _matching_parenthesis(tokens, index + 1) - if closing_index is None or not _condition_is_mutable(tokens, index + 1, closing_index): + for token in locations.unary_not: + candidates.append(( + token, + "boolean-negation", + "", + "remove boolean negation", + )) + for located in locations.conditions: + token = located.condition + if not token.text.startswith("(") or not token.text.endswith(")"): continue - opening = tokens[index + 1] - closing = tokens[closing_index] candidates.append(( - CppToken(opening.start, closing.end, source[opening.start:closing.end], "condition"), + token, "boolean-negation", - f"(!({source[opening.end:closing.start]}))", - f"negate {token.text} condition", + f"(!({token.text[1:-1]}))", + f"negate {located.keyword} condition", )) return candidates -def _planned_mutations(source, operators): - candidates = _mutation_candidates(source, operators) +def _planned_mutations(source, operators, *, locations=None): + candidates = _mutation_candidates(source, operators, locations=locations) candidates.sort(key=lambda item: (item[0].start, item[0].end, item[1], item[2])) mutations = [] for token, operator, replacement, description in candidates: line, column = _line_column(source, token.start) + identity_original = _normalized_mutation_text(token.text) + identity_replacement = _normalized_mutation_text(replacement) identity = hashlib.sha256( - f"{operator}\0{line}\0{column}\0{token.text}\0{replacement}".encode("utf-8") + f"{operator}\0{line}\0{column}\0{identity_original}\0{identity_replacement}".encode("utf-8") ).hexdigest()[:16] mutation_id = f"{MUTATION_OPERATOR_VERSION}:{operator}:{line}:{column}:{identity}" mutations.append(Mutation( @@ -291,11 +219,12 @@ def plan_mutations(source, *, operators=None, max_mutants=MAX_MUTANTS, exclusion code="mutation_limit_invalid", ) exclusion_records = normalize_mutation_exclusions(exclusions) - raw_mutations = _planned_mutations(source, selected) + locations = locate_cpp_mutation_syntax(source) + raw_mutations = _planned_mutations(source, selected, locations=locations) all_mutations = ( raw_mutations if selected == list(MUTATION_OPERATORS) - else _planned_mutations(source, MUTATION_OPERATORS) + else _planned_mutations(source, MUTATION_OPERATORS, locations=locations) ) exclusion_ids = {item["id"] for item in exclusion_records} raw_ids = {item.id for item in raw_mutations} @@ -321,6 +250,7 @@ def plan_mutations(source, *, operators=None, max_mutants=MAX_MUTANTS, exclusion mutation_records = [item.as_dict() for item in mutations] plan_payload = { "operator_version": MUTATION_OPERATOR_VERSION, + "locator_version": MUTATION_LOCATOR_VERSION, "operators": selected, "max_mutants": max_mutants, "source_sha256": hashlib.sha256(source.encode("utf-8")).hexdigest(), @@ -333,6 +263,7 @@ def plan_mutations(source, *, operators=None, max_mutants=MAX_MUTANTS, exclusion return { "schema_version": MUTATION_EVIDENCE_SCHEMA_VERSION, "operator_version": MUTATION_OPERATOR_VERSION, + "locator_version": MUTATION_LOCATOR_VERSION, "operators": selected, "max_mutants": max_mutants, "source_sha256": hashlib.sha256(source.encode("utf-8")).hexdigest(), @@ -375,6 +306,7 @@ def _copy_snapshot(source, target): @lru_cache(maxsize=1) def _compiler_fingerprint(): + parser_identity = mutation_parser_identity() with tempfile.TemporaryDirectory(prefix="probhub-mutation-compiler-") as temp: stdout = Path(temp) / "stdout" stderr = Path(temp) / "stderr" @@ -398,7 +330,12 @@ def _compiler_fingerprint(): "compiler": "g++", "compiler_identity": identity, "compiler_flags": ["-O2", "-std=c++17"], - "parser": "mask-cpp-non-code+token-regex", + "parser": "tree-sitter-cpp", + "locator_version": parser_identity["locator_version"], + "parser_versions": { + "tree_sitter": parser_identity["tree_sitter_version"], + "tree_sitter_cpp": parser_identity["tree_sitter_cpp_version"], + }, "operator_version": MUTATION_OPERATOR_VERSION, } digest = hashlib.sha256(json.dumps(fields, sort_keys=True, separators=(",", ":")).encode()).hexdigest() @@ -627,6 +564,7 @@ def mutation_test_problem( "data_hash": data_hash, "builder_fingerprint": builder_fingerprint, "operator_version": MUTATION_OPERATOR_VERSION, + "locator_version": MUTATION_LOCATOR_VERSION, "operators": plan["operators"], "max_mutants": plan["max_mutants"], "plan_hash": plan["plan_hash"], @@ -675,6 +613,7 @@ def profile(state, *, summary=None, mutants=0, planning=None, exclusions=None, d "summary": dict(summary or {}), "mutants": mutants, "operator_version": MUTATION_OPERATOR_VERSION, + "locator_version": MUTATION_LOCATOR_VERSION, "planning": dict(planning or { "raw": 0, "excluded": 0, @@ -717,6 +656,13 @@ def profile(state, *, summary=None, mutants=0, planning=None, exclusions=None, d or not isinstance(evidence.get("plan_hash"), str) or not re.fullmatch(r"[0-9a-f]{64}", evidence["plan_hash"]) or not isinstance(evidence.get("builder_fingerprint"), dict) + or ( + "locator_version" in evidence + and ( + not isinstance(evidence.get("locator_version"), str) + or len(evidence["locator_version"].encode("utf-8", errors="replace")) > 128 + ) + ) or not isinstance(exclusion_records, list) or len(exclusion_records) > MAX_MUTATION_EXCLUSIONS ): @@ -815,12 +761,18 @@ def profile(state, *, summary=None, mutants=0, planning=None, exclusions=None, d if actual_counts != summary: return profile("invalid", mutants=len(mutations)) fingerprint = evidence["builder_fingerprint"] - fingerprint_required = { + legacy_fingerprint_required = { "schema_version", "probhub_version", "compiler", "compiler_identity", "compiler_flags", "parser", "operator_version", "digest", "available", } + fingerprint_required = legacy_fingerprint_required | { + "locator_version", "parser_versions", + } if ( - set(fingerprint) != fingerprint_required + frozenset(fingerprint) not in { + frozenset(legacy_fingerprint_required), + frozenset(fingerprint_required), + } or not isinstance(fingerprint["schema_version"], int) or isinstance(fingerprint["schema_version"], bool) or fingerprint["schema_version"] != 1 @@ -836,11 +788,19 @@ def profile(state, *, summary=None, mutants=0, planning=None, exclusions=None, d or not re.fullmatch(r"[0-9a-f]{64}", fingerprint["digest"]) ): return profile("invalid", mutants=len(mutations)) + if set(fingerprint) == fingerprint_required and ( + not isinstance(fingerprint["locator_version"], str) + or len(fingerprint["locator_version"].encode("utf-8", errors="replace")) > 128 + or not isinstance(fingerprint["parser_versions"], dict) + or set(fingerprint["parser_versions"]) != {"tree_sitter", "tree_sitter_cpp"} + or not all(isinstance(value, str) for value in fingerprint["parser_versions"].values()) + ): + return profile("invalid", mutants=len(mutations)) # `_compiler_fingerprint` computes the digest over its descriptive fields # before adding the runtime `available` bit. fingerprint_payload = { key: fingerprint[key] - for key in fingerprint_required + for key in set(fingerprint) if key not in {"digest", "available"} } expected_digest = hashlib.sha256(json.dumps( @@ -885,6 +845,7 @@ def profile(state, *, summary=None, mutants=0, planning=None, exclusions=None, d or evidence.get("source_hash") != current_source or evidence.get("data_hash") != current_data or evidence.get("operator_version") != MUTATION_OPERATOR_VERSION + or evidence.get("locator_version") != MUTATION_LOCATOR_VERSION or evidence.get("operators") != plan["operators"] or evidence.get("plan_hash") != plan["plan_hash"] or evidence.get("raw_planned") != plan["raw_planned"] diff --git a/probhub/mutation_syntax.py b/probhub/mutation_syntax.py new file mode 100644 index 0000000..524155f --- /dev/null +++ b/probhub/mutation_syntax.py @@ -0,0 +1,321 @@ +"""Strict C++ syntax locations used by mutation planning.""" + +from __future__ import annotations + +import bisect +from array import array +from dataclasses import dataclass +from functools import lru_cache +from importlib import metadata + +from .errors import ProbHubError + + +MUTATION_LOCATOR_VERSION = "tree-sitter-cpp-v1" +TREE_SITTER_VERSION = "0.26.0" +TREE_SITTER_CPP_VERSION = "0.23.4" +_EXECUTABLE_OWNER_TYPES = frozenset({"function_definition", "lambda_expression"}) +_BLOCKED_CONTEXT_TYPES = frozenset({ + "alignof_expression", + "attribute_declaration", + "attribute_specifier", + "concept_definition", + "decltype", + "noexcept", + "requires_clause", + "requires_expression", + "sizeof_expression", + "static_assert_declaration", + "template_argument_list", + "template_parameter_list", + "typeid_expression", +}) +_BLOCKED_KEYWORD_CALLS = frozenset({"noexcept", "typeid"}) + + +@dataclass(frozen=True) +class CppSyntaxSpan: + start: int + end: int + text: str + kind: str + + +@dataclass(frozen=True) +class CppComparisonLocation: + operator: CppSyntaxSpan + left: CppSyntaxSpan + right: CppSyntaxSpan + + +@dataclass(frozen=True) +class CppConditionLocation: + keyword: str + condition: CppSyntaxSpan + + +@dataclass(frozen=True) +class CppSyntaxLocations: + comparisons: tuple[CppComparisonLocation, ...] + unary_not: tuple[CppSyntaxSpan, ...] + conditions: tuple[CppConditionLocation, ...] + + +class _SourceOffsets: + def __init__(self, source): + self.source = source + self.ascii = source.isascii() + if self.ascii: + self.byte_offsets = None + return + self.byte_offsets = array("Q", [0]) + total = 0 + for character in source: + total += len(character.encode("utf-8")) + self.byte_offsets.append(total) + + def character_offset(self, byte_offset): + if self.ascii: + if not 0 <= byte_offset <= len(self.source): + raise ProbHubError( + "C++ parser returned an out-of-range source location", + code="mutation_location_invalid", + ) + return byte_offset + index = bisect.bisect_left(self.byte_offsets, byte_offset) + if index >= len(self.byte_offsets) or self.byte_offsets[index] != byte_offset: + raise ProbHubError( + "C++ parser returned a location inside a UTF-8 character", + code="mutation_location_invalid", + ) + return index + + def span(self, node): + start = self.character_offset(node.start_byte) + end = self.character_offset(node.end_byte) + if not 0 <= start <= end <= len(self.source): + raise ProbHubError( + "C++ parser returned an out-of-range source location", + code="mutation_location_invalid", + ) + return CppSyntaxSpan(start, end, self.source[start:end], node.type) + + +def mutation_parser_identity(): + try: + tree_sitter_version = metadata.version("tree-sitter") + tree_sitter_cpp_version = metadata.version("tree-sitter-cpp") + except metadata.PackageNotFoundError as exc: + raise ProbHubError( + "mutation syntax planning requires tree-sitter and tree-sitter-cpp", + code="mutation_parser_unavailable", + ) from exc + if ( + tree_sitter_version != TREE_SITTER_VERSION + or tree_sitter_cpp_version != TREE_SITTER_CPP_VERSION + ): + raise ProbHubError( + "mutation syntax parser versions do not match the pinned runtime " + f"(tree-sitter {TREE_SITTER_VERSION}, tree-sitter-cpp {TREE_SITTER_CPP_VERSION})", + code="mutation_parser_unavailable", + ) + return { + "locator_version": MUTATION_LOCATOR_VERSION, + "tree_sitter_version": tree_sitter_version, + "tree_sitter_cpp_version": tree_sitter_cpp_version, + } + + +@lru_cache(maxsize=1) +def _cpp_language(): + mutation_parser_identity() + try: + from tree_sitter import Language + import tree_sitter_cpp + + return Language(tree_sitter_cpp.language()) + except (ImportError, TypeError, ValueError, RuntimeError) as exc: + raise ProbHubError( + f"cannot initialize the C++ mutation parser: {exc}", + code="mutation_parser_unavailable", + ) from exc + + +def _parse_cpp(source_bytes): + try: + from tree_sitter import Parser + + tree = Parser(_cpp_language()).parse(source_bytes) + except ProbHubError: + raise + except (ImportError, TypeError, ValueError, RuntimeError) as exc: + raise ProbHubError( + f"C++ mutation parser failed: {exc}", + code="mutation_parser_failed", + ) from exc + if tree is None or tree.root_node is None: + raise ProbHubError( + "C++ mutation parser returned no syntax tree", + code="mutation_parser_failed", + ) + if tree.root_node.has_error: + error = _first_syntax_error(tree.root_node) + point = error.start_point if error is not None else tree.root_node.start_point + raise ProbHubError( + "accepted C++ source contains syntax the mutation parser cannot locate safely " + f"at line {point.row + 1}, byte column {point.column + 1}", + code="mutation_syntax_invalid", + ) + return tree + + +def _first_syntax_error(node): + stack = [node] + while stack: + current = stack.pop() + if current.type == "ERROR" or current.is_missing: + return current + stack.extend(reversed(current.children)) + return None + + +def _iter_nodes(root): + stack = [(root, ())] + while stack: + node, ancestors = stack.pop() + if node.type.startswith("preproc_"): + continue + yield node, ancestors + child_ancestors = (*ancestors, node) + stack.extend((child, child_ancestors) for child in reversed(node.children)) + + +def _inside_executable_body(ancestors): + owner_index = -1 + for index, ancestor in enumerate(ancestors): + if ancestor.type in _EXECUTABLE_OWNER_TYPES: + owner_index = index + return owner_index >= 0 and any( + ancestor.type == "compound_statement" + for ancestor in ancestors[owner_index + 1:] + ) + + +def _inside_blocked_context(ancestors, offsets): + for ancestor in ancestors: + if ancestor.type != "call_expression": + continue + function = ancestor.child_by_field_name("function") + if function is not None and offsets.span(function).text in _BLOCKED_KEYWORD_CALLS: + return True + return any( + ancestor.type in _BLOCKED_CONTEXT_TYPES + or ancestor.type.startswith("preproc_") + for ancestor in ancestors + ) + + +def _inside_constexpr_condition(node, ancestors): + for ancestor in reversed(ancestors): + if ancestor.type != "if_statement": + continue + if not any(child.type in {"constexpr", "consteval"} for child in ancestor.children): + continue + condition = ancestor.child_by_field_name("condition") + if ( + condition is not None + and condition.start_byte <= node.start_byte + and node.end_byte <= condition.end_byte + ): + return True + return False + + +def _inside_case_label(node, ancestors): + for ancestor in reversed(ancestors): + if ancestor.type != "case_statement": + continue + value = ancestor.child_by_field_name("value") + return ( + value is not None + and value.start_byte <= node.start_byte + and node.end_byte <= value.end_byte + ) + return False + + +def _eligible_expression(node, ancestors, offsets): + return ( + _inside_executable_body(ancestors) + and not _inside_blocked_context(ancestors, offsets) + and not _inside_constexpr_condition(node, ancestors) + and not _inside_case_label(node, ancestors) + ) + + +def locate_cpp_mutation_syntax(source): + """Return conservative AST-backed locations for the existing operators.""" + if not isinstance(source, str): + raise ProbHubError("C++ source must be text", code="mutation_source_invalid") + try: + source_bytes = source.encode("utf-8") + except UnicodeEncodeError as exc: + raise ProbHubError( + "C++ source is not valid UTF-8 text", + code="mutation_source_invalid", + ) from exc + + tree = _parse_cpp(source_bytes) + offsets = _SourceOffsets(source) + comparisons = [] + unary_not = [] + conditions = [] + + for node, ancestors in _iter_nodes(tree.root_node): + if node.type == "binary_expression" and _eligible_expression(node, ancestors, offsets): + operator = node.child_by_field_name("operator") + left = node.child_by_field_name("left") + right = node.child_by_field_name("right") + if operator is not None and left is not None and right is not None: + comparisons.append(CppComparisonLocation( + offsets.span(operator), + offsets.span(left), + offsets.span(right), + )) + elif node.type == "unary_expression" and _eligible_expression(node, ancestors, offsets): + operator = node.child_by_field_name("operator") + if operator is not None: + operator_span = offsets.span(operator) + if operator_span.text == "!": + unary_not.append(operator_span) + elif node.type in {"if_statement", "while_statement", "do_statement"}: + if ( + not _inside_executable_body(ancestors) + or _inside_blocked_context(ancestors, offsets) + ): + continue + if node.type == "if_statement" and any( + child.type in {"constexpr", "consteval"} for child in node.children + ): + continue + condition = node.child_by_field_name("condition") + if condition is None or condition.type not in { + "condition_clause", "parenthesized_expression" + }: + continue + if condition.child_by_field_name("initializer") is not None: + continue + value = condition.child_by_field_name("value") + if ( + condition.type == "condition_clause" + and (value is None or value.type in {"declaration", "condition_declaration"}) + ): + continue + keyword = "if" if node.type == "if_statement" else "while" + conditions.append(CppConditionLocation(keyword, offsets.span(condition))) + + return CppSyntaxLocations( + tuple(comparisons), + tuple(unary_not), + tuple(conditions), + ) diff --git a/references/cli.md b/references/cli.md index 99c2ea7..a3da2a4 100644 --- a/references/cli.md +++ b/references/cli.md @@ -404,7 +404,7 @@ probhub mutation ID [--operator OPERATOR] [--max-mutants N] [--timeout SECONDS] probhub mutate ID [--operator OPERATOR] [--max-mutants N] [--timeout SECONDS] [--no-cache] ``` -`mutation` 只对 `judge.type: standard` 且首个 accepted 为 C++ 源码的题目执行。它在临时快照中生成比较边界、布尔条件和十进制整数边界变异，复用正式 Validator/Judge 逐点运行；不会把变异体登记为 accepted/wrong，也不会改写题目源文件或正式产物。`--operator` 可重复指定，`--max-mutants` 范围为 1..256，`--timeout` 是每题总秒数。题目可用 `mutation.schema_version: 1` 和 `mutation.exclusions: [{id, reason}]` 记录经人工审查的精确排除；排除先于数量上限应用。 +`mutation` 只对 `judge.type: standard` 且首个 accepted 为 C++ 源码的题目执行。它使用固定 Tree-sitter C++ 语法树在函数/lambda 复合语句体内生成比较边界、布尔条件和十进制整数边界变异，跳过模板、宏、运算符声明、`<=>`、`case` 标签和未求值上下文，再在临时快照中复用正式 Validator/Judge 逐点运行；解析器能报告的失败会结构化终止，不回退到 Token 猜测。命令不会把变异体登记为 accepted/wrong，也不会改写题目源文件或正式产物。`--operator` 可重复指定，`--max-mutants` 范围为 1..256，`--timeout` 是每题总秒数。题目可用 `mutation.schema_version: 1` 和 `mutation.exclusions: [{id, reason}]` 记录经人工审查的精确排除；排除先于数量上限应用。 JSON 结果保持与其他多题命令相同的外层结构： @@ -429,7 +429,7 @@ JSON 结果保持与其他多题命令相同的外层结构： `raw_planned` 是本次所选算子从源码生成的原始候选数，`excluded` 是命中本次计划的人工排除数，`planned` 是过滤后的有效候选数，`selected` 是数量上限后实际选择数。evidence 还保存每个排除 ID、理由和 `matched` / `out-of-scope` / `unmatched` 三态：仍存在但不属于本次所选算子的 ID 为 `out-of-scope`，只有已不在完整当前计划中的失效 ID 才产生 `mutation_exclusion_unmatched` warning。 -`status: passed` 只表示所有选择的变异都完成执行；summary 中的 `survived` 必须人工分析。`compile-invalid` 不计入可执行变异分母，`infrastructure-failed` 表示题目基础设施或沙箱失败，不能当成击杀。每个变异最多保留 16 个命中详情，完整数量由 `hit_cases_total` 和 `hit_cases_truncated` 表示；单份 evidence 上限为 4 MiB。成功才原子发布 `/.probhub/mutation-evidence-v2.json`；旧 v1 evidence 不再读取。失败、取消、超时、证据超限、输入变化、锁竞争或发布故障保留旧 evidence。报告会验证 source/data hash、当前 Core/编译器指纹、算子计划 hash、排除记录、计数和有界记录结构；变化后显示 `stale` 或 `invalid`。stale 时 report 会重算当前计划与排除三态，并清空旧 evidence 的执行分类，避免把两代统计并列解释。 +`status: passed` 只表示所有选择的变异都完成执行；summary 中的 `survived` 必须人工分析。`compile-invalid` 不计入可执行变异分母，`infrastructure-failed` 表示题目基础设施或沙箱失败，不能当成击杀。每个变异最多保留 16 个命中详情，完整数量由 `hit_cases_total` 和 `hit_cases_truncated` 表示；单份 evidence 上限为 4 MiB。成功才原子发布 `/.probhub/mutation-evidence-v2.json`；旧 v1 evidence 不再读取。失败、取消、超时、解析错误、证据超限、输入变化、锁竞争或发布故障保留旧 evidence。报告会验证 source/data hash、当前 Core/编译器/解析器指纹、locator 与算子计划 hash、排除记录、计数和有界记录结构；变化后显示 `stale` 或 `invalid`。stale 时 report 会重算当前计划与排除三态，并清空旧 evidence 的执行分类，避免把两代统计并列解释。 完整限制、算子语义和解释边界见 [std 变异测试](mutation-testing.md)。 diff --git a/references/mutation-testing.md b/references/mutation-testing.md index 79f2654..42128ce 100644 --- a/references/mutation-testing.md +++ b/references/mutation-testing.md @@ -12,7 +12,9 @@ | `boolean-negation` | 删除 `!` 或取反 `if`/`while` 条件 | 检查布尔分支覆盖 | | `integer-boundary` | 只对十进制比较边界的常量尝试 `+1`/`-1` | 检查常量边界附近数据 | -源码先遮罩注释、字符串和字符字面量，再扫描 Token；不支持的 C++ 语法不会被猜测式替换。每个变异有稳定的 `cpp-token-v1` ID，计划由源代码、算子列表、人工排除记录和上限共同决定。 +源码使用固定的 `tree-sitter==0.26.0` 与 `tree-sitter-cpp==0.23.4` 构造 C++ 语法树，只定位函数或 lambda 复合语句体内的真实表达式。模板尖括号、运算符声明、`<=>`、预处理宏、concept/requires、`case` 标签、`static_assert`、`sizeof` / `decltype` / `noexcept` 等非执行或未求值上下文不会生成候选；注释、字符串和字符字面量也不进入语法候选。解析器能报告的失败会以结构化错误终止，不回退到 Token 猜测；当前 `tree-sitter-cpp` 对 `typeid(type)` 等少数合法语法的支持不完整，此时命令保守失败而不猜测位置。 + +每个变异继续使用稳定的 `cpp-token-v1` ID，计划由源码、`tree-sitter-cpp-v1` locator、算子列表、人工排除记录和上限共同决定。仍然有效的旧 ID 保持不变；旧 Token 扫描器产生但语法树不再接受的误报 ID 会成为 `unmatched`，需要作者复核后移除。解析器切换会使旧 evidence 显示 `stale`，不会把旧执行分类与新候选计划混用。 ## 使用 @@ -62,6 +64,6 @@ mutation: /.probhub/mutation-evidence-v2.json ``` -证据包含 source/data hash、accepted 源路径、算子版本、计划 hash、当前 Core/编译器指纹、raw/excluded/effective/selected 计数、带三态匹配状态和理由的排除记录、变异 ID、击杀用例 ID、分类计数和有界诊断。每个变异最多保留前 16 个命中详情，并用 `hit_cases_total` / `hit_cases_truncated` 说明完整数量；单份 evidence 最多 4 MiB。失败、取消、超时、证据超限、输入变化、锁竞争或发布故障不会覆盖上一份成功 evidence。evidence 过期时 report 重算当前计划与排除三态，但不继续展示旧执行分类。旧 `mutation-evidence-v1.json` 继续被 Git 忽略，但不作为当前证据读取。证据文件只用于本地 `report`，不进入 PDF、ZIP、Manifest，也不应提交到 Git。 +证据包含 source/data hash、accepted 源路径、算子与 locator 版本、计划 hash、当前 Core/编译器/解析器指纹、raw/excluded/effective/selected 计数、带三态匹配状态和理由的排除记录、变异 ID、击杀用例 ID、分类计数和有界诊断。每个变异最多保留前 16 个命中详情，并用 `hit_cases_total` / `hit_cases_truncated` 说明完整数量；单份 evidence 最多 4 MiB。失败、取消、超时、解析错误、证据超限、输入变化、锁竞争或发布故障不会覆盖上一份成功 evidence。evidence 过期时 report 重算当前计划与排除三态，但不继续展示旧执行分类。旧 `mutation-evidence-v1.json` 继续被 Git 忽略，但不作为当前证据读取。证据文件只用于本地 `report`，不进入 PDF、ZIP、Manifest，也不应提交到 Git。 -变异体和其临时编译产物始终位于临时快照；命令不会写回题目 `code/`、`probhub.yaml`、`data/` 或任何正式构建产物。mutation score 不作为 `build` 的硬门禁。看到幸存变异时，应阅读命中范围、补充边界/定向数据，再重新运行并比较同一 mutation ID；不要仅凭 score 宣称题目已被证明。 +变异体和其临时编译产物始终位于临时快照；命令不会写回题目 `code/`、`probhub.yaml`、`data/` 或任何正式构建产物。语法定位收紧后 raw/selected/compile-invalid 数量与 score 可能相对旧版本变化，这表示候选集合变化，不代表数据自动变强。mutation score 不作为 `build` 的硬门禁。看到幸存变异时，应阅读命中范围、补充边界/定向数据，再重新运行并比较同一 mutation ID；不要仅凭 score 宣称题目已被证明。 diff --git a/references/workspace-schema-v1.md b/references/workspace-schema-v1.md index 526935a..039961e 100644 --- a/references/workspace-schema-v1.md +++ b/references/workspace-schema-v1.md @@ -251,7 +251,7 @@ mutation: reason: 该分支在 Validator 保证的 n >= 1 下与原程序等价 ``` -`mutation` 只适用于 `judge.type: standard`。`exclusions` 最多 256 项；每项只能包含当前 `cpp-token-v1` 的稳定 `id` 和不超过 1024 字节的非空 `reason`，ID 不得重复，也不支持通配符。配置字段、版本、ID、重复项、数量和理由错误会由 lint 以稳定诊断阻断。 +`mutation` 只适用于 `judge.type: standard`。`exclusions` 最多 256 项；每项只能包含当前 `cpp-token-v1` 的稳定 `id` 和不超过 1024 字节的非空 `reason`，ID 不得重复，也不支持通配符。ID 前缀保持兼容，但候选由固定 Tree-sitter C++ 语法树定位；仍有效的旧 ID 保持匹配，旧 Token 误报会显示 `unmatched`，不会自动迁移或删除。配置字段、版本、ID、重复项、数量和理由错误会由 lint 以稳定诊断阻断。 先在未排除状态运行 mutation 并审查源码、变异位置和执行结果，再登记排除。排除在 `--max-mutants` 限额之前应用，因此不会占用有效变异的执行配额。仅运行部分算子时，属于其他算子的有效 ID 标为 `out-of-scope`；源码变化后已不在完整计划中的旧 ID 不会被静默删除，而会标为 `unmatched` warning，供作者更新或移除。排除理由参与 source hash 和计划 hash；修改配置会使旧 evidence 过期。 diff --git a/requirements.txt b/requirements.txt index e5a70a1..04ef7bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,7 @@ Jinja2==3.1.6 MarkupSafe==3.0.3 PyYAML==6.0.3 pypdf==6.15.0 +tree-sitter==0.26.0 +tree-sitter-cpp==0.23.4 typing_extensions==4.16.0; python_version < "3.11" Werkzeug==3.1.8 diff --git a/scripts/check_clean_install.py b/scripts/check_clean_install.py index 0dde246..bcff29b 100644 --- a/scripts/check_clean_install.py +++ b/scripts/check_clean_install.py @@ -275,7 +275,7 @@ def run_clean_install(): "-c", "import importlib.util, json; " "print(json.dumps({name: importlib.util.find_spec(name) is None " - "for name in ('flask', 'yaml', 'pypdf')}))", + "for name in ('flask', 'yaml', 'pypdf', 'tree_sitter', 'tree_sitter_cpp')}))", ], cwd=root, ) @@ -337,8 +337,13 @@ def run_clean_install(): required_tools = {"python", "g++", "typst", "node", "npm"} if required_tools - set(doctor.get("tools", {})): raise CleanInstallError(f"doctor omitted required tools: {doctor!r}") - if not all(doctor.get("python_modules", {}).get(name) for name in ("flask", "yaml", "pypdf")): + if not all( + doctor.get("python_modules", {}).get(name) + for name in ("flask", "yaml", "pypdf", "tree_sitter", "tree_sitter_cpp") + ): raise CleanInstallError(f"doctor omitted required Python modules: {doctor!r}") + if not doctor.get("mutation_parser", {}).get("ok"): + raise CleanInstallError(f"doctor mutation parser smoke failed: {doctor!r}") initialized = _run_json( [probhub, "--json", "init", workspace, "--title", "Clean Install", "--subtitle", "正式赛", "--author", "ProbHub CI"], diff --git a/scripts/check_release.py b/scripts/check_release.py index b0fa103..551f8c6 100644 --- a/scripts/check_release.py +++ b/scripts/check_release.py @@ -192,6 +192,7 @@ def validate_pack_inventories(*, dry_run=True, destination=None): "probhub/judge_qa.py", "probhub/judge_qa_evidence.py", "probhub/judge_qa_runtime.py", "probhub/mutation.py", "probhub/mutation_config.py", + "probhub/mutation_syntax.py", "probhub/webui_runtime.py", "probhub/assets/fonts/NotoSansCJKsc-Regular.otf", "probhub/assets/fonts/OFL.txt", diff --git a/tests/fixtures/mutation-plans/cpp-syntax-v1.cpp b/tests/fixtures/mutation-plans/cpp-syntax-v1.cpp new file mode 100644 index 0000000..1354772 --- /dev/null +++ b/tests/fixtures/mutation-plans/cpp-syntax-v1.cpp @@ -0,0 +1,41 @@ +#include +#include +#include + +#define LESS_THAN(a, b) ((a) < (b)) + +template +concept Small = requires(T value) { + { value < 3 }; +}; + +template +struct Box { + T value; + bool operator<(const Box& other) const { return value < other.value; } + bool operator!() const { return value == 0; } + auto operator<=>(const Box& other) const { return value <=> other.value; } +}; + +static_assert(sizeof(int) <= 8); + +// UTF-8 before executable candidates: 中文 +int solve(int x) { + const char* text = "x < 3 && value == 0"; + std::vector values; + std::array fixed{}; + if constexpr (sizeof(int) < 8) { + x = x < 10 ? x : 10; + } + static_assert(noexcept(x < 3)); + switch (x) { + case (1 < 2): x = x < 12 ? x : 12; break; + default: break; + } + if (int y = x; y < 3) x = y; + if (x < + 4) return x <= 2; + do { --x; } while (!(x > 0)); + while (x != 1) --x; + return 3 == x || LESS_THAN(x, 5); +} diff --git a/tests/fixtures/mutation-plans/cpp-syntax-v1.json b/tests/fixtures/mutation-plans/cpp-syntax-v1.json new file mode 100644 index 0000000..65e5c59 --- /dev/null +++ b/tests/fixtures/mutation-plans/cpp-syntax-v1.json @@ -0,0 +1,38 @@ +{ + "locator_version": "tree-sitter-cpp-v1", + "plan_hash": "bf972b4734eb914e6aeb6e9eed98f135fb100c486defac71f0a27975681ac53b", + "mutation_ids": [ + "cpp-token-v1:comparison-boundary:15:59:7786a79c21b98f67", + "cpp-token-v1:comparison-boundary:16:43:85ca832647af5bd0", + "cpp-token-v1:integer-boundary:16:46:a8fb764bbe512102", + "cpp-token-v1:integer-boundary:16:46:5c45f3a99600106a", + "cpp-token-v1:comparison-boundary:28:15:bb8d8ed160a2fbb2", + "cpp-token-v1:integer-boundary:28:17:4fee3f0ebdf9b896", + "cpp-token-v1:integer-boundary:28:17:7b3c08fa14e4cced", + "cpp-token-v1:comparison-boundary:32:29:03c1821dd9e5cc0a", + "cpp-token-v1:integer-boundary:32:31:ac4855a234367f0f", + "cpp-token-v1:integer-boundary:32:31:898018094fc4f66a", + "cpp-token-v1:comparison-boundary:35:22:5a0751c8d2448838", + "cpp-token-v1:integer-boundary:35:24:803a8fd44360fee6", + "cpp-token-v1:integer-boundary:35:24:f2652ecb50080374", + "cpp-token-v1:boolean-negation:36:8:36751a0a8e97397a", + "cpp-token-v1:comparison-boundary:36:11:3c1192ae8f245d53", + "cpp-token-v1:integer-boundary:37:9:d6c17e9133f7fd30", + "cpp-token-v1:integer-boundary:37:9:b665f79e4bc4663a", + "cpp-token-v1:comparison-boundary:37:21:b49ab3a9237721eb", + "cpp-token-v1:integer-boundary:37:24:35e73a74278be13c", + "cpp-token-v1:integer-boundary:37:24:42197f13cfc8eb15", + "cpp-token-v1:boolean-negation:38:23:98ca44196c100153", + "cpp-token-v1:boolean-negation:38:24:014c251e99cbf06d", + "cpp-token-v1:comparison-boundary:38:28:9349b1b27abb1fb7", + "cpp-token-v1:integer-boundary:38:30:c380111a1e9ae97f", + "cpp-token-v1:integer-boundary:38:30:576cb8a152ea01dc", + "cpp-token-v1:boolean-negation:39:11:f5edfac45906c6a9", + "cpp-token-v1:comparison-boundary:39:14:b6679012132afd50", + "cpp-token-v1:integer-boundary:39:17:37b69659770991f8", + "cpp-token-v1:integer-boundary:39:17:afd2921a24df9672", + "cpp-token-v1:integer-boundary:40:12:0a7f5802319611a5", + "cpp-token-v1:integer-boundary:40:12:4a0b79e427b9e143", + "cpp-token-v1:comparison-boundary:40:14:ec36ba2d2a14ec33" + ] +} diff --git a/tests/test_core.py b/tests/test_core.py index da79ed2..ab2e578 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -719,6 +719,37 @@ def test_doctor_tool_probe_does_not_apply_virtual_memory_limit(self): self.assertEqual(run_managed.call_args.kwargs["output_limit_bytes"], 1024 * 1024) self.assertEqual(run_managed.call_args.kwargs["process_limit"], 8) + def test_doctor_parser_probe_is_bounded_and_reports_native_failure(self): + from probhub.doctor import _mutation_parser_probe + + def failed_parser(command, **kwargs): + Path(kwargs["stdout_path"]).write_text("", encoding="utf-8") + Path(kwargs["stderr_path"]).write_text("native parser failed", encoding="utf-8") + return { + "reason": "completed", + "message": None, + "returncode": 3, + } + + with ( + patch.dict("os.environ", {"PYTHONPATH": "shadow", "PYTHONHOME": "bad"}), + patch( + "probhub.doctor.run_managed_to_files", + side_effect=failed_parser, + ) as run_managed, + ): + probe = _mutation_parser_probe() + + self.assertFalse(probe["ok"]) + self.assertIn("native parser failed", probe["diagnostic"]) + kwargs = run_managed.call_args.kwargs + self.assertEqual(kwargs["timeout"], 10) + self.assertIsNone(kwargs["memory_limit_mb"]) + self.assertEqual(kwargs["output_limit_bytes"], 1024 * 1024) + self.assertEqual(kwargs["process_limit"], 8) + self.assertNotIn("PYTHONPATH", kwargs["env"]) + self.assertNotIn("PYTHONHOME", kwargs["env"]) + def test_doctor_rejects_old_node_and_missing_pinned_cjk_font(self): from probhub.doctor import run_doctor diff --git a/tests/test_dependency_audit.py b/tests/test_dependency_audit.py index f24863c..294a524 100644 --- a/tests/test_dependency_audit.py +++ b/tests/test_dependency_audit.py @@ -104,6 +104,8 @@ def test_requirements_pin_the_complete_runtime_closure(self): "markupsafe", "pypdf", "pyyaml", + "tree-sitter", + "tree-sitter-cpp", "typing_extensions", "werkzeug", }, diff --git a/tests/test_mutation.py b/tests/test_mutation.py index 7d3d508..e83a9d1 100644 --- a/tests/test_mutation.py +++ b/tests/test_mutation.py @@ -614,6 +614,71 @@ def test_returned_fingerprint_cannot_mutate_the_process_cache(self): profile = mutation_evidence_profile(problem, read_yaml(problem / "probhub.yaml")) self.assertEqual(profile["state"], "current", profile) + def test_legacy_token_fingerprint_is_stale_instead_of_invalid(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "workspace" + shutil.copytree(FIXTURE.parent, root) + problem = root / "M01" + passed = { + "ok": True, + "final": {"code": "all_expectations_met"}, + "events": [], + } + with patch("probhub.mutation.judge_problem", return_value=passed): + result = mutation_test_problem( + root, + problem, + operators=["comparison-boundary"], + max_mutants=1, + ) + self.assertTrue(result["ok"], result) + + evidence_path = mutation_evidence_path(problem) + evidence = json.loads(evidence_path.read_text(encoding="utf-8")) + evidence.pop("locator_version") + fingerprint = evidence["builder_fingerprint"] + fingerprint.pop("locator_version") + fingerprint.pop("parser_versions") + fingerprint["parser"] = "mask-cpp-non-code+token-regex" + fingerprint_payload = { + key: value + for key, value in fingerprint.items() + if key not in {"available", "digest"} + } + fingerprint["digest"] = hashlib.sha256(json.dumps( + fingerprint_payload, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8")).hexdigest() + evidence_path.write_text( + json.dumps(evidence, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + profile = mutation_evidence_profile(problem, read_yaml(problem / "probhub.yaml")) + self.assertEqual(profile["state"], "stale", profile) + self.assertEqual(profile["summary"], {}) + + def test_syntax_failure_keeps_previous_evidence(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "workspace" + shutil.copytree(FIXTURE.parent, root) + problem = root / "M01" + evidence = mutation_evidence_path(problem) + evidence.parent.mkdir(parents=True, exist_ok=True) + previous = b"previous evidence\n" + evidence.write_bytes(previous) + (problem / "code/std.cpp").write_text( + "int main( { return value < 3; }\n", + encoding="utf-8", + ) + + with self.assertRaises(ProbHubError) as raised: + mutation_test_problem(root, problem) + + self.assertEqual(raised.exception.code, "mutation_syntax_invalid") + self.assertEqual(evidence.read_bytes(), previous) + def test_evidence_publish_failure_keeps_previous_bytes_and_cleans_temporary(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) / "workspace" diff --git a/tests/test_mutation_syntax.py b/tests/test_mutation_syntax.py new file mode 100644 index 0000000..3184cce --- /dev/null +++ b/tests/test_mutation_syntax.py @@ -0,0 +1,168 @@ +import json +import tempfile +import unittest +from importlib import metadata +from pathlib import Path +from unittest.mock import patch + +from probhub.errors import ProbHubError +from probhub.mutation import MUTATION_OPERATOR_VERSION, plan_mutations +from probhub.mutation_syntax import ( + MUTATION_LOCATOR_VERSION, + TREE_SITTER_CPP_VERSION, + TREE_SITTER_VERSION, + _cpp_language, + mutation_parser_identity, +) + + +FIXTURE_ROOT = Path(__file__).parent / "fixtures/mutation-plans" +SOURCE_FIXTURE = FIXTURE_ROOT / "cpp-syntax-v1.cpp" +PLAN_FIXTURE = FIXTURE_ROOT / "cpp-syntax-v1.json" + + +def mutation_records(plan): + return [item.as_dict() for item in plan["mutations"]] + + +class MutationSyntaxTests(unittest.TestCase): + def test_pinned_parser_identity_is_explicit(self): + identity = mutation_parser_identity() + self.assertEqual(identity["locator_version"], MUTATION_LOCATOR_VERSION) + self.assertEqual(identity["tree_sitter_version"], TREE_SITTER_VERSION) + self.assertEqual(identity["tree_sitter_cpp_version"], TREE_SITTER_CPP_VERSION) + + def test_canonical_plan_is_stable_across_lf_crlf_and_utf8_path(self): + source = SOURCE_FIXTURE.read_text(encoding="utf-8") + expected = json.loads(PLAN_FIXTURE.read_text(encoding="utf-8")) + lf = plan_mutations(source, max_mutants=256) + crlf = plan_mutations(source.replace("\n", "\r\n"), max_mutants=256) + + self.assertEqual(lf["locator_version"], MUTATION_LOCATOR_VERSION) + self.assertEqual(expected["locator_version"], MUTATION_LOCATOR_VERSION) + self.assertEqual( + [item.id for item in lf["mutations"]], + expected["mutation_ids"], + ) + self.assertEqual(lf["plan_hash"], expected["plan_hash"]) + self.assertEqual( + [item.id for item in crlf["mutations"]], + [item.id for item in lf["mutations"]], + ) + self.assertEqual( + [ + { + **item, + "original": item["original"].replace("\r\n", "\n"), + "replacement": item["replacement"].replace("\r\n", "\n"), + } + for item in mutation_records(crlf) + ], + mutation_records(lf), + ) + self.assertTrue(all( + item.id.startswith(MUTATION_OPERATOR_VERSION + ":") + for item in lf["mutations"] + )) + self.assertNotEqual(lf["source_sha256"], crlf["source_sha256"]) + for mutation in lf["mutations"]: + self.assertEqual(source[mutation.start:mutation.end], mutation.original) + line_start = source.rfind("\n", 0, mutation.start) + self.assertEqual(mutation.line, source.count("\n", 0, mutation.start) + 1) + self.assertEqual(mutation.column, mutation.start - line_start) + + with tempfile.TemporaryDirectory() as temp: + utf8_dir = Path(temp) / "题目" + utf8_dir.mkdir() + copied = utf8_dir / SOURCE_FIXTURE.name + copied.write_text(source, encoding="utf-8", newline="") + from_utf8_path = plan_mutations(copied.read_text(encoding="utf-8")) + self.assertEqual(mutation_records(from_utf8_path), mutation_records(lf)) + + def test_ast_plan_excludes_non_runtime_syntax(self): + source = SOURCE_FIXTURE.read_text(encoding="utf-8") + records = mutation_records(plan_mutations(source)) + lines = {item["line"] for item in records} + + self.assertNotIn(5, lines) # macro body + self.assertNotIn(7, lines) # template parameter default + self.assertNotIn(9, lines) # requires expression + self.assertNotIn(17, lines) # spaceship expression + self.assertNotIn(20, lines) # static_assert + self.assertNotIn(26, lines) # template argument expression + self.assertNotIn(27, lines) # if constexpr condition + self.assertIn(15, lines) # operator< function body + self.assertIn(16, lines) # operator! function body comparison + self.assertIn(28, lines) # executable if constexpr body + self.assertNotIn(30, lines) # noexcept operand + self.assertFalse(any( + item["line"] == 32 and item["column"] < 29 + for item in records + )) # case label + self.assertIn(32, lines) # executable case body + self.assertIn(36, lines) # ordinary multi-line if condition + + def test_noexcept_and_case_labels_do_not_produce_candidates(self): + source = ( + "int f(int x) {\n" + " static_assert(noexcept(x < 3));\n" + " switch (x) { case (1 < 2): return x < 4; default: return 0; }\n" + "}\n" + ) + records = mutation_records(plan_mutations(source)) + self.assertEqual( + [(item["line"], item["column"], item["operator"], item["original"]) for item in records], + [ + (3, 39, "comparison-boundary", "<"), + (3, 41, "integer-boundary", "4"), + (3, 41, "integer-boundary", "4"), + ], + ) + + def test_old_id_is_preserved_for_unchanged_runtime_expression(self): + source = "int f(int x) { return x <= 3 ? 1 : 0; }\n" + ids = [item.id for item in plan_mutations(source)["mutations"]] + self.assertEqual(ids, [ + "cpp-token-v1:comparison-boundary:1:25:e42035632288c6a1", + "cpp-token-v1:integer-boundary:1:28:0e315b4334f3ff07", + "cpp-token-v1:integer-boundary:1:28:5612be40c1831dac", + ]) + + def test_invalid_syntax_fails_closed_without_token_fallback(self): + with self.assertRaises(ProbHubError) as raised: + plan_mutations("int f( { return x < 3; }\n") + self.assertEqual(raised.exception.code, "mutation_syntax_invalid") + self.assertIn("line", str(raised.exception)) + + def test_unsupported_typeid_type_form_fails_closed(self): + with self.assertRaises(ProbHubError) as raised: + plan_mutations( + "#include \n" + "int f(int x) { return typeid(x < 3) == typeid(bool); }\n" + ) + self.assertEqual(raised.exception.code, "mutation_syntax_invalid") + + def test_missing_or_mismatched_parser_is_structured(self): + _cpp_language.cache_clear() + try: + with patch( + "probhub.mutation_syntax.metadata.version", + side_effect=metadata.PackageNotFoundError("tree-sitter"), + ): + with self.assertRaises(ProbHubError) as missing: + plan_mutations("int f() { return 1 < 2; }\n") + self.assertEqual(missing.exception.code, "mutation_parser_unavailable") + + with patch( + "probhub.mutation_syntax.metadata.version", + side_effect=["0.0.0", TREE_SITTER_CPP_VERSION], + ): + with self.assertRaises(ProbHubError) as mismatched: + plan_mutations("int f() { return 1 < 2; }\n") + self.assertEqual(mismatched.exception.code, "mutation_parser_unavailable") + finally: + _cpp_language.cache_clear() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_npm_packages.py b/tests/test_npm_packages.py index 901afc7..75d0429 100644 --- a/tests/test_npm_packages.py +++ b/tests/test_npm_packages.py @@ -53,6 +53,7 @@ def test_judge_qa_runtime_and_contract_are_published(self): self.assertIn("probhub/judge_qa_runtime.py", main_files) self.assertIn("probhub/mutation.py", main_files) self.assertIn("probhub/mutation_config.py", main_files) + self.assertIn("probhub/mutation_syntax.py", main_files) self.assertIn("references/checker-interactor.md", main_files) self.assertIn("references/mutation-testing.md", main_files) @@ -438,6 +439,8 @@ def test_windows_dependency_installer_uses_a_bounded_node_supervisor(self): command = install_deps._pip_install_command(requirements, user_install=False) self.assertEqual(command[:2], ["C:/node/node.exe", "-e"]) self.assertEqual(command[3:7], ["C:/python/python.exe", "-m", "pip", "install"]) + self.assertIn("--only-binary", command) + self.assertIn("tree-sitter,tree-sitter-cpp", command) self.assertEqual(command[-1], str(requirements)) @unittest.skipUnless(shutil.which("node"), "node is required")