diff --git a/crispen/engine.py b/crispen/engine.py deleted file mode 100644 index ae873cc..0000000 --- a/crispen/engine.py +++ /dev/null @@ -1,1448 +0,0 @@ -"""Load files, apply refactors, verify, and write back.""" - -import ast -import os -import sys -import threading -import time -from pathlib import Path -from typing import Dict, Generator, List, NamedTuple, Optional, Set, Tuple - -from .stats import RunStats - -import libcst as cst -from libcst.metadata import FullRepoManager, MetadataWrapper, QualifiedNameProvider - -from .config import CrispenConfig, format_header, load_config -from .errors import CrispenAPIError -from .file_limiter.runner import FileLimiterResult, run_file_limiter -from .patch_rewriter import ( - _FLContext, - RewriteAccumulator, - apply_patch_callgraph, - apply_patch_rewrite, -) -from .patch_updater import apply_patch_strings -from .refactors.caller_updater import CallerUpdater -from .refactors.duplicate_extractor import DuplicateExtractor -from .refactors.function_splitter import FunctionSplitter -from .refactors.if_not_else import IfNotElse -from .refactors.tuple_dataclass import TransformInfo, TupleDataclass - -# Single-file refactors applied in order before TupleDataclass. -_REFACTORS = [IfNotElse, DuplicateExtractor, FunctionSplitter] - -# Refactor keys that invoke LLM calls (used to decide whether to print config). -_LLM_REFACTOR_KEYS = frozenset( - {"duplicate_extractor", "function_splitter", "tuple_dataclass", "file_limiter"} -) - -# Canonical snake_case name for each refactor class (used by _should_run). -_REFACTOR_KEY: Dict[type, str] = { - IfNotElse: "if_not_else", - DuplicateExtractor: "duplicate_extractor", - FunctionSplitter: "function_splitter", -} - - -def _should_run(name: str, config: CrispenConfig) -> bool: - """Return True if the named refactor should run given the config. - - When ``config.enabled_refactors`` is non-empty only names in that list run. - Otherwise names in ``config.disabled_refactors`` are skipped. - """ - if config.enabled_refactors: - return name in config.enabled_refactors - return name not in config.disabled_refactors - - -# Directory names excluded from the outside-caller scan (e.g. virtual environments). -_EXCLUDED_DIR_NAMES = frozenset( - {".venv", "venv", "env", ".tox", "__pycache__", "node_modules"} -) - -# Total wall-clock budget for all files in _find_outside_callers (seconds). -_SCOPE_ANALYSIS_TIMEOUT = 10 - - -# --------------------------------------------------------------------------- -# update_diff_file_callers helpers -# --------------------------------------------------------------------------- - - -def _has_callers_outside_ranges( - source: str, func_name: str, ranges: List[Tuple[int, int]] -) -> bool: - """Return True if func_name is called at any line outside the given ranges.""" - try: - tree = ast.parse(source) - except SyntaxError: - return False - for node in ast.walk(tree): - if ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == func_name - ): - line = node.lineno - if not any(start <= line <= end for start, end in ranges): - return True - return False - - -def _blocked_private_scopes(source: str, ranges: List[Tuple[int, int]]) -> Set[str]: - """Return names of private functions that have callers outside the diff ranges.""" - try: - tree = ast.parse(source) - except SyntaxError: - return set() - blocked: Set[str] = set() - for node in ast.walk(tree): - if ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id.startswith("_") - ): - line = node.lineno - if not any(start <= line <= end for start, end in ranges): - blocked.add(node.func.id) - return blocked - - -# --------------------------------------------------------------------------- -# FileLimiter: inline-import redirect helpers -# --------------------------------------------------------------------------- - -_PROJECT_MARKERS = frozenset({"pyproject.toml", "setup.py", "setup.cfg", ".git"}) - - -def _collect_top_level_names(source: str) -> Set[str]: - """Return all names defined or imported at the module top level of *source*. - - Covers functions, classes, module-level variable assignments, and all - import styles. Returns an empty set when *source* cannot be parsed. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - names: Set[str] = set() - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - names.add(node.name) - elif isinstance(node, ast.Assign): - for target in node.targets: - if isinstance(target, ast.Name): - names.add(target.id) - elif isinstance(node, (ast.AugAssign, ast.AnnAssign)): - if isinstance(node.target, ast.Name): - names.add(node.target.id) - elif isinstance(node, ast.Import): - for alias in node.names: - names.add(alias.asname if alias.asname else alias.name.split(".")[-1]) - elif isinstance(node, ast.ImportFrom): - for alias in node.names: - if alias.name != "*": - names.add(alias.asname if alias.asname else alias.name) - return names - - -def _collect_assignment_names(source: str) -> Set[str]: - """Return names from top-level variable assignments in *source*. - - Covers ``X = …``, ``X: T = …``, and ``X += …`` where the target is a - plain ``ast.Name``. Functions, classes, and imports are excluded (they - are handled elsewhere). Returns an empty set on parse failure. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - names: Set[str] = set() - for node in tree.body: - if isinstance(node, ast.Assign): - for target in node.targets: - if isinstance(target, ast.Name): - names.add(target.id) - elif isinstance(node, (ast.AugAssign, ast.AnnAssign)): - if isinstance(node.target, ast.Name): - names.add(node.target.id) - return names - - -def _collect_imported_names(source: str) -> Set[str]: - """Return names imported at the top level of *source*. - - Handles ``import X``, ``import X as Y``, ``from X import Y``, and - ``from X import Y as Z``. Star imports (``from X import *``) are skipped. - Returns an empty set when *source* cannot be parsed. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - names: Set[str] = set() - for node in tree.body: - if isinstance(node, ast.Import): - for alias in node.names: - names.add(alias.asname if alias.asname else alias.name.split(".")[-1]) - elif isinstance(node, ast.ImportFrom): - for alias in node.names: - if alias.name != "*": - names.add(alias.asname if alias.asname else alias.name) - return names - - -def _collect_code_referenced_names(source: str) -> Set[str]: - """Return names referenced in code as *Load* expressions. - - Walks the AST looking for ``ast.Name`` nodes with ``Load`` context — - actual uses of a name in code (calls, attribute targets, right-hand-side - expressions). Pure re-export stubs (``from .sub import X # noqa: F401``) - produce no such nodes because the import alias itself is an ``ast.alias``, - not an ``ast.Name``. Returns an empty set on parse failure. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - return { - node.id - for node in ast.walk(tree) - if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) - } - - -def _module_path_for_file(file_path: str) -> Optional[str]: - """Return the dotted Python module path of *file_path*. - - Walks up from the file's directory to find the project root (the first - ancestor containing pyproject.toml, setup.py, setup.cfg, or .git), then - computes the dotted path relative to that root. - - ``__init__.py`` files are mapped to their package name (e.g. - ``mypkg/__init__.py`` → ``mypkg``) so that patch paths always use the - public package namespace rather than the internal ``.__init__`` segment. - - Returns ``None`` when the project root cannot be determined or when the - resolved path is not under the project root. - """ - abs_path = Path(file_path).resolve() - current = abs_path.parent - while True: - if any((current / m).exists() for m in _PROJECT_MARKERS): - rel = abs_path.relative_to(current) - module = ".".join(rel.with_suffix("").parts) - if module.endswith(".__init__"): - module = module[:-9] - return module - parent = current.parent - if parent == current: - return None - current = parent - - -def _redirect_inline_module_imports( - source: str, - old_mod: str, - name_to_new_mod: Dict[str, str], -) -> str: - """Replace ``from old_mod import names`` statements with new module paths. - - Scans all ``from import …`` statements at every level - (module-level and inside function/class bodies). Each imported name is - redirected to the module given by *name_to_new_mod*; names absent from the - map are kept pointing at *old_mod*. - - Returns *source* unchanged when there are no matching imports or when - *source* cannot be parsed as Python. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return source - lines = source.splitlines(keepends=True) - replacements: Dict[Tuple[int, int], str] = {} - - def _scan(stmts: list) -> None: - for node in stmts: - if ( - isinstance(node, ast.ImportFrom) - and node.level == 0 - and node.module == old_mod - ): - names = [alias.name for alias in node.names] - new_mod_to_names: Dict[str, List[str]] = {} - kept: List[str] = [] - for n in names: - dest = name_to_new_mod.get(n) - if dest: - new_mod_to_names.setdefault(dest, []).append(n) - else: - kept.append(n) - if not new_mod_to_names: - continue # No names moved; leave unchanged. - raw_line = lines[node.lineno - 1] - indent = raw_line[: len(raw_line) - len(raw_line.lstrip())] - parts: List[str] = [] - if kept: - parts.append(f"{indent}from {old_mod} import {', '.join(kept)}") - for dest_mod, dest_names in sorted(new_mod_to_names.items()): - joined = ", ".join(sorted(dest_names)) - parts.append(f"{indent}from {dest_mod} import {joined}") - replacements[(node.lineno, node.end_lineno)] = "\n".join(parts) - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - _scan(node.body) - - _scan(tree.body) - if not replacements: - return source - - result = list(lines) - for (start, end), text in sorted(replacements.items(), reverse=True): - last_line = result[end - 1] - trailing = "\n" if last_line.endswith("\n") else "" - result[start - 1 : end] = [text + trailing] - return "".join(result) - - -def _patch_inline_imports_after_test_deletion( - deleted_path: str, - deleted_dir: Path, - new_files: Dict[str, str], - per_file: Dict, - fl_new_file_final: Dict[str, Optional[str]], -) -> None: - """Redirect inline imports pointing to a deleted test module. - - When a test file is deleted after a recursive subdir split (because all - entities migrated away, leaving an empty ``original_source``), any parent - file that received injected inline imports during the *first* split still - references the now-gone module. This function rewrites those stale imports - to point directly at the new sub-file locations. - - Updates both ``per_file`` states (written to disk later by the write loop) - and ``fl_new_file_final`` entries (already on disk; re-written immediately). - """ - old_mod = _module_path_for_file(deleted_path) - if old_mod is None: - return - - # Build name → new_module by parsing top-level definitions in new_files. - name_to_new_mod: Dict[str, str] = {} - for rel_path, content in new_files.items(): - new_abs = (deleted_dir / rel_path).resolve() - new_mod = _module_path_for_file(str(new_abs)) - if new_mod is None: - continue - try: - sub_tree = ast.parse(content) - except SyntaxError: - continue - for node in sub_tree.body: - if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): - name_to_new_mod[node.name] = new_mod - - if not name_to_new_mod: - return - - for state in per_file.values(): - updated = _redirect_inline_module_imports( - state["source"], old_mod, name_to_new_mod - ) - if updated != state["source"]: - state["source"] = updated - - for path, content in list(fl_new_file_final.items()): - if not content: - continue - updated = _redirect_inline_module_imports(content, old_mod, name_to_new_mod) - if updated != content: - fl_new_file_final[path] = updated - Path(path).write_text(updated, encoding="utf-8") - - -def _build_patch_map( - filepath: str, - fl_result: "FileLimiterResult", - original_dir: Path, - pre_split_source: str = "", -) -> Dict[str, str]: - """Build old_dotted_path.EntityName → new_dotted_path.EntityName map. - - Uses "basic" mode logic: for each entity, maps to the single new file that - imports it (the "caller") rather than its definition file. If multiple new - files import the entity (forking), the entity is skipped. Import aliases - present in *pre_split_source* that appear in exactly one new file are also - included. - - Returns an empty dict when the module path cannot be determined or when no - entities were moved. - """ - old_module = _module_path_for_file(filepath) - if old_module is None: - return {} - - # Build import index: name → list of new-file rel_paths that import it. - # Build usage index: name → set of new-file rel_paths that reference it in - # code (ast.Name Load nodes — actual calls or expressions, not re-exports). - import_index: Dict[str, List[str]] = {} - usage_index: Dict[str, Set[str]] = {} - for rel_path, src in fl_result.new_files.items(): - if not src: - continue - for name in _collect_imported_names(src): - import_index.setdefault(name, []).append(rel_path) - for name in _collect_code_referenced_names(src): - usage_index.setdefault(name, set()).add(rel_path) - - patch_map: Dict[str, str] = {} - for entity_name, def_rel_target in fl_result.entity_to_target.items(): - # Callers = new files that both import this entity and reference it in - # code, excluding its definer. Pure re-export stubs import the name - # but produce no ast.Name Load nodes, so they are naturally excluded. - callers = [ - p - for p in import_index.get(entity_name, []) - if p != def_rel_target and p in usage_index.get(entity_name, set()) - ] - if len(callers) > 1: - continue # forking: multiple callers, skip - target_rel = callers[0] if callers else def_rel_target - new_module = _module_path_for_file(str(original_dir / target_rel)) - if new_module is None: - continue - patch_map[f"{old_module}.{entity_name}"] = f"{new_module}.{entity_name}" - - # Import aliases: names imported by the original file that appear in - # exactly one new sub-file (and are not already moved entities). - # Only count files that actually reference the alias in code, not stubs. - for alias_name in _collect_imported_names(pre_split_source): - if alias_name in fl_result.entity_to_target: - continue # already handled above - importers = [ - p - for p in import_index.get(alias_name, []) - if p in usage_index.get(alias_name, set()) - ] - if len(importers) != 1: - continue # zero or multiple: skip - new_module = _module_path_for_file(str(original_dir / importers[0])) - if new_module is None: - continue - patch_map[f"{old_module}.{alias_name}"] = f"{new_module}.{alias_name}" - - # Variable assignments: names assigned at the module level that were present - # in the original file and are not already tracked as entities or import - # aliases. Uses the same caller logic as named entities: 0 callers → only - # used in its defining file; 1 caller → migrated and consumed by exactly one - # other file; 2+ → forking. Only names from the original file are considered - # to avoid spurious entries for helper variables introduced by code generation. - # Names defined in multiple new files are skipped (ambiguous origin). - orig_assignments = _collect_assignment_names(pre_split_source) - def_index: Dict[str, List[str]] = {} - for rel_path, src in fl_result.new_files.items(): - if not src: - continue - for name in _collect_assignment_names(src): - if name in orig_assignments and name not in fl_result.entity_to_target: - def_index.setdefault(name, []).append(rel_path) - - for assign_name, def_rel_targets in def_index.items(): - if len(def_rel_targets) != 1: - continue # defined in multiple files → ambiguous - old_path = f"{old_module}.{assign_name}" - if old_path in patch_map: - continue # already handled by entity or import-alias section - def_rel_target = def_rel_targets[0] - callers = [ - p - for p in import_index.get(assign_name, []) - if p != def_rel_target and p in usage_index.get(assign_name, set()) - ] - if len(callers) > 1: - continue # forking: multiple consumers, skip - target_rel = callers[0] if callers else def_rel_target - new_module = _module_path_for_file(str(original_dir / target_rel)) - if new_module is None: - continue - patch_map[old_path] = f"{new_module}.{assign_name}" - - return patch_map - - -def _add_fl_context( - fl_all_contexts: List["_FLContext"], - filepath: str, - pre_split_src: str, - fl_result: "FileLimiterResult", - combined_patch_map: Dict[str, str], -) -> None: - """Append an _FLContext to *fl_all_contexts* for "rewrite" patch mode. - - Computes the forking old paths (entities in entity_to_target that basic - mode skipped because they appeared in multiple callers) and builds the - new module path map for all sub-files. Does nothing when no forking - entities exist or when the module path cannot be determined. - - When no forking entities exist but TOP_LEVEL blocks (_block_N) were - moved, also scans the new target files to find names that came from - those blocks (module-level vars, constants, imported aliases) but are - not individually tracked in entity_to_target. Each such name is added - as a specific old path (``old_module.name``) so the LLM can find any - ``with patch(old_module.name)`` calls without matching already-updated - paths like ``old_module.sub.name`` that basic mode already rewrote. - - Import aliases from the original file that basic mode skipped (forked - into multiple new sub-files) are also added so the LLM rewrite step - can determine the correct per-function patch target. - """ - old_mod = _module_path_for_file(filepath) - if old_mod is None: - return - forking_old_paths = { - f"{old_mod}.{name}" - for name in fl_result.entity_to_target - if f"{old_mod}.{name}" not in combined_patch_map - } - # Also collect names from moved _block_N entities that are NOT individually - # tracked (i.e., not in entity_to_target). These are block-internal names - # (vars, constants, imported aliases) that basic mode never maps, regardless - # of whether forking entities were also found above. - all_entity_names = set(fl_result.entity_to_target) - for entity_name, target_rel in fl_result.entity_to_target.items(): - if not entity_name.startswith("_block_"): - continue - new_src = fl_result.new_files.get(target_rel, "") - for name in _collect_top_level_names(new_src): - old_path = f"{old_mod}.{name}" - if name not in all_entity_names and old_path not in combined_patch_map: - forking_old_paths.add(old_path) - # Also add import aliases from the original file that basic mode skipped - # because they appeared in multiple new sub-files (forking). These - # aliases are absent from combined_patch_map but may still appear as - # @patch string targets in test files — the LLM rewrite step can resolve - # the correct sub-module for each test function individually. - for alias_name in _collect_imported_names(pre_split_src): - if alias_name in all_entity_names: - continue - old_path = f"{old_mod}.{alias_name}" - if old_path not in combined_patch_map: - forking_old_paths.add(old_path) - if not forking_old_paths: - return - orig_dir = Path(filepath).parent - new_mod_paths = { - rel: _module_path_for_file(str(orig_dir / rel)) or rel - for rel in fl_result.new_files - } - # For non-test subdir splits the original file stays on disk unchanged and - # fl_result.original_source is the pre-split source (runner.py restores it - # at line 704 so the original file is left untouched). The post-split - # module state lives in new_files["{subdir_name}/__init__.py"]. Use that - # as modified_source so _build_rename_guard_sets and the BFS terminal - # builder both see the correct set of names still present in the module. - init_key = f"{fl_result.subdir_name}/__init__.py" if fl_result.subdir_name else None - if init_key and init_key in fl_result.new_files: - modified_src = fl_result.new_files[init_key] or fl_result.original_source or "" - else: - modified_src = fl_result.original_source or "" - fl_all_contexts.append( - _FLContext( - filepath=filepath, - old_module=old_mod, - original_source=pre_split_src, - modified_source=modified_src, - new_files=dict(fl_result.new_files), - new_module_paths=new_mod_paths, - entity_to_target=dict(fl_result.entity_to_target), - forking_old_paths=forking_old_paths, - ) - ) - - -# --------------------------------------------------------------------------- -# Repo-root helpers -# --------------------------------------------------------------------------- - - -def _find_repo_root(changed: Dict[str, List]) -> Optional[str]: - """Find git repo root by searching parent directories for .git.""" - for filepath in changed.keys(): - p = Path(filepath).resolve().parent - while p != p.parent: - if (p / ".git").is_dir(): - return str(p) - p = p.parent - return None - - -def _file_to_module(repo_root: str, filepath: str) -> str: - """Convert an absolute file path to a dotted Python module name.""" - path = Path(filepath).resolve().relative_to(Path(repo_root).resolve()) - module = str(path.with_suffix("")).replace(os.sep, ".") - if module.endswith(".__init__"): - module = module[:-9] - return module - - -def _compute_qname(repo_root: str, filepath: str, func_name: str) -> str: - """Compute the qualified name of a function defined in filepath.""" - return f"{_file_to_module(repo_root, filepath)}.{func_name}" - - -# --------------------------------------------------------------------------- -# __init__.py alias resolution -# --------------------------------------------------------------------------- - - -def _build_alias_map(repo_root: str, canonical_qnames: Set[str]) -> Dict[str, str]: - """Map alias qualified names → canonical qualified names. - - Handles explicit re-exports like ``from .service import get_user`` in - ``pkg/__init__.py``, which creates the alias ``pkg.get_user`` for the - canonical name ``pkg.service.get_user``. - """ - alias_map: Dict[str, str] = {q: q for q in canonical_qnames} - - for init_path in Path(repo_root).rglob("__init__.py"): - pkg_parts = list(init_path.relative_to(repo_root).parts[:-1]) - pkg_qname = ".".join(pkg_parts) - - try: - source = init_path.read_text(encoding="utf-8") - tree = cst.parse_module(source) - except Exception: - continue - - for stmt in tree.body: - if not isinstance(stmt, cst.SimpleStatementLine): - continue - for s in stmt.body: - if not isinstance(s, cst.ImportFrom): - continue - if isinstance(s.names, cst.ImportStar) or not isinstance( - s.names, (list, tuple) - ): - continue - for al in s.names: - if not isinstance(al, cst.ImportAlias) or not isinstance( - al.name, cst.Name - ): - continue # pragma: no cover - func_name = al.name.value - alias_qname = f"{pkg_qname}.{func_name}" if pkg_qname else func_name - # Map this alias to a canonical qname if unambiguous - matches = [ - c for c in canonical_qnames if c.split(".")[-1] == func_name - ] - if len(matches) == 1: - alias_map[alias_qname] = matches[0] - - return alias_map - - -# --------------------------------------------------------------------------- -# Outside-caller detection using FullRepoManager -# --------------------------------------------------------------------------- - - -class _CallerFinder(cst.CSTVisitor): - """Visit a file and record which target qualified names are called.""" - - METADATA_DEPENDENCIES = (QualifiedNameProvider,) - - def __init__(self, target_qnames: Set[str]) -> None: - self.target_qnames = target_qnames - self.found: Set[str] = set() - - def visit_Call(self, node: cst.Call) -> None: - qnames = self.get_metadata(QualifiedNameProvider, node.func, set()) - for qn in qnames: - if qn.name in self.target_qnames: - self.found.add(qn.name) - - -def _visit_with_timeout(wrapper, finder, timeout: float) -> bool: - """Run wrapper.visit(finder) in a daemon thread with a wall-clock timeout. - - Returns True if the call completed within *timeout* seconds, False if it - timed out (libcst scope analysis can hang on large files). - """ - done = threading.Event() - - def _target(): - try: - wrapper.visit(finder) - finally: - done.set() - - t = threading.Thread(target=_target, daemon=True) - t.start() - return done.wait(timeout=timeout) - - -def _find_outside_callers( - repo_root: str, - target_qnames: Set[str], - diff_files: Set[str], -) -> Set[str]: - """Return the subset of *target_qnames* called in files outside *diff_files*.""" - if not target_qnames: - return set() - - repo_root_path = Path(repo_root) - outside_py = [ - p - for p in repo_root_path.rglob("*.py") - if str(p.resolve()) not in diff_files - and not any( - part in _EXCLUDED_DIR_NAMES - for part in p.relative_to(repo_root_path).parts[:-1] - ) - ] - if not outside_py: - return set() - - rel_paths = [str(p.relative_to(repo_root)) for p in outside_py] - - try: - manager = FullRepoManager(repo_root, rel_paths, {QualifiedNameProvider}) - except Exception: - # Can't build the manager → conservatively block all transforms. - return set(target_qnames) - - found_outside: Set[str] = set() - deadline = time.monotonic() + _SCOPE_ANALYSIS_TIMEOUT - for rel_path in rel_paths: - remaining = deadline - time.monotonic() - if remaining <= 0: - # Total budget exhausted: conservatively block all remaining. - found_outside.update(target_qnames) - break - try: - wrapper = manager.get_metadata_wrapper_for_path(rel_path) - finder = _CallerFinder(target_qnames) - if not _visit_with_timeout(wrapper, finder, remaining): - # This file timed out: conservatively block all transforms. - found_outside.update(target_qnames) - break - found_outside.update(finder.found) - except Exception: - continue - - return found_outside - - -# --------------------------------------------------------------------------- -# TupleDataclass helper (used in both passes) -# --------------------------------------------------------------------------- - - -class _ApplyResult(NamedTuple): - """Return type of _apply_tuple_dataclass.""" - - source: str - msgs: List[str] - td: Optional[TupleDataclass] - - -def _apply_tuple_dataclass( - filepath: str, - ranges: List[Tuple[int, int]], - source: str, - verbose: bool, - approved_public_funcs: Set[str], - min_size: int = 4, - blocked_scopes: Optional[Set[str]] = None, -) -> "_ApplyResult": - """Run TupleDataclass on *source*. Returns (new_source, messages, transformer).""" - try: - tree = cst.parse_module(source) - except cst.ParserSyntaxError as exc: - return _ApplyResult( - source, [f"SKIP {filepath} (TupleDataclass): parse error: {exc}"], None - ) - - wrapper = MetadataWrapper(tree) - try: - td = TupleDataclass( - ranges, - source=source, - verbose=verbose, - approved_public_funcs=approved_public_funcs, - min_size=min_size, - blocked_scopes=blocked_scopes, - ) - new_tree = wrapper.visit(td) - except CrispenAPIError: - raise - except Exception as exc: - return _ApplyResult( - source, - [f"SKIP {filepath} (TupleDataclass): transform error: {exc}"], - None, - ) - - new_source = td.get_rewritten_source() or new_tree.code - if new_source == source: - return _ApplyResult(source, [], td) - - try: - compile(new_source, filepath, "exec") - except SyntaxError as exc: # pragma: no cover - return _ApplyResult( - source, - [f"SKIP {filepath} (TupleDataclass): output not valid Python: {exc}"], - td, - ) - - msgs = [f"{filepath}: {m}" for m in td.get_changes()] - return _ApplyResult(new_source, msgs, td) - - -# --------------------------------------------------------------------------- -# Stats helpers -# --------------------------------------------------------------------------- - - -def _categorize_into_stats(stats: RunStats, msg: str) -> None: - """Increment the appropriate counter in *stats* for a raw change message.""" - if msg.startswith("IfNotElse:"): - stats.if_not_else += 1 - elif msg.startswith("TupleDataclass:"): - stats.tuple_to_dataclass += 1 - elif msg.startswith("DuplicateExtractor:") and "with call to" in msg: - stats.duplicate_matched += 1 - elif msg.startswith("DuplicateExtractor:"): - stats.duplicate_extracted += 1 - elif msg.startswith("split "): - stats.function_split += 1 - - -# --------------------------------------------------------------------------- -# Main engine -# --------------------------------------------------------------------------- - - -def run_engine( - changed: Dict[str, List[Tuple[int, int]]], - verbose: bool = True, - _repo_root: Optional[str] = None, - config: Optional[CrispenConfig] = None, - stats: Optional[RunStats] = None, -) -> Generator[str, None, None]: - """Apply all refactors to changed files and yield summary messages.""" - if config is None: - config = load_config() - _stats = stats if stats is not None else RunStats() - - if changed and any(_should_run(k, config) for k in _LLM_REFACTOR_KEYS): - for line in format_header(config): - print(line, file=sys.stderr, flush=True) - - # ------------------------------------------------------------------ # - # Phase 1 — single-file refactors + TupleDataclass (private only) # - # ------------------------------------------------------------------ # - per_file: Dict[str, dict] = {} - - for filepath, ranges in changed.items(): - path = Path(filepath) - if not path.exists(): - yield f"SKIP {filepath}: file not found" - continue - - original_source = path.read_text(encoding="utf-8") - current_source = original_source - file_msgs: List[str] = [] - had_parse_error = False - - for RefactorClass in _REFACTORS: - key = _REFACTOR_KEY.get(RefactorClass) - if key is not None and not _should_run(key, config): - continue - try: - current_tree = cst.parse_module(current_source) - except cst.ParserSyntaxError as exc: - file_msgs.append( - f"SKIP {filepath} ({RefactorClass.name()}): parse error: {exc}" - ) - had_parse_error = True - break - - wrapper = MetadataWrapper(current_tree) - try: - if RefactorClass is DuplicateExtractor: - transformer = DuplicateExtractor( - ranges, - source=current_source, - verbose=verbose, - min_weight=config.min_duplicate_weight, - max_seq_len=config.max_duplicate_seq_len, - model=config.model, - helper_docstrings=config.helper_docstrings, - provider=config.provider, - extraction_retries=config.extraction_retries, - llm_verify_retries=config.llm_verify_retries, - base_url=config.base_url, - tool_choice=config.tool_choice, - api_timeout=config.api_timeout, - match_functions=_should_run("match_function", config), - timing=config.timing, - current_file=filepath, - rate_limit_retries=config.rate_limit_retries, - rate_limit_backoff=config.rate_limit_backoff, - ) - elif RefactorClass is FunctionSplitter: - transformer = FunctionSplitter( - ranges, - source=current_source, - verbose=verbose, - max_lines=config.max_function_length, - model=config.model, - provider=config.provider, - helper_docstrings=config.helper_docstrings, - base_url=config.base_url, - tool_choice=config.tool_choice, - api_timeout=config.api_timeout, - current_file=filepath, - rate_limit_retries=config.rate_limit_retries, - rate_limit_backoff=config.rate_limit_backoff, - ) - else: - transformer = RefactorClass( - ranges, source=current_source, verbose=verbose - ) - transformer.current_file = filepath - transformer.timing = config.timing - new_tree = wrapper.visit(transformer) - except CrispenAPIError: - raise - except Exception as exc: - name = RefactorClass.name() - file_msgs.append(f"SKIP {filepath} ({name}): transform error: {exc}") - continue - - rewritten = transformer.get_rewritten_source() - new_source = rewritten if rewritten is not None else new_tree.code - if new_source == current_source: - continue - - try: - compile(new_source, filepath, "exec") - except SyntaxError as exc: # pragma: no cover - name = RefactorClass.name() - file_msgs.append( - f"SKIP {filepath} ({name}): output not valid Python: {exc}" - ) - continue - - for msg in transformer.get_changes(): - file_msgs.append(f"{filepath}: {msg}") - _categorize_into_stats(_stats, msg) - _stats.merge(transformer.stats) - current_source = new_source - - # Apply TupleDataclass — private functions only in this pass. - candidates: Dict[str, TransformInfo] = {} - if not had_parse_error and _should_run("tuple_dataclass", config): - blocked: Set[str] = set() - if not config.update_diff_file_callers: - blocked = _blocked_private_scopes(current_source, ranges) - new_source, msgs, td = _apply_tuple_dataclass( - filepath, - ranges, - current_source, - verbose, - approved_public_funcs=set(), - min_size=config.min_tuple_size, - blocked_scopes=blocked, - ) - current_source = new_source - file_msgs.extend(msgs) - if td is not None: - for m in td.get_changes(): - _categorize_into_stats(_stats, m) - candidates = td.get_candidate_public_transforms() - # Run CallerUpdater for private function callers in this file. - private_transforms = td.get_private_transforms() - if private_transforms: - try: - cu_tree = cst.parse_module(current_source) - cu_wrapper = MetadataWrapper(cu_tree) - cu = CallerUpdater( - ranges, - transforms={}, - local_transforms=private_transforms, - source=current_source, - verbose=verbose, - ) - cu_new_source = cu_wrapper.visit(cu).code - except Exception: - cu_new_source = current_source - if cu_new_source != current_source: - try: - compile(cu_new_source, filepath, "exec") - except SyntaxError: # pragma: no cover - pass - else: - for msg in cu.get_changes(): - file_msgs.append(f"{filepath}: {msg}") - _categorize_into_stats(_stats, msg) - current_source = cu_new_source - - per_file[filepath] = { - "original": original_source, - "source": current_source, - "msgs": file_msgs, - "candidates": candidates, - "ranges": ranges, - } - - # ------------------------------------------------------------------ # - # Phase 2 — cross-file public-function transforms + caller updates # - # ------------------------------------------------------------------ # - repo_root = _repo_root if _repo_root is not None else _find_repo_root(changed) - - if repo_root and per_file: - # Collect all public-function candidates with their qualified names. - all_candidates: Dict[str, Tuple[TransformInfo, str]] = {} - for filepath, state in per_file.items(): - for func_name, info in state["candidates"].items(): - try: - qname = _compute_qname(repo_root, filepath, func_name) - all_candidates[qname] = (info, filepath) - except ValueError: - pass # file not under repo_root - - if all_candidates: - canonical_qnames = set(all_candidates.keys()) - alias_map = _build_alias_map(repo_root, canonical_qnames) - all_qnames = set(alias_map.keys()) # canonical + __init__ aliases - - diff_files = {str(Path(f).resolve()) for f in per_file} - outside_callers = _find_outside_callers(repo_root, all_qnames, diff_files) - - # Any alias with an outside caller blocks its canonical transform. - outside_canonical = { - alias_map[q] for q in outside_callers if q in alias_map - } - - # When update_diff_file_callers is disabled, also block functions - # that have callers within diff files but outside the diff ranges. - if not config.update_diff_file_callers: - for qname in list(canonical_qnames - outside_canonical): - info, _ = all_candidates[qname] - for caller_state in per_file.values(): - if _has_callers_outside_ranges( - caller_state["source"], - info.func_name, - caller_state["ranges"], - ): - outside_canonical.add(qname) - break - - approved_canonical = canonical_qnames - outside_canonical - - for qname in canonical_qnames - approved_canonical: - info, filepath = all_candidates[qname] - yield ( - f"SKIP {filepath}: {info.func_name}:" - f" callers exist outside the diff" - ) - - if approved_canonical: - # Build the transforms dict for CallerUpdater (all names → info). - approved_transforms: Dict[str, TransformInfo] = {} - approved_by_file: Dict[str, Set[str]] = {} - - for qname in approved_canonical: - info, filepath = all_candidates[qname] - approved_transforms[qname] = info - approved_by_file.setdefault(filepath, set()).add(info.func_name) - - for alias, canonical in alias_map.items(): - if canonical in approved_canonical: - approved_transforms[alias] = all_candidates[canonical][0] - - # Second TupleDataclass pass — approved public functions only. - for filepath, funcs in approved_by_file.items(): - state = per_file[filepath] - new_source, msgs, td2 = _apply_tuple_dataclass( - filepath, - state["ranges"], - state["source"], - verbose, - approved_public_funcs=funcs, - min_size=config.min_tuple_size, - ) - state["source"] = new_source - state["msgs"].extend(msgs) - if td2 is not None: - for m in td2.get_changes(): - _categorize_into_stats(_stats, m) - - # CallerUpdater pass — all diff files. - for filepath, state in per_file.items(): - try: - file_module = _file_to_module(repo_root, filepath) - except ValueError: - continue - - try: - current_tree = cst.parse_module(state["source"]) - except cst.ParserSyntaxError: - continue - - wrapper = MetadataWrapper(current_tree) - try: - cu = CallerUpdater( - state["ranges"], - approved_transforms, - file_module=file_module, - source=state["source"], - verbose=verbose, - ) - new_tree = wrapper.visit(cu) - except Exception: - continue - - new_source = new_tree.code - if new_source == state["source"]: - continue - - try: - compile(new_source, filepath, "exec") - except SyntaxError: # pragma: no cover - continue - - for msg in cu.get_changes(): - state["msgs"].append(f"{filepath}: {msg}") - _categorize_into_stats(_stats, msg) - state["source"] = new_source - - # ------------------------------------------------------------------ # - # Phase 3 — FileLimiter: split files exceeding max_file_lines # - # ------------------------------------------------------------------ # - combined_patch_map: Dict[str, str] = {} - _fl_all_contexts: List[_FLContext] = [] - if config.max_file_lines > 0 and _should_run("file_limiter", config): - # Pending queue for recursive FileLimiter processing: (filepath, source) - # pairs for newly-created files that are still over the limit. - _fl_recursive: List[Tuple[str, str]] = [] - # Track the final content of each new file created by FileLimiter so - # lines_added/deleted counts reflect the net result, not interim states. - _fl_new_file_final: Dict[str, Optional[str]] = {} - # Deduplicate verified functions/classes across recursive passes so - # entities migrated more than once are not counted multiple times. - _fl_verified_func_names: Set[str] = set() - _fl_verified_class_names: Set[str] = set() - _fl_verified_entity_lines: Dict[str, int] = {} - - for filepath, state in per_file.items(): - if len(state["source"].splitlines()) <= config.max_file_lines: - continue - - try: - fl_result = run_file_limiter( - filepath=filepath, - original_source=state["original"], - post_source=state["source"], - diff_ranges=state["ranges"], - config=config, - verbose=verbose, - timing=config.timing, - ) - except CrispenAPIError: - raise - - _stats.file_limiter_llm_calls += fl_result.llm_calls - if fl_result.llm_elapsed > 0 or fl_result.llm_input_tokens > 0: - _stats.record_llm_call( - fl_result.llm_elapsed, - fl_result.llm_input_tokens, - fl_result.llm_output_tokens, - "file_limiter", - "file_limiter", - filepath, - ) - _fl_verified_func_names |= fl_result.verified_function_names - _fl_verified_class_names |= fl_result.verified_class_names - _fl_verified_entity_lines.update(fl_result.verified_entity_line_counts) - - if fl_result.messages: - state["msgs"].extend(fl_result.messages) - - if fl_result.abort or not fl_result.new_files: - continue - - original_dir = Path(filepath).parent - for rel_path, new_source in fl_result.new_files.items(): - new_path = original_dir / rel_path - new_path.parent.mkdir(parents=True, exist_ok=True) - if new_path.parent != original_dir: - init_py = new_path.parent / "__init__.py" - if not init_py.exists(): - init_py.write_text("", encoding="utf-8") - new_path.write_text(new_source, encoding="utf-8") - _stats.files_edited.append(str(new_path)) - _stats.file_limiter_edits += 1 - _fl_new_file_final[str(new_path)] = new_source - if ( - config.file_limiter_recursive - and len(new_source.splitlines()) > config.max_file_lines - ): - _fl_recursive.append((str(new_path), new_source)) - - pre_split_src = state["source"] - state["source"] = fl_result.original_source - - if fl_result.entity_to_target: - combined_patch_map.update( - _build_patch_map( - filepath, fl_result, Path(filepath).parent, pre_split_src - ) - ) - if config.file_limiter_patch_update in ("basic", "rewrite"): - _add_fl_context( - _fl_all_contexts, - filepath, - pre_split_src, - fl_result, - combined_patch_map, - ) - - # For non-test whole-file subdir splits (without __main__), delete - # the original file now that service/__init__.py takes its place as - # the public entry point. state["source"] was reset to - # state["original"] above, so the final write loop will see no diff - # and skip the (deleted) file. Count the original lines as deleted - # so stats stay accurate. - # When has_main is True the original file is kept on disk as the - # runnable script entry point; the engine's write loop will update - # it with the re-export stubs from fl_result.original_source. - if ( - fl_result.subdir_name is not None - and not Path(filepath).name.startswith("test_") - and not fl_result.has_main - ): - Path(filepath).unlink() - _stats.count_lines_changed(state["original"], "") - - # Recursive pass: process any newly-created files that are still over - # the limit. Each iteration may enqueue further files; the loop ends - # when no oversized new files remain. - _recursive_msgs: List[str] = [] - while _fl_recursive: - r_path, r_source = _fl_recursive.pop(0) - n_lines = len(r_source.splitlines()) - try: - r_result = run_file_limiter( - filepath=r_path, - original_source="", - post_source=r_source, - diff_ranges=[(1, n_lines)], - config=config, - verbose=verbose, - ) - except CrispenAPIError: - raise - - _stats.file_limiter_llm_calls += r_result.llm_calls - if r_result.llm_elapsed > 0 or r_result.llm_input_tokens > 0: - _stats.record_llm_call( - r_result.llm_elapsed, - r_result.llm_input_tokens, - r_result.llm_output_tokens, - "file_limiter", - "file_limiter", - r_path, - ) - _fl_verified_func_names |= r_result.verified_function_names - _fl_verified_class_names |= r_result.verified_class_names - _fl_verified_entity_lines.update(r_result.verified_entity_line_counts) - - _recursive_msgs.extend(r_result.messages) - - if r_result.abort or not r_result.new_files: - continue - - r_dir = Path(r_path).parent - for rel_path, new_source in r_result.new_files.items(): - new_path = r_dir / rel_path - new_path.parent.mkdir(parents=True, exist_ok=True) - if new_path.parent != r_dir: - init_py = new_path.parent / "__init__.py" - if not init_py.exists(): - init_py.write_text("", encoding="utf-8") - new_path.write_text(new_source, encoding="utf-8") - _stats.files_edited.append(str(new_path)) - _stats.file_limiter_edits += 1 - _fl_new_file_final[str(new_path)] = new_source - if len(new_source.splitlines()) > config.max_file_lines: - _fl_recursive.append((str(new_path), new_source)) - - if r_result.entity_to_target and not r_result.abort: - combined_patch_map.update( - _build_patch_map(r_path, r_result, Path(r_path).parent, r_source) - ) - if config.file_limiter_patch_update in ("basic", "rewrite"): - _add_fl_context( - _fl_all_contexts, - r_path, - r_source, - r_result, - combined_patch_map, - ) - - # Subdir split of a recursively-processed file: delete the file - # that was replaced by a package __init__.py. Handle before the - # rewrite check so we don't write-then-delete (and double-count lines). - # Skip deletion when has_main is True (original kept as entry point). - if ( - r_result.subdir_name is not None - and not Path(r_path).name.startswith("test_") - and not r_result.has_main - ): - Path(r_path).unlink() - _fl_new_file_final.pop(str(r_path), None) - elif r_result.original_source != r_source: - if r_result.original_source: - Path(r_path).write_text(r_result.original_source, encoding="utf-8") - _fl_new_file_final[str(r_path)] = r_result.original_source - elif Path(r_path).name == "__init__.py": - # Keep __init__.py even when empty — it defines the package. - Path(r_path).write_text("", encoding="utf-8") - _fl_new_file_final[str(r_path)] = "" - else: - # Before deleting a test file, redirect any inline imports - # in parent or sibling files that point to the old module. - if Path(r_path).name.startswith("test_"): - _patch_inline_imports_after_test_deletion( - r_path, - r_dir, - r_result.new_files, - per_file, - _fl_new_file_final, - ) - Path(r_path).unlink() - _fl_new_file_final.pop(str(r_path), None) - - for path, content in _fl_new_file_final.items(): - _stats.count_lines_changed("", content) - _stats.file_limiter_functions_verified = len(_fl_verified_func_names) - _stats.file_limiter_classes_verified = len(_fl_verified_class_names) - _stats.file_limiter_lines_verified = sum(_fl_verified_entity_lines.values()) - yield from _recursive_msgs - - # Flatten transitive chains in combined_patch_map. When recursive splits - # run, round 1 may produce A→B and round 2 may produce B→C. Without - # flattening, apply_patch_strings (single-pass) would leave consumers of - # A pointing at the intermediate path B instead of the final path C. - if combined_patch_map: - changed = True - while changed: - changed = False - for k in list(combined_patch_map): - v = combined_patch_map[k] - if v in combined_patch_map and combined_patch_map[v] != v: - combined_patch_map[k] = combined_patch_map[v] - changed = True - - # ------------------------------------------------------------------ # - # Phase 4 — Update @patch strings after FileLimiter entity moves # - # ------------------------------------------------------------------ # - _patch_acc = RewriteAccumulator() - if ( - config.file_limiter_patch_update in ("basic", "rewrite") - and combined_patch_map - and repo_root - ): - _stats.patch_single_candidate += len(combined_patch_map) - # Update per_file sources still in memory (not yet written to disk). - for filepath, state in per_file.items(): - new_src = apply_patch_strings(state["source"], combined_patch_map) - if new_src != state["source"]: - state["source"] = new_src - state["msgs"].append( - f"{filepath}: patch_update: updated @patch strings" - ) - _stats.patch_update_edits += 1 - # Scan every other *.py file in the repo and update on disk. - per_file_abs = {str(Path(f).resolve()) for f in per_file} - repo_root_path = Path(repo_root) - for py_file in sorted(repo_root_path.rglob("*.py")): - if str(py_file.resolve()) in per_file_abs: - continue - if any( - part in _EXCLUDED_DIR_NAMES - for part in py_file.relative_to(repo_root_path).parts[:-1] - ): - continue - try: - src = py_file.read_text(encoding="utf-8") - except OSError: - continue - new_src = apply_patch_strings(src, combined_patch_map) - if new_src != src: - py_file.write_text(new_src, encoding="utf-8") - _stats.patch_update_edits += 1 - yield f"{py_file}: patch_update: updated @patch strings" - - _cg_candidates: Dict[str, Dict[str, Dict[str, List[str]]]] = {} - if config.file_limiter_patch_update in ("basic", "rewrite") and _fl_all_contexts: - for _cg_msg in apply_patch_callgraph( - _fl_all_contexts, - per_file, - repo_root, - verbose=verbose, - candidates_out=_cg_candidates, - config=config, - _acc=_patch_acc, - ): - _stats.patch_update_edits += 1 - yield _cg_msg - - if config.file_limiter_patch_update == "rewrite" and _fl_all_contexts: - yield from apply_patch_rewrite( - _fl_all_contexts, - per_file, - repo_root, - config, - verbose=verbose, - _acc=_patch_acc, - cg_candidates=_cg_candidates or None, - ) - _stats.patch_rewrite_llm_calls += _patch_acc.calls - if _patch_acc.elapsed > 0 or _patch_acc.input_tokens > 0: - _stats.record_llm_call( - _patch_acc.elapsed, - _patch_acc.input_tokens, - _patch_acc.output_tokens, - "file_limiter", - "patch_rewriter", - "", - ) - _stats.patch_update_edits += _patch_acc.files_updated - - _stats.patch_cg_resolved += _patch_acc.cg_resolved - _stats.patch_llm_no_change += _patch_acc.no_change - _stats.patch_llm_rename += _patch_acc.rename - _stats.patch_llm_rewrite += _patch_acc.rewrite - _stats.patch_edit_failures += _patch_acc.edit_failures - - # ------------------------------------------------------------------ # - # Write modified files and yield all messages # - # ------------------------------------------------------------------ # - for filepath, state in per_file.items(): - if state["source"] != state["original"]: - if state["source"]: - Path(filepath).write_text(state["source"], encoding="utf-8") - elif Path(filepath).name == "__init__.py": - # Keep __init__.py even when empty — it defines the package. - Path(filepath).write_text("", encoding="utf-8") - elif Path(filepath).exists(): - Path(filepath).unlink() - _stats.files_edited.append(filepath) - _stats.count_lines_changed(state["original"], state["source"]) - yield from state["msgs"] diff --git a/crispen/engine/__init__.py b/crispen/engine/__init__.py new file mode 100644 index 0000000..2ef438a --- /dev/null +++ b/crispen/engine/__init__.py @@ -0,0 +1,690 @@ +"""Load files, apply refactors, verify, and write back.""" + +import sys +from pathlib import Path +from typing import Dict, Generator, List, Optional, Set, Tuple + +from ..stats import RunStats + +import libcst as cst +from libcst.metadata import MetadataWrapper + +from ..config import CrispenConfig, format_header, load_config +from ..errors import CrispenAPIError +from ..file_limiter.runner import run_file_limiter +from ..patch_rewriter import ( + _FLContext, + RewriteAccumulator, + apply_patch_callgraph, + apply_patch_rewrite, +) +from ..patch_updater import apply_patch_strings +from ..refactors.caller_updater import CallerUpdater +from ..refactors.duplicate_extractor import DuplicateExtractor +from ..refactors.function_splitter import FunctionSplitter +from ..refactors.if_not_else import IfNotElse +from ..refactors.tuple_dataclass import TransformInfo +from .file_limiter import _add_fl_context # fmt: skip # noqa: F401, E501 +from .file_limiter import _build_patch_map # fmt: skip # noqa: F401, E501 +from .file_limiter import _categorize_into_stats # fmt: skip # noqa: F401, E501 +from .file_limiter import _collect_assignment_names # fmt: skip # noqa: F401, E501 +from .file_limiter import _collect_code_referenced_names # fmt: skip # noqa: F401, E501 +from .file_limiter import _collect_imported_names # fmt: skip # noqa: F401, E501 +from .file_limiter import _collect_top_level_names # fmt: skip # noqa: F401, E501 +from .file_limiter import _module_path_for_file # fmt: skip # noqa: F401, E501 +from .file_limiter import _patch_inline_imports_after_test_deletion # fmt: skip # noqa: F401, E501 +from .file_limiter import _redirect_inline_module_imports # fmt: skip # noqa: F401, E501 +from .helpers import _EXCLUDED_DIR_NAMES # fmt: skip # noqa: F401, E501 +from .helpers import _apply_tuple_dataclass # fmt: skip # noqa: F401, E501 +from .helpers import _blocked_private_scopes # fmt: skip # noqa: F401, E501 +from .helpers import _build_alias_map # fmt: skip # noqa: F401, E501 +from .helpers import _compute_qname # fmt: skip # noqa: F401, E501 +from .helpers import _file_to_module # fmt: skip # noqa: F401, E501 +from .helpers import _find_outside_callers # fmt: skip # noqa: F401, E501 +from .helpers import _find_repo_root # fmt: skip # noqa: F401, E501 +from .helpers import _has_callers_outside_ranges # fmt: skip # noqa: F401, E501 +from .helpers import _should_run # fmt: skip # noqa: F401, E501 +from .helpers import _visit_with_timeout # fmt: skip # noqa: F401, E501 + +# Single-file refactors applied in order before TupleDataclass. +_REFACTORS = [IfNotElse, DuplicateExtractor, FunctionSplitter] + +# Refactor keys that invoke LLM calls (used to decide whether to print config). +_LLM_REFACTOR_KEYS = frozenset( + {"duplicate_extractor", "function_splitter", "tuple_dataclass", "file_limiter"} +) + +# Canonical snake_case name for each refactor class (used by _should_run). +_REFACTOR_KEY: Dict[type, str] = { + IfNotElse: "if_not_else", + DuplicateExtractor: "duplicate_extractor", + FunctionSplitter: "function_splitter", +} + + +# --------------------------------------------------------------------------- +# Main engine +# --------------------------------------------------------------------------- + + +def run_engine( + changed: Dict[str, List[Tuple[int, int]]], + verbose: bool = True, + _repo_root: Optional[str] = None, + config: Optional[CrispenConfig] = None, + stats: Optional[RunStats] = None, +) -> Generator[str, None, None]: + """Apply all refactors to changed files and yield summary messages.""" + if config is None: + config = load_config() + _stats = stats if stats is not None else RunStats() + + if changed and any(_should_run(k, config) for k in _LLM_REFACTOR_KEYS): + for line in format_header(config): + print(line, file=sys.stderr, flush=True) + + # ------------------------------------------------------------------ # + # Phase 1 — single-file refactors + TupleDataclass (private only) # + # ------------------------------------------------------------------ # + per_file: Dict[str, dict] = {} + + for filepath, ranges in changed.items(): + path = Path(filepath) + if not path.exists(): + yield f"SKIP {filepath}: file not found" + continue + + original_source = path.read_text(encoding="utf-8") + current_source = original_source + file_msgs: List[str] = [] + had_parse_error = False + + for RefactorClass in _REFACTORS: + key = _REFACTOR_KEY.get(RefactorClass) + if key is not None and not _should_run(key, config): + continue + try: + current_tree = cst.parse_module(current_source) + except cst.ParserSyntaxError as exc: + file_msgs.append( + f"SKIP {filepath} ({RefactorClass.name()}): parse error: {exc}" + ) + had_parse_error = True + break + + wrapper = MetadataWrapper(current_tree) + try: + if RefactorClass is DuplicateExtractor: + transformer = DuplicateExtractor( + ranges, + source=current_source, + verbose=verbose, + min_weight=config.min_duplicate_weight, + max_seq_len=config.max_duplicate_seq_len, + model=config.model, + helper_docstrings=config.helper_docstrings, + provider=config.provider, + extraction_retries=config.extraction_retries, + llm_verify_retries=config.llm_verify_retries, + base_url=config.base_url, + tool_choice=config.tool_choice, + api_timeout=config.api_timeout, + match_functions=_should_run("match_function", config), + timing=config.timing, + current_file=filepath, + rate_limit_retries=config.rate_limit_retries, + rate_limit_backoff=config.rate_limit_backoff, + ) + elif RefactorClass is FunctionSplitter: + transformer = FunctionSplitter( + ranges, + source=current_source, + verbose=verbose, + max_lines=config.max_function_length, + model=config.model, + provider=config.provider, + helper_docstrings=config.helper_docstrings, + base_url=config.base_url, + tool_choice=config.tool_choice, + api_timeout=config.api_timeout, + current_file=filepath, + rate_limit_retries=config.rate_limit_retries, + rate_limit_backoff=config.rate_limit_backoff, + ) + else: + transformer = RefactorClass( + ranges, source=current_source, verbose=verbose + ) + transformer.current_file = filepath + transformer.timing = config.timing + new_tree = wrapper.visit(transformer) + except CrispenAPIError: + raise + except Exception as exc: + name = RefactorClass.name() + file_msgs.append(f"SKIP {filepath} ({name}): transform error: {exc}") + continue + + rewritten = transformer.get_rewritten_source() + new_source = rewritten if rewritten is not None else new_tree.code + if new_source == current_source: + continue + + try: + compile(new_source, filepath, "exec") + except SyntaxError as exc: # pragma: no cover + name = RefactorClass.name() + file_msgs.append( + f"SKIP {filepath} ({name}): output not valid Python: {exc}" + ) + continue + + for msg in transformer.get_changes(): + file_msgs.append(f"{filepath}: {msg}") + _categorize_into_stats(_stats, msg) + _stats.merge(transformer.stats) + current_source = new_source + + # Apply TupleDataclass — private functions only in this pass. + candidates: Dict[str, TransformInfo] = {} + if not had_parse_error and _should_run("tuple_dataclass", config): + blocked: Set[str] = set() + if not config.update_diff_file_callers: + blocked = _blocked_private_scopes(current_source, ranges) + new_source, msgs, td = _apply_tuple_dataclass( + filepath, + ranges, + current_source, + verbose, + approved_public_funcs=set(), + min_size=config.min_tuple_size, + blocked_scopes=blocked, + ) + current_source = new_source + file_msgs.extend(msgs) + if td is not None: + for m in td.get_changes(): + _categorize_into_stats(_stats, m) + candidates = td.get_candidate_public_transforms() + # Run CallerUpdater for private function callers in this file. + private_transforms = td.get_private_transforms() + if private_transforms: + try: + cu_tree = cst.parse_module(current_source) + cu_wrapper = MetadataWrapper(cu_tree) + cu = CallerUpdater( + ranges, + transforms={}, + local_transforms=private_transforms, + source=current_source, + verbose=verbose, + ) + cu_new_source = cu_wrapper.visit(cu).code + except Exception: + cu_new_source = current_source + if cu_new_source != current_source: + try: + compile(cu_new_source, filepath, "exec") + except SyntaxError: # pragma: no cover + pass + else: + for msg in cu.get_changes(): + file_msgs.append(f"{filepath}: {msg}") + _categorize_into_stats(_stats, msg) + current_source = cu_new_source + + per_file[filepath] = { + "original": original_source, + "source": current_source, + "msgs": file_msgs, + "candidates": candidates, + "ranges": ranges, + } + + # ------------------------------------------------------------------ # + # Phase 2 — cross-file public-function transforms + caller updates # + # ------------------------------------------------------------------ # + repo_root = _repo_root if _repo_root is not None else _find_repo_root(changed) + + if repo_root and per_file: + # Collect all public-function candidates with their qualified names. + all_candidates: Dict[str, Tuple[TransformInfo, str]] = {} + for filepath, state in per_file.items(): + for func_name, info in state["candidates"].items(): + try: + qname = _compute_qname(repo_root, filepath, func_name) + all_candidates[qname] = (info, filepath) + except ValueError: + pass # file not under repo_root + + if all_candidates: + canonical_qnames = set(all_candidates.keys()) + alias_map = _build_alias_map(repo_root, canonical_qnames) + all_qnames = set(alias_map.keys()) # canonical + __init__ aliases + + diff_files = {str(Path(f).resolve()) for f in per_file} + outside_callers = _find_outside_callers(repo_root, all_qnames, diff_files) + + # Any alias with an outside caller blocks its canonical transform. + outside_canonical = { + alias_map[q] for q in outside_callers if q in alias_map + } + + # When update_diff_file_callers is disabled, also block functions + # that have callers within diff files but outside the diff ranges. + if not config.update_diff_file_callers: + for qname in list(canonical_qnames - outside_canonical): + info, _ = all_candidates[qname] + for caller_state in per_file.values(): + if _has_callers_outside_ranges( + caller_state["source"], + info.func_name, + caller_state["ranges"], + ): + outside_canonical.add(qname) + break + + approved_canonical = canonical_qnames - outside_canonical + + for qname in canonical_qnames - approved_canonical: + info, filepath = all_candidates[qname] + yield ( + f"SKIP {filepath}: {info.func_name}:" + f" callers exist outside the diff" + ) + + if approved_canonical: + # Build the transforms dict for CallerUpdater (all names → info). + approved_transforms: Dict[str, TransformInfo] = {} + approved_by_file: Dict[str, Set[str]] = {} + + for qname in approved_canonical: + info, filepath = all_candidates[qname] + approved_transforms[qname] = info + approved_by_file.setdefault(filepath, set()).add(info.func_name) + + for alias, canonical in alias_map.items(): + if canonical in approved_canonical: + approved_transforms[alias] = all_candidates[canonical][0] + + # Second TupleDataclass pass — approved public functions only. + for filepath, funcs in approved_by_file.items(): + state = per_file[filepath] + new_source, msgs, td2 = _apply_tuple_dataclass( + filepath, + state["ranges"], + state["source"], + verbose, + approved_public_funcs=funcs, + min_size=config.min_tuple_size, + ) + state["source"] = new_source + state["msgs"].extend(msgs) + if td2 is not None: + for m in td2.get_changes(): + _categorize_into_stats(_stats, m) + + # CallerUpdater pass — all diff files. + for filepath, state in per_file.items(): + try: + file_module = _file_to_module(repo_root, filepath) + except ValueError: + continue + + try: + current_tree = cst.parse_module(state["source"]) + except cst.ParserSyntaxError: + continue + + wrapper = MetadataWrapper(current_tree) + try: + cu = CallerUpdater( + state["ranges"], + approved_transforms, + file_module=file_module, + source=state["source"], + verbose=verbose, + ) + new_tree = wrapper.visit(cu) + except Exception: + continue + + new_source = new_tree.code + if new_source == state["source"]: + continue + + try: + compile(new_source, filepath, "exec") + except SyntaxError: # pragma: no cover + continue + + for msg in cu.get_changes(): + state["msgs"].append(f"{filepath}: {msg}") + _categorize_into_stats(_stats, msg) + state["source"] = new_source + + # ------------------------------------------------------------------ # + # Phase 3 — FileLimiter: split files exceeding max_file_lines # + # ------------------------------------------------------------------ # + combined_patch_map: Dict[str, str] = {} + _fl_all_contexts: List[_FLContext] = [] + if config.max_file_lines > 0 and _should_run("file_limiter", config): + # Pending queue for recursive FileLimiter processing: (filepath, source) + # pairs for newly-created files that are still over the limit. + _fl_recursive: List[Tuple[str, str]] = [] + # Track the final content of each new file created by FileLimiter so + # lines_added/deleted counts reflect the net result, not interim states. + _fl_new_file_final: Dict[str, Optional[str]] = {} + # Deduplicate verified functions/classes across recursive passes so + # entities migrated more than once are not counted multiple times. + _fl_verified_func_names: Set[str] = set() + _fl_verified_class_names: Set[str] = set() + _fl_verified_entity_lines: Dict[str, int] = {} + + for filepath, state in per_file.items(): + if len(state["source"].splitlines()) <= config.max_file_lines: + continue + + try: + fl_result = run_file_limiter( + filepath=filepath, + original_source=state["original"], + post_source=state["source"], + diff_ranges=state["ranges"], + config=config, + verbose=verbose, + timing=config.timing, + ) + except CrispenAPIError: + raise + + _stats.file_limiter_llm_calls += fl_result.llm_calls + if fl_result.llm_elapsed > 0 or fl_result.llm_input_tokens > 0: + _stats.record_llm_call( + fl_result.llm_elapsed, + fl_result.llm_input_tokens, + fl_result.llm_output_tokens, + "file_limiter", + "file_limiter", + filepath, + ) + _fl_verified_func_names |= fl_result.verified_function_names + _fl_verified_class_names |= fl_result.verified_class_names + _fl_verified_entity_lines.update(fl_result.verified_entity_line_counts) + + if fl_result.messages: + state["msgs"].extend(fl_result.messages) + + if fl_result.abort or not fl_result.new_files: + continue + + original_dir = Path(filepath).parent + for rel_path, new_source in fl_result.new_files.items(): + new_path = original_dir / rel_path + new_path.parent.mkdir(parents=True, exist_ok=True) + if new_path.parent != original_dir: + init_py = new_path.parent / "__init__.py" + if not init_py.exists(): + init_py.write_text("", encoding="utf-8") + new_path.write_text(new_source, encoding="utf-8") + _stats.files_edited.append(str(new_path)) + _stats.file_limiter_edits += 1 + _fl_new_file_final[str(new_path)] = new_source + if ( + config.file_limiter_recursive + and len(new_source.splitlines()) > config.max_file_lines + ): + _fl_recursive.append((str(new_path), new_source)) + + pre_split_src = state["source"] + state["source"] = fl_result.original_source + + if fl_result.entity_to_target: + combined_patch_map.update( + _build_patch_map( + filepath, fl_result, Path(filepath).parent, pre_split_src + ) + ) + if config.file_limiter_patch_update in ("basic", "rewrite"): + _add_fl_context( + _fl_all_contexts, + filepath, + pre_split_src, + fl_result, + combined_patch_map, + ) + + # For non-test whole-file subdir splits (without __main__), delete + # the original file now that service/__init__.py takes its place as + # the public entry point. state["source"] was reset to + # state["original"] above, so the final write loop will see no diff + # and skip the (deleted) file. Count the original lines as deleted + # so stats stay accurate. + # When has_main is True the original file is kept on disk as the + # runnable script entry point; the engine's write loop will update + # it with the re-export stubs from fl_result.original_source. + if ( + fl_result.subdir_name is not None + and not Path(filepath).name.startswith("test_") + and not fl_result.has_main + ): + Path(filepath).unlink() + _stats.count_lines_changed(state["original"], "") + + # Recursive pass: process any newly-created files that are still over + # the limit. Each iteration may enqueue further files; the loop ends + # when no oversized new files remain. + _recursive_msgs: List[str] = [] + while _fl_recursive: + r_path, r_source = _fl_recursive.pop(0) + n_lines = len(r_source.splitlines()) + try: + r_result = run_file_limiter( + filepath=r_path, + original_source="", + post_source=r_source, + diff_ranges=[(1, n_lines)], + config=config, + verbose=verbose, + ) + except CrispenAPIError: + raise + + _stats.file_limiter_llm_calls += r_result.llm_calls + if r_result.llm_elapsed > 0 or r_result.llm_input_tokens > 0: + _stats.record_llm_call( + r_result.llm_elapsed, + r_result.llm_input_tokens, + r_result.llm_output_tokens, + "file_limiter", + "file_limiter", + r_path, + ) + _fl_verified_func_names |= r_result.verified_function_names + _fl_verified_class_names |= r_result.verified_class_names + _fl_verified_entity_lines.update(r_result.verified_entity_line_counts) + + _recursive_msgs.extend(r_result.messages) + + if r_result.abort or not r_result.new_files: + continue + + r_dir = Path(r_path).parent + for rel_path, new_source in r_result.new_files.items(): + new_path = r_dir / rel_path + new_path.parent.mkdir(parents=True, exist_ok=True) + if new_path.parent != r_dir: + init_py = new_path.parent / "__init__.py" + if not init_py.exists(): + init_py.write_text("", encoding="utf-8") + new_path.write_text(new_source, encoding="utf-8") + _stats.files_edited.append(str(new_path)) + _stats.file_limiter_edits += 1 + _fl_new_file_final[str(new_path)] = new_source + if len(new_source.splitlines()) > config.max_file_lines: + _fl_recursive.append((str(new_path), new_source)) + + if r_result.entity_to_target and not r_result.abort: + combined_patch_map.update( + _build_patch_map(r_path, r_result, Path(r_path).parent, r_source) + ) + if config.file_limiter_patch_update in ("basic", "rewrite"): + _add_fl_context( + _fl_all_contexts, + r_path, + r_source, + r_result, + combined_patch_map, + ) + + # Subdir split of a recursively-processed file: delete the file + # that was replaced by a package __init__.py. Handle before the + # rewrite check so we don't write-then-delete (and double-count lines). + # Skip deletion when has_main is True (original kept as entry point). + if ( + r_result.subdir_name is not None + and not Path(r_path).name.startswith("test_") + and not r_result.has_main + ): + Path(r_path).unlink() + _fl_new_file_final.pop(str(r_path), None) + elif r_result.original_source != r_source: + if r_result.original_source: + Path(r_path).write_text(r_result.original_source, encoding="utf-8") + _fl_new_file_final[str(r_path)] = r_result.original_source + elif Path(r_path).name == "__init__.py": + # Keep __init__.py even when empty — it defines the package. + Path(r_path).write_text("", encoding="utf-8") + _fl_new_file_final[str(r_path)] = "" + else: + # Before deleting a test file, redirect any inline imports + # in parent or sibling files that point to the old module. + if Path(r_path).name.startswith("test_"): + _patch_inline_imports_after_test_deletion( + r_path, + r_dir, + r_result.new_files, + per_file, + _fl_new_file_final, + ) + Path(r_path).unlink() + _fl_new_file_final.pop(str(r_path), None) + + for path, content in _fl_new_file_final.items(): + _stats.count_lines_changed("", content) + _stats.file_limiter_functions_verified = len(_fl_verified_func_names) + _stats.file_limiter_classes_verified = len(_fl_verified_class_names) + _stats.file_limiter_lines_verified = sum(_fl_verified_entity_lines.values()) + yield from _recursive_msgs + + # Flatten transitive chains in combined_patch_map. When recursive splits + # run, round 1 may produce A→B and round 2 may produce B→C. Without + # flattening, apply_patch_strings (single-pass) would leave consumers of + # A pointing at the intermediate path B instead of the final path C. + if combined_patch_map: + changed = True + while changed: + changed = False + for k in list(combined_patch_map): + v = combined_patch_map[k] + if v in combined_patch_map and combined_patch_map[v] != v: + combined_patch_map[k] = combined_patch_map[v] + changed = True + + # ------------------------------------------------------------------ # + # Phase 4 — Update @patch strings after FileLimiter entity moves # + # ------------------------------------------------------------------ # + _patch_acc = RewriteAccumulator() + if ( + config.file_limiter_patch_update in ("basic", "rewrite") + and combined_patch_map + and repo_root + ): + _stats.patch_single_candidate += len(combined_patch_map) + # Update per_file sources still in memory (not yet written to disk). + for filepath, state in per_file.items(): + new_src = apply_patch_strings(state["source"], combined_patch_map) + if new_src != state["source"]: + state["source"] = new_src + state["msgs"].append( + f"{filepath}: patch_update: updated @patch strings" + ) + _stats.patch_update_edits += 1 + # Scan every other *.py file in the repo and update on disk. + per_file_abs = {str(Path(f).resolve()) for f in per_file} + repo_root_path = Path(repo_root) + for py_file in sorted(repo_root_path.rglob("*.py")): + if str(py_file.resolve()) in per_file_abs: + continue + if any( + part in _EXCLUDED_DIR_NAMES + for part in py_file.relative_to(repo_root_path).parts[:-1] + ): + continue + try: + src = py_file.read_text(encoding="utf-8") + except OSError: + continue + new_src = apply_patch_strings(src, combined_patch_map) + if new_src != src: + py_file.write_text(new_src, encoding="utf-8") + _stats.patch_update_edits += 1 + yield f"{py_file}: patch_update: updated @patch strings" + + _cg_candidates: Dict[str, Dict[str, Dict[str, List[str]]]] = {} + if config.file_limiter_patch_update in ("basic", "rewrite") and _fl_all_contexts: + for _cg_msg in apply_patch_callgraph( + _fl_all_contexts, + per_file, + repo_root, + verbose=verbose, + candidates_out=_cg_candidates, + config=config, + _acc=_patch_acc, + ): + _stats.patch_update_edits += 1 + yield _cg_msg + + if config.file_limiter_patch_update == "rewrite" and _fl_all_contexts: + yield from apply_patch_rewrite( + _fl_all_contexts, + per_file, + repo_root, + config, + verbose=verbose, + _acc=_patch_acc, + cg_candidates=_cg_candidates or None, + ) + _stats.patch_rewrite_llm_calls += _patch_acc.calls + if _patch_acc.elapsed > 0 or _patch_acc.input_tokens > 0: + _stats.record_llm_call( + _patch_acc.elapsed, + _patch_acc.input_tokens, + _patch_acc.output_tokens, + "file_limiter", + "patch_rewriter", + "", + ) + _stats.patch_update_edits += _patch_acc.files_updated + + _stats.patch_cg_resolved += _patch_acc.cg_resolved + _stats.patch_llm_no_change += _patch_acc.no_change + _stats.patch_llm_rename += _patch_acc.rename + _stats.patch_llm_rewrite += _patch_acc.rewrite + _stats.patch_edit_failures += _patch_acc.edit_failures + + # ------------------------------------------------------------------ # + # Write modified files and yield all messages # + # ------------------------------------------------------------------ # + for filepath, state in per_file.items(): + if state["source"] != state["original"]: + if state["source"]: + Path(filepath).write_text(state["source"], encoding="utf-8") + elif Path(filepath).name == "__init__.py": + # Keep __init__.py even when empty — it defines the package. + Path(filepath).write_text("", encoding="utf-8") + elif Path(filepath).exists(): + Path(filepath).unlink() + _stats.files_edited.append(filepath) + _stats.count_lines_changed(state["original"], state["source"]) + yield from state["msgs"] diff --git a/crispen/engine/file_limiter.py b/crispen/engine/file_limiter.py new file mode 100644 index 0000000..e77a1df --- /dev/null +++ b/crispen/engine/file_limiter.py @@ -0,0 +1,471 @@ +from pathlib import Path +from typing import Dict, List, Optional, Set, TYPE_CHECKING, Tuple +import ast +from ..stats import RunStats +from ..patch_rewriter import _FLContext + + +if TYPE_CHECKING: + from ..file_limiter.runner import FileLimiterResult + + +_PROJECT_MARKERS = frozenset({"pyproject.toml", "setup.py", "setup.cfg", ".git"}) + + +def _collect_top_level_names(source: str) -> Set[str]: + """Return all names defined or imported at the module top level of *source*. + + Covers functions, classes, module-level variable assignments, and all + import styles. Returns an empty set when *source* cannot be parsed. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + names: Set[str] = set() + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + elif isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + names.add(target.id) + elif isinstance(node, (ast.AugAssign, ast.AnnAssign)): + if isinstance(node.target, ast.Name): + names.add(node.target.id) + elif isinstance(node, ast.Import): + for alias in node.names: + names.add(alias.asname if alias.asname else alias.name.split(".")[-1]) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + if alias.name != "*": + names.add(alias.asname if alias.asname else alias.name) + return names + + +def _collect_assignment_names(source: str) -> Set[str]: + """Return names from top-level variable assignments in *source*. + + Covers ``X = …``, ``X: T = …``, and ``X += …`` where the target is a + plain ``ast.Name``. Functions, classes, and imports are excluded (they + are handled elsewhere). Returns an empty set on parse failure. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + names: Set[str] = set() + for node in tree.body: + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + names.add(target.id) + elif isinstance(node, (ast.AugAssign, ast.AnnAssign)): + if isinstance(node.target, ast.Name): + names.add(node.target.id) + return names + + +def _collect_imported_names(source: str) -> Set[str]: + """Return names imported at the top level of *source*. + + Handles ``import X``, ``import X as Y``, ``from X import Y``, and + ``from X import Y as Z``. Star imports (``from X import *``) are skipped. + Returns an empty set when *source* cannot be parsed. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + names: Set[str] = set() + for node in tree.body: + if isinstance(node, ast.Import): + for alias in node.names: + names.add(alias.asname if alias.asname else alias.name.split(".")[-1]) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + if alias.name != "*": + names.add(alias.asname if alias.asname else alias.name) + return names + + +def _collect_code_referenced_names(source: str) -> Set[str]: + """Return names referenced in code as *Load* expressions. + + Walks the AST looking for ``ast.Name`` nodes with ``Load`` context — + actual uses of a name in code (calls, attribute targets, right-hand-side + expressions). Pure re-export stubs (``from .sub import X # noqa: F401``) + produce no such nodes because the import alias itself is an ``ast.alias``, + not an ``ast.Name``. Returns an empty set on parse failure. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + return { + node.id + for node in ast.walk(tree) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) + } + + +def _module_path_for_file(file_path: str) -> Optional[str]: + """Return the dotted Python module path of *file_path*. + + Walks up from the file's directory to find the project root (the first + ancestor containing pyproject.toml, setup.py, setup.cfg, or .git), then + computes the dotted path relative to that root. + + ``__init__.py`` files are mapped to their package name (e.g. + ``mypkg/__init__.py`` → ``mypkg``) so that patch paths always use the + public package namespace rather than the internal ``.__init__`` segment. + + Returns ``None`` when the project root cannot be determined or when the + resolved path is not under the project root. + """ + abs_path = Path(file_path).resolve() + current = abs_path.parent + while True: + if any((current / m).exists() for m in _PROJECT_MARKERS): + rel = abs_path.relative_to(current) + module = ".".join(rel.with_suffix("").parts) + if module.endswith(".__init__"): + module = module[:-9] + return module + parent = current.parent + if parent == current: + return None + current = parent + + +def _redirect_inline_module_imports( + source: str, + old_mod: str, + name_to_new_mod: Dict[str, str], +) -> str: + """Replace ``from old_mod import names`` statements with new module paths. + + Scans all ``from import …`` statements at every level + (module-level and inside function/class bodies). Each imported name is + redirected to the module given by *name_to_new_mod*; names absent from the + map are kept pointing at *old_mod*. + + Returns *source* unchanged when there are no matching imports or when + *source* cannot be parsed as Python. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return source + lines = source.splitlines(keepends=True) + replacements: Dict[Tuple[int, int], str] = {} + + def _scan(stmts: list) -> None: + for node in stmts: + if ( + isinstance(node, ast.ImportFrom) + and node.level == 0 + and node.module == old_mod + ): + names = [alias.name for alias in node.names] + new_mod_to_names: Dict[str, List[str]] = {} + kept: List[str] = [] + for n in names: + dest = name_to_new_mod.get(n) + if dest: + new_mod_to_names.setdefault(dest, []).append(n) + else: + kept.append(n) + if not new_mod_to_names: + continue # No names moved; leave unchanged. + raw_line = lines[node.lineno - 1] + indent = raw_line[: len(raw_line) - len(raw_line.lstrip())] + parts: List[str] = [] + if kept: + parts.append(f"{indent}from {old_mod} import {', '.join(kept)}") + for dest_mod, dest_names in sorted(new_mod_to_names.items()): + joined = ", ".join(sorted(dest_names)) + parts.append(f"{indent}from {dest_mod} import {joined}") + replacements[(node.lineno, node.end_lineno)] = "\n".join(parts) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + _scan(node.body) + + _scan(tree.body) + if not replacements: + return source + + result = list(lines) + for (start, end), text in sorted(replacements.items(), reverse=True): + last_line = result[end - 1] + trailing = "\n" if last_line.endswith("\n") else "" + result[start - 1 : end] = [text + trailing] + return "".join(result) + + +def _patch_inline_imports_after_test_deletion( + deleted_path: str, + deleted_dir: Path, + new_files: Dict[str, str], + per_file: Dict, + fl_new_file_final: Dict[str, Optional[str]], +) -> None: + """Redirect inline imports pointing to a deleted test module. + + When a test file is deleted after a recursive subdir split (because all + entities migrated away, leaving an empty ``original_source``), any parent + file that received injected inline imports during the *first* split still + references the now-gone module. This function rewrites those stale imports + to point directly at the new sub-file locations. + + Updates both ``per_file`` states (written to disk later by the write loop) + and ``fl_new_file_final`` entries (already on disk; re-written immediately). + """ + old_mod = _module_path_for_file(deleted_path) + if old_mod is None: + return + + # Build name → new_module by parsing top-level definitions in new_files. + name_to_new_mod: Dict[str, str] = {} + for rel_path, content in new_files.items(): + new_abs = (deleted_dir / rel_path).resolve() + new_mod = _module_path_for_file(str(new_abs)) + if new_mod is None: + continue + try: + sub_tree = ast.parse(content) + except SyntaxError: + continue + for node in sub_tree.body: + if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + name_to_new_mod[node.name] = new_mod + + if not name_to_new_mod: + return + + for state in per_file.values(): + updated = _redirect_inline_module_imports( + state["source"], old_mod, name_to_new_mod + ) + if updated != state["source"]: + state["source"] = updated + + for path, content in list(fl_new_file_final.items()): + if not content: + continue + updated = _redirect_inline_module_imports(content, old_mod, name_to_new_mod) + if updated != content: + fl_new_file_final[path] = updated + Path(path).write_text(updated, encoding="utf-8") + + +def _build_patch_map( + filepath: str, + fl_result: "FileLimiterResult", + original_dir: Path, + pre_split_source: str = "", +) -> Dict[str, str]: + """Build old_dotted_path.EntityName → new_dotted_path.EntityName map. + + Uses "basic" mode logic: for each entity, maps to the single new file that + imports it (the "caller") rather than its definition file. If multiple new + files import the entity (forking), the entity is skipped. Import aliases + present in *pre_split_source* that appear in exactly one new file are also + included. + + Returns an empty dict when the module path cannot be determined or when no + entities were moved. + """ + old_module = _module_path_for_file(filepath) + if old_module is None: + return {} + + # Build import index: name → list of new-file rel_paths that import it. + # Build usage index: name → set of new-file rel_paths that reference it in + # code (ast.Name Load nodes — actual calls or expressions, not re-exports). + import_index: Dict[str, List[str]] = {} + usage_index: Dict[str, Set[str]] = {} + for rel_path, src in fl_result.new_files.items(): + if not src: + continue + for name in _collect_imported_names(src): + import_index.setdefault(name, []).append(rel_path) + for name in _collect_code_referenced_names(src): + usage_index.setdefault(name, set()).add(rel_path) + + patch_map: Dict[str, str] = {} + for entity_name, def_rel_target in fl_result.entity_to_target.items(): + # Callers = new files that both import this entity and reference it in + # code, excluding its definer. Pure re-export stubs import the name + # but produce no ast.Name Load nodes, so they are naturally excluded. + callers = [ + p + for p in import_index.get(entity_name, []) + if p != def_rel_target and p in usage_index.get(entity_name, set()) + ] + if len(callers) > 1: + continue # forking: multiple callers, skip + target_rel = callers[0] if callers else def_rel_target + new_module = _module_path_for_file(str(original_dir / target_rel)) + if new_module is None: + continue + patch_map[f"{old_module}.{entity_name}"] = f"{new_module}.{entity_name}" + + # Import aliases: names imported by the original file that appear in + # exactly one new sub-file (and are not already moved entities). + # Only count files that actually reference the alias in code, not stubs. + for alias_name in _collect_imported_names(pre_split_source): + if alias_name in fl_result.entity_to_target: + continue # already handled above + importers = [ + p + for p in import_index.get(alias_name, []) + if p in usage_index.get(alias_name, set()) + ] + if len(importers) != 1: + continue # zero or multiple: skip + new_module = _module_path_for_file(str(original_dir / importers[0])) + if new_module is None: + continue + patch_map[f"{old_module}.{alias_name}"] = f"{new_module}.{alias_name}" + + # Variable assignments: names assigned at the module level that were present + # in the original file and are not already tracked as entities or import + # aliases. Uses the same caller logic as named entities: 0 callers → only + # used in its defining file; 1 caller → migrated and consumed by exactly one + # other file; 2+ → forking. Only names from the original file are considered + # to avoid spurious entries for helper variables introduced by code generation. + # Names defined in multiple new files are skipped (ambiguous origin). + orig_assignments = _collect_assignment_names(pre_split_source) + def_index: Dict[str, List[str]] = {} + for rel_path, src in fl_result.new_files.items(): + if not src: + continue + for name in _collect_assignment_names(src): + if name in orig_assignments and name not in fl_result.entity_to_target: + def_index.setdefault(name, []).append(rel_path) + + for assign_name, def_rel_targets in def_index.items(): + if len(def_rel_targets) != 1: + continue # defined in multiple files → ambiguous + old_path = f"{old_module}.{assign_name}" + if old_path in patch_map: + continue # already handled by entity or import-alias section + def_rel_target = def_rel_targets[0] + callers = [ + p + for p in import_index.get(assign_name, []) + if p != def_rel_target and p in usage_index.get(assign_name, set()) + ] + if len(callers) > 1: + continue # forking: multiple consumers, skip + target_rel = callers[0] if callers else def_rel_target + new_module = _module_path_for_file(str(original_dir / target_rel)) + if new_module is None: + continue + patch_map[old_path] = f"{new_module}.{assign_name}" + + return patch_map + + +def _add_fl_context( + fl_all_contexts: List["_FLContext"], + filepath: str, + pre_split_src: str, + fl_result: "FileLimiterResult", + combined_patch_map: Dict[str, str], +) -> None: + """Append an _FLContext to *fl_all_contexts* for "rewrite" patch mode. + + Computes the forking old paths (entities in entity_to_target that basic + mode skipped because they appeared in multiple callers) and builds the + new module path map for all sub-files. Does nothing when no forking + entities exist or when the module path cannot be determined. + + When no forking entities exist but TOP_LEVEL blocks (_block_N) were + moved, also scans the new target files to find names that came from + those blocks (module-level vars, constants, imported aliases) but are + not individually tracked in entity_to_target. Each such name is added + as a specific old path (``old_module.name``) so the LLM can find any + ``with patch(old_module.name)`` calls without matching already-updated + paths like ``old_module.sub.name`` that basic mode already rewrote. + + Import aliases from the original file that basic mode skipped (forked + into multiple new sub-files) are also added so the LLM rewrite step + can determine the correct per-function patch target. + """ + old_mod = _module_path_for_file(filepath) + if old_mod is None: + return + forking_old_paths = { + f"{old_mod}.{name}" + for name in fl_result.entity_to_target + if f"{old_mod}.{name}" not in combined_patch_map + } + # Also collect names from moved _block_N entities that are NOT individually + # tracked (i.e., not in entity_to_target). These are block-internal names + # (vars, constants, imported aliases) that basic mode never maps, regardless + # of whether forking entities were also found above. + all_entity_names = set(fl_result.entity_to_target) + for entity_name, target_rel in fl_result.entity_to_target.items(): + if not entity_name.startswith("_block_"): + continue + new_src = fl_result.new_files.get(target_rel, "") + for name in _collect_top_level_names(new_src): + old_path = f"{old_mod}.{name}" + if name not in all_entity_names and old_path not in combined_patch_map: + forking_old_paths.add(old_path) + # Also add import aliases from the original file that basic mode skipped + # because they appeared in multiple new sub-files (forking). These + # aliases are absent from combined_patch_map but may still appear as + # @patch string targets in test files — the LLM rewrite step can resolve + # the correct sub-module for each test function individually. + for alias_name in _collect_imported_names(pre_split_src): + if alias_name in all_entity_names: + continue + old_path = f"{old_mod}.{alias_name}" + if old_path not in combined_patch_map: + forking_old_paths.add(old_path) + if not forking_old_paths: + return + orig_dir = Path(filepath).parent + new_mod_paths = { + rel: _module_path_for_file(str(orig_dir / rel)) or rel + for rel in fl_result.new_files + } + # For non-test subdir splits the original file stays on disk unchanged and + # fl_result.original_source is the pre-split source (runner.py restores it + # at line 704 so the original file is left untouched). The post-split + # module state lives in new_files["{subdir_name}/__init__.py"]. Use that + # as modified_source so _build_rename_guard_sets and the BFS terminal + # builder both see the correct set of names still present in the module. + init_key = f"{fl_result.subdir_name}/__init__.py" if fl_result.subdir_name else None + if init_key and init_key in fl_result.new_files: + modified_src = fl_result.new_files[init_key] or fl_result.original_source or "" + else: + modified_src = fl_result.original_source or "" + fl_all_contexts.append( + _FLContext( + filepath=filepath, + old_module=old_mod, + original_source=pre_split_src, + modified_source=modified_src, + new_files=dict(fl_result.new_files), + new_module_paths=new_mod_paths, + entity_to_target=dict(fl_result.entity_to_target), + forking_old_paths=forking_old_paths, + ) + ) + + +def _categorize_into_stats(stats: RunStats, msg: str) -> None: + """Increment the appropriate counter in *stats* for a raw change message.""" + if msg.startswith("IfNotElse:"): + stats.if_not_else += 1 + elif msg.startswith("TupleDataclass:"): + stats.tuple_to_dataclass += 1 + elif msg.startswith("DuplicateExtractor:") and "with call to" in msg: + stats.duplicate_matched += 1 + elif msg.startswith("DuplicateExtractor:"): + stats.duplicate_extracted += 1 + elif msg.startswith("split "): + stats.function_split += 1 diff --git a/crispen/engine/helpers.py b/crispen/engine/helpers.py new file mode 100644 index 0000000..ce8f68f --- /dev/null +++ b/crispen/engine/helpers.py @@ -0,0 +1,290 @@ +from pathlib import Path +from typing import Dict, List, NamedTuple, Optional, Set, Tuple +import ast +import os +import threading +import time +from libcst.metadata import FullRepoManager, MetadataWrapper, QualifiedNameProvider +import libcst as cst +from ..config import CrispenConfig +from ..errors import CrispenAPIError +from ..refactors.tuple_dataclass import TupleDataclass + + +def _should_run(name: str, config: CrispenConfig) -> bool: + """Return True if the named refactor should run given the config. + + When ``config.enabled_refactors`` is non-empty only names in that list run. + Otherwise names in ``config.disabled_refactors`` are skipped. + """ + if config.enabled_refactors: + return name in config.enabled_refactors + return name not in config.disabled_refactors + + +# Directory names excluded from the outside-caller scan (e.g. virtual environments). +_EXCLUDED_DIR_NAMES = frozenset( + {".venv", "venv", "env", ".tox", "__pycache__", "node_modules"} +) + +# Total wall-clock budget for all files in _find_outside_callers (seconds). +_SCOPE_ANALYSIS_TIMEOUT = 10 + + +def _has_callers_outside_ranges( + source: str, func_name: str, ranges: List[Tuple[int, int]] +) -> bool: + """Return True if func_name is called at any line outside the given ranges.""" + try: + tree = ast.parse(source) + except SyntaxError: + return False + for node in ast.walk(tree): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == func_name + ): + line = node.lineno + if not any(start <= line <= end for start, end in ranges): + return True + return False + + +def _blocked_private_scopes(source: str, ranges: List[Tuple[int, int]]) -> Set[str]: + """Return names of private functions that have callers outside the diff ranges.""" + try: + tree = ast.parse(source) + except SyntaxError: + return set() + blocked: Set[str] = set() + for node in ast.walk(tree): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id.startswith("_") + ): + line = node.lineno + if not any(start <= line <= end for start, end in ranges): + blocked.add(node.func.id) + return blocked + + +def _find_repo_root(changed: Dict[str, List]) -> Optional[str]: + """Find git repo root by searching parent directories for .git.""" + for filepath in changed.keys(): + p = Path(filepath).resolve().parent + while p != p.parent: + if (p / ".git").is_dir(): + return str(p) + p = p.parent + return None + + +def _file_to_module(repo_root: str, filepath: str) -> str: + """Convert an absolute file path to a dotted Python module name.""" + path = Path(filepath).resolve().relative_to(Path(repo_root).resolve()) + module = str(path.with_suffix("")).replace(os.sep, ".") + if module.endswith(".__init__"): + module = module[:-9] + return module + + +def _compute_qname(repo_root: str, filepath: str, func_name: str) -> str: + """Compute the qualified name of a function defined in filepath.""" + return f"{_file_to_module(repo_root, filepath)}.{func_name}" + + +def _build_alias_map(repo_root: str, canonical_qnames: Set[str]) -> Dict[str, str]: + """Map alias qualified names → canonical qualified names. + + Handles explicit re-exports like ``from .service import get_user`` in + ``pkg/__init__.py``, which creates the alias ``pkg.get_user`` for the + canonical name ``pkg.service.get_user``. + """ + alias_map: Dict[str, str] = {q: q for q in canonical_qnames} + + for init_path in Path(repo_root).rglob("__init__.py"): + pkg_parts = list(init_path.relative_to(repo_root).parts[:-1]) + pkg_qname = ".".join(pkg_parts) + + try: + source = init_path.read_text(encoding="utf-8") + tree = cst.parse_module(source) + except Exception: + continue + + for stmt in tree.body: + if not isinstance(stmt, cst.SimpleStatementLine): + continue + for s in stmt.body: + if not isinstance(s, cst.ImportFrom): + continue + if isinstance(s.names, cst.ImportStar) or not isinstance( + s.names, (list, tuple) + ): + continue + for al in s.names: + if not isinstance(al, cst.ImportAlias) or not isinstance( + al.name, cst.Name + ): + continue # pragma: no cover + func_name = al.name.value + alias_qname = f"{pkg_qname}.{func_name}" if pkg_qname else func_name + # Map this alias to a canonical qname if unambiguous + matches = [ + c for c in canonical_qnames if c.split(".")[-1] == func_name + ] + if len(matches) == 1: + alias_map[alias_qname] = matches[0] + + return alias_map + + +class _CallerFinder(cst.CSTVisitor): + """Visit a file and record which target qualified names are called.""" + + METADATA_DEPENDENCIES = (QualifiedNameProvider,) + + def __init__(self, target_qnames: Set[str]) -> None: + self.target_qnames = target_qnames + self.found: Set[str] = set() + + def visit_Call(self, node: cst.Call) -> None: + qnames = self.get_metadata(QualifiedNameProvider, node.func, set()) + for qn in qnames: + if qn.name in self.target_qnames: + self.found.add(qn.name) + + +def _visit_with_timeout(wrapper, finder, timeout: float) -> bool: + """Run wrapper.visit(finder) in a daemon thread with a wall-clock timeout. + + Returns True if the call completed within *timeout* seconds, False if it + timed out (libcst scope analysis can hang on large files). + """ + done = threading.Event() + + def _target(): + try: + wrapper.visit(finder) + finally: + done.set() + + t = threading.Thread(target=_target, daemon=True) + t.start() + return done.wait(timeout=timeout) + + +def _find_outside_callers( + repo_root: str, + target_qnames: Set[str], + diff_files: Set[str], +) -> Set[str]: + """Return the subset of *target_qnames* called in files outside *diff_files*.""" + if not target_qnames: + return set() + + repo_root_path = Path(repo_root) + outside_py = [ + p + for p in repo_root_path.rglob("*.py") + if str(p.resolve()) not in diff_files + and not any( + part in _EXCLUDED_DIR_NAMES + for part in p.relative_to(repo_root_path).parts[:-1] + ) + ] + if not outside_py: + return set() + + rel_paths = [str(p.relative_to(repo_root)) for p in outside_py] + + try: + manager = FullRepoManager(repo_root, rel_paths, {QualifiedNameProvider}) + except Exception: + # Can't build the manager → conservatively block all transforms. + return set(target_qnames) + + found_outside: Set[str] = set() + deadline = time.monotonic() + _SCOPE_ANALYSIS_TIMEOUT + for rel_path in rel_paths: + remaining = deadline - time.monotonic() + if remaining <= 0: + # Total budget exhausted: conservatively block all remaining. + found_outside.update(target_qnames) + break + try: + wrapper = manager.get_metadata_wrapper_for_path(rel_path) + finder = _CallerFinder(target_qnames) + if not _visit_with_timeout(wrapper, finder, remaining): + # This file timed out: conservatively block all transforms. + found_outside.update(target_qnames) + break + found_outside.update(finder.found) + except Exception: + continue + + return found_outside + + +class _ApplyResult(NamedTuple): + """Return type of _apply_tuple_dataclass.""" + + source: str + msgs: List[str] + td: Optional[TupleDataclass] + + +def _apply_tuple_dataclass( + filepath: str, + ranges: List[Tuple[int, int]], + source: str, + verbose: bool, + approved_public_funcs: Set[str], + min_size: int = 4, + blocked_scopes: Optional[Set[str]] = None, +) -> "_ApplyResult": + """Run TupleDataclass on *source*. Returns (new_source, messages, transformer).""" + try: + tree = cst.parse_module(source) + except cst.ParserSyntaxError as exc: + return _ApplyResult( + source, [f"SKIP {filepath} (TupleDataclass): parse error: {exc}"], None + ) + + wrapper = MetadataWrapper(tree) + try: + td = TupleDataclass( + ranges, + source=source, + verbose=verbose, + approved_public_funcs=approved_public_funcs, + min_size=min_size, + blocked_scopes=blocked_scopes, + ) + new_tree = wrapper.visit(td) + except CrispenAPIError: + raise + except Exception as exc: + return _ApplyResult( + source, + [f"SKIP {filepath} (TupleDataclass): transform error: {exc}"], + None, + ) + + new_source = td.get_rewritten_source() or new_tree.code + if new_source == source: + return _ApplyResult(source, [], td) + + try: + compile(new_source, filepath, "exec") + except SyntaxError as exc: # pragma: no cover + return _ApplyResult( + source, + [f"SKIP {filepath} (TupleDataclass): output not valid Python: {exc}"], + td, + ) + + msgs = [f"{filepath}: {m}" for m in td.get_changes()] + return _ApplyResult(new_source, msgs, td) diff --git a/crispen/file_limiter/advisor/__init__.py b/crispen/file_limiter/advisor/__init__.py new file mode 100644 index 0000000..6fd1656 --- /dev/null +++ b/crispen/file_limiter/advisor/__init__.py @@ -0,0 +1,159 @@ +"""LLM advisor for FileLimiter: plans entity migration to new files.""" + +from __future__ import annotations + +from pathlib import Path +from typing import List, Optional + +from ...config import CrispenConfig +from ...llm_client import get_api_key, make_client +from ..classifier import ClassifiedEntities +from .helpers import _advise_set3 # fmt: skip # noqa: F401, E501 +from .helpers import _build_group_mermaid # fmt: skip # noqa: F401, E501 +from .helpers import _compute_projected_lines # fmt: skip # noqa: F401, E501 +from .helpers import _group_summary # fmt: skip # noqa: F401, E501 +from .models import FileLimiterPlan # fmt: skip # noqa: F401, E501 +from .models import GroupPlacement # fmt: skip # noqa: F401, E501 +from .models import _LLMAccumulator # fmt: skip # noqa: F401, E501 +from .models import _PLACEMENT_CHUNK_SIZE # fmt: skip # noqa: F401, E501 +from .placement import _assign_placements +from .placement import _assign_placements_chunk # fmt: skip # noqa: F401, E501 +from .placement import _find_conflicting_placement_indices # fmt: skip # noqa: F401, E501 +from .placement import _propose_files_step # fmt: skip # noqa: F401, E501 +from .placement import _refine_merge_tiny # fmt: skip # noqa: F401, E501 +from .placement import resolve_naming_conflicts # fmt: skip # noqa: F401, E501 + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def advise_file_limiter( + classified: ClassifiedEntities, + original_path: str, + config: CrispenConfig, + existing_files: frozenset = frozenset(), + prev_set3_failure: str = "", + prev_placement_failure: str = "", + verbose: bool = False, + timing: str = "detailed", + subdir_name: Optional[str] = None, +) -> FileLimiterPlan: + """Ask the LLM to plan entity placement across new files. + + Returns a :class:`FileLimiterPlan` with ``abort=True`` when planning fails + or the file cannot be split (e.g. single SCC covering all entities). + """ + if classified.abort: + return FileLimiterPlan( + set3_migrate=[], + placements=[], + abort=True, + abort_reason=classified.abort_reason, + ) + + if not classified.set_2_groups and not classified.set_3_groups: + return FileLimiterPlan(set3_migrate=[], placements=[], abort=False) + + api_key = get_api_key(config.provider, caller="FileLimiter") + client = make_client( + config.provider, api_key, timeout=config.api_timeout, base_url=config.base_url + ) + + counter = _LLMAccumulator() + + # Call 1: advise Set 3 groups (only if set_3 is non-empty). + # In a test-file subdir split every group must migrate — there is no public + # API contract that forces anything to stay, and leaving test functions + # behind in the original causes fixture-visibility problems. Skip the LLM + # call and treat all set-3 groups as migrating. + is_test_subdir = subdir_name is not None and Path(original_path).name.startswith( + "test_" + ) + set3_migrate: List[List[str]] = [] + if classified.set_3_groups: + if is_test_subdir: + set3_migrate = classified.set_3_groups + else: + result = _advise_set3( + classified, + original_path, + client, + config, + prev_failure=prev_set3_failure, + verbose=verbose, + timing=timing, + _acc=counter, + ) + if result is None: + return FileLimiterPlan( + set3_migrate=[], + placements=[], + abort=True, + abort_reason="LLM failed to plan set-3 groups", + llm_calls=counter.calls, + llm_elapsed=counter.elapsed, + llm_input_tokens=counter.input_tokens, + llm_output_tokens=counter.output_tokens, + ) + set3_migrate = result + + # Calls 2+: propose files, assign groups, refine (merge tiny). + groups_to_place = classified.set_2_groups + set3_migrate + if not groups_to_place: + return FileLimiterPlan( + set3_migrate=set3_migrate, + placements=[], + abort=False, + llm_calls=counter.calls, + llm_elapsed=counter.elapsed, + llm_input_tokens=counter.input_tokens, + llm_output_tokens=counter.output_tokens, + ) + + entity_map = {e.name: e for e in classified.entities} + total_lines = sum( + entity_map[name].end_line - entity_map[name].start_line + 1 + for group in groups_to_place + for name in group + if name in entity_map + ) + original_target = ( + max(2, -(-(2 * total_lines) // config.max_file_lines)) if total_lines > 0 else 2 + ) + placements = _assign_placements( + groups_to_place, + classified, + original_path, + existing_files, + client, + config, + prev_failure=prev_placement_failure, + verbose=verbose, + timing=timing, + _acc=counter, + subdir_name=subdir_name, + target_files=original_target, + ) + if placements is None: + return FileLimiterPlan( + set3_migrate=set3_migrate, + placements=[], + abort=True, + abort_reason="LLM failed to assign file placements", + llm_calls=counter.calls, + llm_elapsed=counter.elapsed, + llm_input_tokens=counter.input_tokens, + llm_output_tokens=counter.output_tokens, + ) + + return FileLimiterPlan( + set3_migrate=set3_migrate, + placements=placements, + abort=False, + llm_calls=counter.calls, + llm_elapsed=counter.elapsed, + llm_input_tokens=counter.input_tokens, + llm_output_tokens=counter.output_tokens, + ) diff --git a/crispen/file_limiter/advisor/helpers.py b/crispen/file_limiter/advisor/helpers.py new file mode 100644 index 0000000..0740c4d --- /dev/null +++ b/crispen/file_limiter/advisor/helpers.py @@ -0,0 +1,175 @@ +from __future__ import annotations +from typing import Dict, List, Optional, TYPE_CHECKING +import sys +from ...config import CrispenConfig +from ...llm_client import call_with_tool +from ..classifier import ClassifiedEntities +from ..entity_parser import Entity +from .models import GroupPlacement, _SET3_TOOL + + +if TYPE_CHECKING: + from .models import _LLMAccumulator + + +def _group_summary(group: List[str], entity_map: Dict[str, Entity]) -> str: + """Return a brief text description of an SCC group for LLM context.""" + parts = [] + for name in group: + ent = entity_map.get(name) + if ent: + size = ent.end_line - ent.start_line + 1 + desc = f"{name} ({size} lines)" + extras = [] + if ent.section_header: + extras.append(f'section: "{ent.section_header}"') + if ent.docstring: + flat = ent.docstring.replace("\n", " ") + idx = flat.find(". ") + first = flat[: idx + 1] if idx >= 0 else flat + extras.append(f'"{first}"') + if ent.params: + extras.append(f"params: {', '.join(ent.params)}") + if extras: + desc += " \u2014 " + "; ".join(extras) + parts.append(desc) + else: + parts.append(name) + return ", ".join(parts) + + +def _build_group_mermaid(chunk: List[List[str]], classified: ClassifiedEntities) -> str: + """Return a Mermaid graph showing inter-group dependencies, or '' if none.""" + entity_to_gid = {name: gid for gid, group in enumerate(chunk) for name in group} + edges: set = set() + for gid, group in enumerate(chunk): + for entity_name in group: + for dep_name in classified.graph.get(entity_name, set()): + dep_gid = entity_to_gid.get(dep_name) + if dep_gid is not None and dep_gid != gid: + edges.add((gid, dep_gid)) + if not edges: + return "" + lines = ["```mermaid", "graph TD"] + for g_from, g_to in sorted(edges): + lines.append(f" G{g_from} --> G{g_to}") + lines.append("```") + return "\n".join(lines) + + +def _compute_projected_lines( + placements: List[GroupPlacement], + entity_map: Dict[str, Entity], +) -> Dict[str, int]: + """Return projected line count per target filename.""" + projected: Dict[str, int] = {} + for p in placements: + for name in p.group: + ent = entity_map.get(name) + if ent: + size = ent.end_line - ent.start_line + 1 + projected[p.target_file] = projected.get(p.target_file, 0) + size + return projected + + +def _advise_set3( + classified: ClassifiedEntities, + original_path: str, + client: object, + config: CrispenConfig, + prev_failure: str = "", + verbose: bool = False, + timing: str = "detailed", + _acc: Optional["_LLMAccumulator"] = None, +) -> Optional[List[List[str]]]: + """Ask the LLM which Set 3 groups should migrate. Returns None on failure.""" + entity_map = {e.name: e for e in classified.entities} + group_lines = [] + for idx, group in enumerate(classified.set_3_groups): + summary = _group_summary(group, entity_map) + group_lines.append(f" [{idx}]: {summary}") + groups_text = "\n".join(group_lines) + + mermaid_text = _build_group_mermaid(classified.set_3_groups, classified) + n_groups = len(classified.set_3_groups) + content = ( + f"The file '{original_path}' is over the maximum line limit and MUST " + "be reduced in size by splitting it. The following entity groups are " + "MODIFIED (they existed before the diff and were changed by the " + "current diff). Each group is a mutual dependency cycle and must be " + "moved as an indivisible unit — it cannot be split further.\n\n" + f"Groups:\n{groups_text}\n\n" + "IMPORTANT: 'migrate' is the preferred action. The goal is to move " + "as many groups as possible to new files so the original file shrinks " + "below the line limit. Choose 'stay' ONLY if there is a compelling " + "reason the group cannot be extracted (for example, it is the sole " + "public API entry-point of the module and callers import it by name " + "from this specific file). If ALL groups stay, no split will occur " + "and the file will remain over the limit, which is not acceptable.\n\n" + "CIRCULAR IMPORT CONSTRAINT: A migrated group cannot safely reference " + "names defined by groups that stay in the original — this creates a " + "circular import (the new file imports from the original while the " + "original also imports from the new file). You cannot migrate group A " + "if any group that A depends on is staying. To migrate A, either also " + "migrate everything A depends on (to the same or a compatible file), " + "or keep A in the original. Migrating a dependency while leaving the " + "dependent in the original is always safe. Groups with no outgoing " + "arrows in the dependency graph (leaf groups) can always be migrated " + "independently.\n\n" + "For each group, return 'migrate' (preferred) or 'stay' (exceptional)." + ) + if mermaid_text: + content += f"\n\nDependency graph between groups:\n{mermaid_text}" + if prev_failure: + content += f"\n\nFeedback from the previous attempt: {prev_failure}" + messages = [{"role": "user", "content": content}] + max_tokens = max(512, 20 + n_groups * 25) + if verbose: + print( + f"crispen: FileLimiter: asking LLM whether to migrate" + f" {n_groups} set-3 group(s) in '{original_path}'", + file=sys.stderr, + flush=True, + ) + if _acc is not None: + _acc.calls += 1 + result = call_with_tool( + client, + config.provider, + config.model, + max_tokens, + _SET3_TOOL, + "advise_set3_actions", + messages, + caller="FileLimiter", + tool_choice_override=config.tool_choice, + rate_limit_retries=config.rate_limit_retries, + rate_limit_backoff=config.rate_limit_backoff, + ) + if _acc is not None: + _acc.elapsed += result.elapsed + _acc.input_tokens += result.input_tokens + _acc.output_tokens += result.output_tokens + if verbose and timing == "detailed": + print( + f"crispen: FileLimiter: → done [{result.elapsed:.2f}s," + f" {result.input_tokens:,} in / {result.output_tokens:,} out]", + file=sys.stderr, + flush=True, + ) + if result.tool_input is None: + return None + + migrate_ids = set() + for decision in result.tool_input.get("decisions", []): + gid = decision.get("group_id") + action = decision.get("action") + if isinstance(gid, int) and 0 <= gid < len(classified.set_3_groups): + if action == "migrate": + migrate_ids.add(gid) + + return [ + classified.set_3_groups[i] + for i in range(len(classified.set_3_groups)) + if i in migrate_ids + ] diff --git a/crispen/file_limiter/advisor/models.py b/crispen/file_limiter/advisor/models.py new file mode 100644 index 0000000..d8128c0 --- /dev/null +++ b/crispen/file_limiter/advisor/models.py @@ -0,0 +1,172 @@ +from __future__ import annotations +from dataclasses import dataclass +from typing import List + + +@dataclass +class _LLMAccumulator: + """Mutable accumulator for LLM call counts, timing, and token usage.""" + + calls: int = 0 + elapsed: float = 0.0 + input_tokens: int = 0 + output_tokens: int = 0 + + +@dataclass +class GroupPlacement: + """Placement decision for one SCC group.""" + + group: List[str] # entity names in the SCC + target_file: str # relative filename (e.g. "utils.py") + + +@dataclass +class FileLimiterPlan: + """Complete placement plan from the LLM advisor.""" + + # Set 3 groups the LLM chose to migrate (rest stay in original file). + set3_migrate: List[List[str]] + # Placement for set_2 groups + migrating set_3 groups. + placements: List[GroupPlacement] + # True if planning failed and the file should not be split. + abort: bool + abort_reason: str = "" # human-readable explanation when abort=True + llm_calls: int = 0 # number of LLM API calls made during planning + llm_elapsed: float = 0.0 + llm_input_tokens: int = 0 + llm_output_tokens: int = 0 + + +_SET3_TOOL: dict = { + "name": "advise_set3_actions", + "description": ( + "For each modified-entity group, decide whether to migrate it to a new " + "file or leave it in the original file." + ), + "input_schema": { + "type": "object", + "properties": { + "decisions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "group_id": { + "type": "integer", + "description": "0-based index of the group", + }, + "action": { + "type": "string", + "enum": ["migrate", "stay"], + "description": ( + "'migrate' to move to a new file, " + "'stay' to keep in original" + ), + }, + }, + "required": ["group_id", "action"], + }, + } + }, + "required": ["decisions"], + }, +} + +_PLACEMENT_TOOL: dict = { + "name": "assign_file_placements", + "description": ( + "Assign each entity group to a target Python filename. " + "Each group will be written to a new file in the same directory " + "as the original." + ), + "input_schema": { + "type": "object", + "properties": { + "placements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "group_id": { + "type": "integer", + "description": "0-based index of the group", + }, + "target_file": { + "type": "string", + "description": ( + "Relative filename, e.g. 'utils.py' or " + "'helpers/io.py'" + ), + }, + }, + "required": ["group_id", "target_file"], + }, + } + }, + "required": ["placements"], + }, +} + +_RENAME_CONFLICTS_TOOL: dict = { + "name": "rename_conflicting_placements", + "description": ( + "Assign new, non-conflicting target filenames to entity groups " + "that currently have naming conflicts." + ), + "input_schema": { + "type": "object", + "properties": { + "placements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "group_id": {"type": "integer"}, + "target_file": {"type": "string"}, + }, + "required": ["group_id", "target_file"], + }, + } + }, + "required": ["placements"], + }, +} + +_PROPOSE_FILES_TOOL: dict = { + "name": "propose_output_files", + "description": ( + "Propose the set of Python files to create when splitting a large module. " + "Return exactly the files you plan to use, with descriptive names and " + "a brief summary of what each file will contain." + ), + "input_schema": { + "type": "object", + "properties": { + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "filename": { + "type": "string", + "description": "Python filename, e.g. 'utils.py'", + }, + "description": { + "type": "string", + "description": "What this file will contain", + }, + }, + "required": ["filename", "description"], + }, + } + }, + "required": ["files"], + }, +} + + +# Maximum number of groups per placement LLM call. Large files may have +# dozens of groups; sending them all in one call frequently causes timeouts +# or incomplete responses. This limit keeps each call small and reliable. +_PLACEMENT_CHUNK_SIZE = 100 diff --git a/crispen/file_limiter/advisor.py b/crispen/file_limiter/advisor/placement.py similarity index 63% rename from crispen/file_limiter/advisor.py rename to crispen/file_limiter/advisor/placement.py index ff86e871..ecfb7bd 100644 --- a/crispen/file_limiter/advisor.py +++ b/crispen/file_limiter/advisor/placement.py @@ -1,363 +1,22 @@ -"""LLM advisor for FileLimiter: plans entity migration to new files.""" - from __future__ import annotations - -import sys -from dataclasses import dataclass from pathlib import Path -from typing import Dict, List, Optional, Tuple - -from ..config import CrispenConfig -from ..llm_client import call_with_tool, get_api_key, make_client -from .classifier import ClassifiedEntities -from .entity_parser import Entity - - -# --------------------------------------------------------------------------- -# Public data classes -# --------------------------------------------------------------------------- - - -@dataclass -class _LLMAccumulator: - """Mutable accumulator for LLM call counts, timing, and token usage.""" - - calls: int = 0 - elapsed: float = 0.0 - input_tokens: int = 0 - output_tokens: int = 0 - - -@dataclass -class GroupPlacement: - """Placement decision for one SCC group.""" - - group: List[str] # entity names in the SCC - target_file: str # relative filename (e.g. "utils.py") - - -@dataclass -class FileLimiterPlan: - """Complete placement plan from the LLM advisor.""" - - # Set 3 groups the LLM chose to migrate (rest stay in original file). - set3_migrate: List[List[str]] - # Placement for set_2 groups + migrating set_3 groups. - placements: List[GroupPlacement] - # True if planning failed and the file should not be split. - abort: bool - abort_reason: str = "" # human-readable explanation when abort=True - llm_calls: int = 0 # number of LLM API calls made during planning - llm_elapsed: float = 0.0 - llm_input_tokens: int = 0 - llm_output_tokens: int = 0 - - -# --------------------------------------------------------------------------- -# LLM tool schemas -# --------------------------------------------------------------------------- - - -_SET3_TOOL: dict = { - "name": "advise_set3_actions", - "description": ( - "For each modified-entity group, decide whether to migrate it to a new " - "file or leave it in the original file." - ), - "input_schema": { - "type": "object", - "properties": { - "decisions": { - "type": "array", - "items": { - "type": "object", - "properties": { - "group_id": { - "type": "integer", - "description": "0-based index of the group", - }, - "action": { - "type": "string", - "enum": ["migrate", "stay"], - "description": ( - "'migrate' to move to a new file, " - "'stay' to keep in original" - ), - }, - }, - "required": ["group_id", "action"], - }, - } - }, - "required": ["decisions"], - }, -} - -_PLACEMENT_TOOL: dict = { - "name": "assign_file_placements", - "description": ( - "Assign each entity group to a target Python filename. " - "Each group will be written to a new file in the same directory " - "as the original." - ), - "input_schema": { - "type": "object", - "properties": { - "placements": { - "type": "array", - "items": { - "type": "object", - "properties": { - "group_id": { - "type": "integer", - "description": "0-based index of the group", - }, - "target_file": { - "type": "string", - "description": ( - "Relative filename, e.g. 'utils.py' or " - "'helpers/io.py'" - ), - }, - }, - "required": ["group_id", "target_file"], - }, - } - }, - "required": ["placements"], - }, -} - -_RENAME_CONFLICTS_TOOL: dict = { - "name": "rename_conflicting_placements", - "description": ( - "Assign new, non-conflicting target filenames to entity groups " - "that currently have naming conflicts." - ), - "input_schema": { - "type": "object", - "properties": { - "placements": { - "type": "array", - "items": { - "type": "object", - "properties": { - "group_id": {"type": "integer"}, - "target_file": {"type": "string"}, - }, - "required": ["group_id", "target_file"], - }, - } - }, - "required": ["placements"], - }, -} - -_PROPOSE_FILES_TOOL: dict = { - "name": "propose_output_files", - "description": ( - "Propose the set of Python files to create when splitting a large module. " - "Return exactly the files you plan to use, with descriptive names and " - "a brief summary of what each file will contain." - ), - "input_schema": { - "type": "object", - "properties": { - "files": { - "type": "array", - "items": { - "type": "object", - "properties": { - "filename": { - "type": "string", - "description": "Python filename, e.g. 'utils.py'", - }, - "description": { - "type": "string", - "description": "What this file will contain", - }, - }, - "required": ["filename", "description"], - }, - } - }, - "required": ["files"], - }, -} - - -# Maximum number of groups per placement LLM call. Large files may have -# dozens of groups; sending them all in one call frequently causes timeouts -# or incomplete responses. This limit keeps each call small and reliable. -_PLACEMENT_CHUNK_SIZE = 100 - - -# --------------------------------------------------------------------------- -# Internal helpers -# --------------------------------------------------------------------------- - - -def _group_summary(group: List[str], entity_map: Dict[str, Entity]) -> str: - """Return a brief text description of an SCC group for LLM context.""" - parts = [] - for name in group: - ent = entity_map.get(name) - if ent: - size = ent.end_line - ent.start_line + 1 - desc = f"{name} ({size} lines)" - extras = [] - if ent.section_header: - extras.append(f'section: "{ent.section_header}"') - if ent.docstring: - flat = ent.docstring.replace("\n", " ") - idx = flat.find(". ") - first = flat[: idx + 1] if idx >= 0 else flat - extras.append(f'"{first}"') - if ent.params: - extras.append(f"params: {', '.join(ent.params)}") - if extras: - desc += " \u2014 " + "; ".join(extras) - parts.append(desc) - else: - parts.append(name) - return ", ".join(parts) - - -def _build_group_mermaid(chunk: List[List[str]], classified: ClassifiedEntities) -> str: - """Return a Mermaid graph showing inter-group dependencies, or '' if none.""" - entity_to_gid = {name: gid for gid, group in enumerate(chunk) for name in group} - edges: set = set() - for gid, group in enumerate(chunk): - for entity_name in group: - for dep_name in classified.graph.get(entity_name, set()): - dep_gid = entity_to_gid.get(dep_name) - if dep_gid is not None and dep_gid != gid: - edges.add((gid, dep_gid)) - if not edges: - return "" - lines = ["```mermaid", "graph TD"] - for g_from, g_to in sorted(edges): - lines.append(f" G{g_from} --> G{g_to}") - lines.append("```") - return "\n".join(lines) - - -def _compute_projected_lines( - placements: List[GroupPlacement], - entity_map: Dict[str, Entity], -) -> Dict[str, int]: - """Return projected line count per target filename.""" - projected: Dict[str, int] = {} - for p in placements: - for name in p.group: - ent = entity_map.get(name) - if ent: - size = ent.end_line - ent.start_line + 1 - projected[p.target_file] = projected.get(p.target_file, 0) + size - return projected - - -def _advise_set3( - classified: ClassifiedEntities, - original_path: str, - client: object, - config: CrispenConfig, - prev_failure: str = "", - verbose: bool = False, - timing: str = "detailed", - _acc: Optional["_LLMAccumulator"] = None, -) -> Optional[List[List[str]]]: - """Ask the LLM which Set 3 groups should migrate. Returns None on failure.""" - entity_map = {e.name: e for e in classified.entities} - group_lines = [] - for idx, group in enumerate(classified.set_3_groups): - summary = _group_summary(group, entity_map) - group_lines.append(f" [{idx}]: {summary}") - groups_text = "\n".join(group_lines) - - mermaid_text = _build_group_mermaid(classified.set_3_groups, classified) - n_groups = len(classified.set_3_groups) - content = ( - f"The file '{original_path}' is over the maximum line limit and MUST " - "be reduced in size by splitting it. The following entity groups are " - "MODIFIED (they existed before the diff and were changed by the " - "current diff). Each group is a mutual dependency cycle and must be " - "moved as an indivisible unit — it cannot be split further.\n\n" - f"Groups:\n{groups_text}\n\n" - "IMPORTANT: 'migrate' is the preferred action. The goal is to move " - "as many groups as possible to new files so the original file shrinks " - "below the line limit. Choose 'stay' ONLY if there is a compelling " - "reason the group cannot be extracted (for example, it is the sole " - "public API entry-point of the module and callers import it by name " - "from this specific file). If ALL groups stay, no split will occur " - "and the file will remain over the limit, which is not acceptable.\n\n" - "CIRCULAR IMPORT CONSTRAINT: A migrated group cannot safely reference " - "names defined by groups that stay in the original — this creates a " - "circular import (the new file imports from the original while the " - "original also imports from the new file). You cannot migrate group A " - "if any group that A depends on is staying. To migrate A, either also " - "migrate everything A depends on (to the same or a compatible file), " - "or keep A in the original. Migrating a dependency while leaving the " - "dependent in the original is always safe. Groups with no outgoing " - "arrows in the dependency graph (leaf groups) can always be migrated " - "independently.\n\n" - "For each group, return 'migrate' (preferred) or 'stay' (exceptional)." - ) - if mermaid_text: - content += f"\n\nDependency graph between groups:\n{mermaid_text}" - if prev_failure: - content += f"\n\nFeedback from the previous attempt: {prev_failure}" - messages = [{"role": "user", "content": content}] - max_tokens = max(512, 20 + n_groups * 25) - if verbose: - print( - f"crispen: FileLimiter: asking LLM whether to migrate" - f" {n_groups} set-3 group(s) in '{original_path}'", - file=sys.stderr, - flush=True, - ) - if _acc is not None: - _acc.calls += 1 - result = call_with_tool( - client, - config.provider, - config.model, - max_tokens, - _SET3_TOOL, - "advise_set3_actions", - messages, - caller="FileLimiter", - tool_choice_override=config.tool_choice, - rate_limit_retries=config.rate_limit_retries, - rate_limit_backoff=config.rate_limit_backoff, - ) - if _acc is not None: - _acc.elapsed += result.elapsed - _acc.input_tokens += result.input_tokens - _acc.output_tokens += result.output_tokens - if verbose and timing == "detailed": - print( - f"crispen: FileLimiter: → done [{result.elapsed:.2f}s," - f" {result.input_tokens:,} in / {result.output_tokens:,} out]", - file=sys.stderr, - flush=True, - ) - if result.tool_input is None: - return None +from typing import Dict, List, Optional, TYPE_CHECKING, Tuple +import sys +from ...config import CrispenConfig +from ...llm_client import call_with_tool, get_api_key, make_client +from ..classifier import ClassifiedEntities +from .helpers import _build_group_mermaid, _compute_projected_lines, _group_summary +from .models import ( + GroupPlacement, + _PLACEMENT_CHUNK_SIZE, + _PLACEMENT_TOOL, + _PROPOSE_FILES_TOOL, + _RENAME_CONFLICTS_TOOL, +) - migrate_ids = set() - for decision in result.tool_input.get("decisions", []): - gid = decision.get("group_id") - action = decision.get("action") - if isinstance(gid, int) and 0 <= gid < len(classified.set_3_groups): - if action == "migrate": - migrate_ids.add(gid) - return [ - classified.set_3_groups[i] - for i in range(len(classified.set_3_groups)) - if i in migrate_ids - ] +if TYPE_CHECKING: + from .models import _LLMAccumulator def _propose_files_step( @@ -1062,11 +721,6 @@ def _assign_placements( return all_placements -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - - def resolve_naming_conflicts( placements: List[GroupPlacement], classified: ClassifiedEntities, @@ -1146,133 +800,3 @@ def resolve_naming_conflicts( for renamed, idx in zip(all_renamed, conflict_idxs): result[idx] = renamed return result - - -def advise_file_limiter( - classified: ClassifiedEntities, - original_path: str, - config: CrispenConfig, - existing_files: frozenset = frozenset(), - prev_set3_failure: str = "", - prev_placement_failure: str = "", - verbose: bool = False, - timing: str = "detailed", - subdir_name: Optional[str] = None, -) -> FileLimiterPlan: - """Ask the LLM to plan entity placement across new files. - - Returns a :class:`FileLimiterPlan` with ``abort=True`` when planning fails - or the file cannot be split (e.g. single SCC covering all entities). - """ - if classified.abort: - return FileLimiterPlan( - set3_migrate=[], - placements=[], - abort=True, - abort_reason=classified.abort_reason, - ) - - if not classified.set_2_groups and not classified.set_3_groups: - return FileLimiterPlan(set3_migrate=[], placements=[], abort=False) - - api_key = get_api_key(config.provider, caller="FileLimiter") - client = make_client( - config.provider, api_key, timeout=config.api_timeout, base_url=config.base_url - ) - - counter = _LLMAccumulator() - - # Call 1: advise Set 3 groups (only if set_3 is non-empty). - # In a test-file subdir split every group must migrate — there is no public - # API contract that forces anything to stay, and leaving test functions - # behind in the original causes fixture-visibility problems. Skip the LLM - # call and treat all set-3 groups as migrating. - is_test_subdir = subdir_name is not None and Path(original_path).name.startswith( - "test_" - ) - set3_migrate: List[List[str]] = [] - if classified.set_3_groups: - if is_test_subdir: - set3_migrate = classified.set_3_groups - else: - result = _advise_set3( - classified, - original_path, - client, - config, - prev_failure=prev_set3_failure, - verbose=verbose, - timing=timing, - _acc=counter, - ) - if result is None: - return FileLimiterPlan( - set3_migrate=[], - placements=[], - abort=True, - abort_reason="LLM failed to plan set-3 groups", - llm_calls=counter.calls, - llm_elapsed=counter.elapsed, - llm_input_tokens=counter.input_tokens, - llm_output_tokens=counter.output_tokens, - ) - set3_migrate = result - - # Calls 2+: propose files, assign groups, refine (merge tiny). - groups_to_place = classified.set_2_groups + set3_migrate - if not groups_to_place: - return FileLimiterPlan( - set3_migrate=set3_migrate, - placements=[], - abort=False, - llm_calls=counter.calls, - llm_elapsed=counter.elapsed, - llm_input_tokens=counter.input_tokens, - llm_output_tokens=counter.output_tokens, - ) - - entity_map = {e.name: e for e in classified.entities} - total_lines = sum( - entity_map[name].end_line - entity_map[name].start_line + 1 - for group in groups_to_place - for name in group - if name in entity_map - ) - original_target = ( - max(2, -(-(2 * total_lines) // config.max_file_lines)) if total_lines > 0 else 2 - ) - placements = _assign_placements( - groups_to_place, - classified, - original_path, - existing_files, - client, - config, - prev_failure=prev_placement_failure, - verbose=verbose, - timing=timing, - _acc=counter, - subdir_name=subdir_name, - target_files=original_target, - ) - if placements is None: - return FileLimiterPlan( - set3_migrate=set3_migrate, - placements=[], - abort=True, - abort_reason="LLM failed to assign file placements", - llm_calls=counter.calls, - llm_elapsed=counter.elapsed, - llm_input_tokens=counter.input_tokens, - llm_output_tokens=counter.output_tokens, - ) - - return FileLimiterPlan( - set3_migrate=set3_migrate, - placements=placements, - abort=False, - llm_calls=counter.calls, - llm_elapsed=counter.elapsed, - llm_input_tokens=counter.input_tokens, - llm_output_tokens=counter.output_tokens, - ) diff --git a/crispen/file_limiter/code_gen.py b/crispen/file_limiter/code_gen.py deleted file mode 100644 index fc35e7f..0000000 --- a/crispen/file_limiter/code_gen.py +++ /dev/null @@ -1,3055 +0,0 @@ -"""Code generation for FileLimiter: build new files and update original source.""" - -from __future__ import annotations - -import ast -import io -import re -import tokenize -from dataclasses import dataclass, field -from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple - -from ..import_sort import _sort_imports_pep8 -from .advisor import FileLimiterPlan, GroupPlacement -from .classifier import ClassifiedEntities -from .dep_graph import find_sccs -from .entity_parser import Entity, EntityKind, _parse_section_headers - - -# --------------------------------------------------------------------------- -# Public data classes -# --------------------------------------------------------------------------- - - -@dataclass -class ImportInfo: - """A top-level import statement and the names it introduces.""" - - names: List[str] # names made available by this import - source: str # the import statement text (no trailing newline) - is_future: bool # True if `from __future__ import ...` - is_type_checking: bool = False # True if inside `if TYPE_CHECKING:` block - - -@dataclass -class SplitResult: - """Output of :func:`generate_file_splits`.""" - - new_files: Dict[str, str] # {target_file: source_code} - original_source: str # updated original file source - abort: bool # True if generation failed / nothing to split - abort_reason: str = "" # human-readable explanation when abort=True - entity_name_rewrites: Dict[str, Dict[str, str]] = field( - default_factory=dict - ) # {entity_name: {old_name: new_qualified_name}} per migrated entity - actual_placements: List[GroupPlacement] = field( - default_factory=list - ) # final placements after conftest routing (for accurate output messages) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -# Matches any line that is an import statement (plain or from-import). -_IMPORT_LINE_RE = re.compile(r"^(import\s+|from\s+\S.*\s+import\s+)") - -# Matches a `from __future__ import …` line (with optional trailing newline). -_FUTURE_IMPORT_LINE_RE = re.compile(r"^from __future__ import .*\n?", re.MULTILINE) - -# Matches the leading dots of a relative import (``from .foo`` or ``from ..``). -_REL_IMPORT_RE = re.compile(r"^from (\.+)", re.MULTILINE) - -# Matches four or more consecutive newlines (= 3+ blank lines between entities). -_EXCESS_BLANK_RE = re.compile(r"\n{4,}") -# Matches 3+ consecutive newlines followed by indented content (= 2+ blank lines -# inside a function/class body, where flake8 E303 allows at most one blank line). -_EXCESS_BLANK_BODY_RE = re.compile(r"\n{3,}(?=[ \t])") - - -def _multiline_string_ranges(source: str) -> List[Tuple[int, int]]: - """Return (start, end) character offsets for every multi-line string literal. - - Uses the tokenizer so that triple-quoted strings containing blank lines - followed by indented content are not mistakenly collapsed by blank-line - normalization regexes. Falls back to an empty list on tokenization error - (e.g. if the source is not yet valid Python), preserving original behavior. - """ - ranges: List[Tuple[int, int]] = [] - lines = source.splitlines(keepends=True) - # cumulative[i] = byte offset of the start of line i (0-indexed) - cumulative = [0] - for line in lines: - cumulative.append(cumulative[-1] + len(line)) - try: - tokens = tokenize.generate_tokens(io.StringIO(source).readline) - for tok_type, tok_string, tok_start, tok_end, _ in tokens: - if tok_type == tokenize.STRING and "\n" in tok_string: - start = cumulative[tok_start[0] - 1] + tok_start[1] - end = cumulative[tok_end[0] - 1] + tok_end[1] - ranges.append((start, end)) - except tokenize.TokenError: - pass - return ranges - - -def _sub_skip_strings(pattern: re.Pattern, repl: str, source: str) -> str: - """Apply *pattern*.sub(*repl*, ...) to *source*, skipping string literals. - - Blank-line normalization must not alter content inside string literals (e.g. - source code stored in a dedented triple-quoted string used in tests). - """ - ranges = _multiline_string_ranges(source) - if not ranges: - return pattern.sub(repl, source) - parts: List[str] = [] - last = 0 - for start, end in ranges: - parts.append(pattern.sub(repl, source[last:start])) - parts.append(source[start:end]) - last = end - parts.append(pattern.sub(repl, source[last:])) - return "".join(parts) - - -def _normalize_blank_lines(source: str) -> str: - """Collapse excess blank lines; ensure exactly one trailing newline. - - Removes blank-line artefacts produced by entity removal (original file) - and entity-source stripping (new files): - - - Strips leading blank lines at the start of the file (E303). - - Collapses 3+ consecutive blank lines between top-level definitions to 2 - (E303; PEP 8 allows at most two blank lines at module level). - - Collapses 2+ consecutive blank lines inside indented bodies to 1 - (E303; PEP 8 allows at most one blank line inside a function/class). - - Returns an empty string when *source* contains only whitespace, signalling - that the file should be deleted rather than written with a lone blank line. - - Multi-line string literals are protected: blank lines inside them are never - collapsed, so stored source-code snippets (e.g. in test fixtures) are not - mutated. - """ - source = _sub_skip_strings(_EXCESS_BLANK_RE, "\n\n\n", source) - source = _sub_skip_strings(_EXCESS_BLANK_BODY_RE, "\n\n", source) - source = source.lstrip("\n") - stripped = source.rstrip("\n") - if not stripped.strip(): - return "" - return stripped + "\n" - - -def _strip_orphaned_section_headers(source: str) -> str: - """Remove section header comment blocks with no substantive code after them. - - When entities are removed from the original file, section headers that - labelled a group of functions may be left with nothing beneath them. - This function detects both 3-line (``# ---...--- / # Label / # ---...---``) - and single-line (``# --- Label ---``, ``# === LABEL ===``) patterns and - removes any whose remaining content (non-blank, non-header lines) has - been entirely stripped away. - """ - lines = source.splitlines(keepends=True) - headers = _parse_section_headers(lines) - if not headers: - return source - - # 1-indexed set of lines that belong to any header block. - header_1idx: Set[int] = set() - for start, end, _ in headers: - header_1idx.update(range(start, end + 1)) - - # A header is orphaned when no substantive line (non-blank and not part of - # any header block) falls between it and the *next* header (or EOF). - orphaned_0idx: Set[int] = set() - for h_idx, (start_1, end_1, _) in enumerate(headers): - # Scan only up to the start of the next header so that content beneath - # a later header does not rescue an earlier, empty one. - if h_idx + 1 < len(headers): - scan_end_0 = headers[h_idx + 1][0] - 1 # 0-indexed exclusive - else: - scan_end_0 = len(lines) - has_content = False - for j0 in range(end_1, scan_end_0): # 0-indexed, past the header block - stripped = lines[j0].strip() - if stripped and (j0 + 1) not in header_1idx: - has_content = True - break - if not has_content: - for i1 in range(start_1, end_1 + 1): - orphaned_0idx.add(i1 - 1) # convert to 0-indexed - - if not orphaned_0idx: - return source - return "".join(line for i, line in enumerate(lines) if i not in orphaned_0idx) - - -def _strip_orphaned_indented_comments(source: str) -> str: - """Remove indented comment lines that appear at module level. - - After FileLimiter moves a function to a new file using AST line ranges, - trailing comments that were inside the function body may be left behind - in the original file. These comments retain their original indentation - (e.g. four spaces) even though they are now at module level, causing - flake8 E116 (unexpected indentation: comment). - - This function uses ``ast.parse`` to build the set of line numbers covered - by any AST node. Any comment line with leading whitespace whose line - number falls outside that set is considered orphaned and removed. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return source - - covered: Set[int] = set() - for node in ast.walk(tree): - if hasattr(node, "lineno") and hasattr(node, "end_lineno"): - for lineno in range(node.lineno, node.end_lineno + 1): - covered.add(lineno) - - lines = source.splitlines(keepends=True) - result = [] - for i, line in enumerate(lines): - lineno = i + 1 # 1-indexed - stripped = line.lstrip() - is_indented_comment = stripped.startswith("#") and len(line) > len(stripped) - if is_indented_comment and lineno not in covered: - continue - result.append(line) - return "".join(result) - - -def _import_derived_names(source: str) -> Set[str]: - """Return names introduced solely by import statements in *source*. - - These names live in the original file's namespace via its import - statements and cannot be re-exported from a new module the way - assignment-defined names can. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - names: Set[str] = set() - for node in tree.body: - if isinstance(node, ast.Import): - for alias in node.names: - names.add(alias.asname if alias.asname else alias.name.split(".")[0]) - elif isinstance(node, ast.ImportFrom): - for alias in node.names: - names.add(alias.asname if alias.asname else alias.name) - return names - - -def _collect_name_loads(source: str) -> Set[str]: - """Return Name loads in *source* that are not shadowed by function parameters - or local variable assignments. - - For each function or async function, names that appear as parameters of that - function or are assigned anywhere in the function body are excluded from Name - loads within its body. This prevents generating spurious cross-file imports - for names that are satisfied locally (e.g. pytest fixture names that appear as - test function parameters, or local variables like ``helpers = tmp_path / ...``). - - Decorators, argument default values, and return/argument annotations are - always evaluated in the outer scope and are never excluded. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - names: Set[str] = set() - - def _body_stores(stmts) -> frozenset: - """Names stored/deleted at this scope level in *stmts*. - - Recurses into control-flow nodes (if/for/while/with/try) but stops at - nested FunctionDef/AsyncFunctionDef/ClassDef scopes so only names that - are local to the current function are returned. - """ - stores: Set[str] = set() - work = list(stmts) - while work: - node = work.pop() - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - continue - if isinstance(node, ast.Name) and isinstance( - node.ctx, (ast.Store, ast.Del) - ): - stores.add(node.id) - work.extend(ast.iter_child_nodes(node)) - return frozenset(stores) - - def _walk(node: ast.AST, excluded: frozenset) -> None: - if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): - if node.id not in excluded: - names.add(node.id) - return - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - args = node.args - own_params: frozenset = frozenset( - a.arg - for a in ( - args.args - + args.posonlyargs - + args.kwonlyargs - + ([args.vararg] if args.vararg else []) - + ([args.kwarg] if args.kwarg else []) - ) - ) - # Decorators are evaluated in the outer scope. - for dec in node.decorator_list: - _walk(dec, excluded) - # Default values are evaluated in the outer scope. - for default in args.defaults + args.kw_defaults: - if default is not None: - _walk(default, excluded) - # Annotations are in the outer scope (PEP 563 / regular annotations). - for arg in args.args + args.posonlyargs + args.kwonlyargs: - if arg.annotation: - _walk(arg.annotation, excluded) - if args.vararg and args.vararg.annotation: - _walk(args.vararg.annotation, excluded) - if args.kwarg and args.kwarg.annotation: - _walk(args.kwarg.annotation, excluded) - if node.returns: - _walk(node.returns, excluded) - # Function body uses params + local stores as the excluded set. - own_locals = _body_stores(node.body) - new_excluded = excluded | own_params | own_locals - for child in node.body: - _walk(child, new_excluded) - return - for child in ast.iter_child_nodes(node): - _walk(child, excluded) - - _walk(tree, frozenset()) - return names - - -def _collect_quoted_annotation_names(source: str) -> Set[str]: - """Return names referenced inside quoted type annotations in *source*. - - Finds names like ``_LLMAccumulator`` in ``Optional["_LLMAccumulator"]`` - (string literals used as forward references in type annotations). These - names are only needed at type-checking time — not at runtime — and should - be imported under ``if TYPE_CHECKING:`` rather than as regular imports. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - names: Set[str] = set() - - def _scan_annotation(node: ast.AST) -> None: - """Recursively scan an annotation, extracting names from string constants.""" - if isinstance(node, ast.Constant) and isinstance(node.value, str): - try: - inner = ast.parse(node.value, mode="eval") - for n in ast.walk(inner): - if isinstance(n, ast.Name): - names.add(n.id) - except SyntaxError: - pass - return - for child in ast.iter_child_nodes(node): - _scan_annotation(child) - - def _walk(node: ast.AST) -> None: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - args = node.args - for arg in args.args + args.posonlyargs + args.kwonlyargs: - if arg.annotation: - _scan_annotation(arg.annotation) - if args.vararg and args.vararg.annotation: - _scan_annotation(args.vararg.annotation) - if args.kwarg and args.kwarg.annotation: - _scan_annotation(args.kwarg.annotation) - if node.returns: - _scan_annotation(node.returns) - for child in node.body: - _walk(child) - return - if isinstance(node, ast.AnnAssign): - _scan_annotation(node.annotation) - if node.value: - _walk(node.value) - return - for child in ast.iter_child_nodes(node): - _walk(child) - - _walk(tree) - return names - - -def _collect_name_stores(source: str) -> Set[str]: - """Return names assigned at module level in *source*. - - Detects ``x = ...``, ``x += ...``, and annotated assignments with a value - (``x: int = ...``) at the top level of the module. Used to identify - TOP_LEVEL constants that are mutated outside their defining entity so that - cross-file references must use ``module.NAME`` rather than a plain - ``from .module import NAME``. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - stores: Set[str] = set() - for node in tree.body: - if isinstance(node, ast.Assign): - for target in node.targets: - if isinstance(target, ast.Name): - stores.add(target.id) - elif isinstance(node, ast.AugAssign): - if isinstance(node.target, ast.Name): - stores.add(node.target.id) - elif isinstance(node, ast.AnnAssign): - if node.value is not None and isinstance(node.target, ast.Name): - stores.add(node.target.id) - return stores - - -def _inject_module_level_imports(source: str, imports: List[str]) -> str: - """Insert *imports* after the last existing import line in *source*. - - Uses the same insertion logic as :func:`_add_re_exports` so that module - imports for reassigned TOP_LEVEL variables land in the same position as - other imports added to the original file. - """ - if not imports: - return source - lines = source.splitlines(keepends=True) - last_import_line = 0 - try: - tree = ast.parse(source) - except SyntaxError: - return "\n".join(sorted(imports)) + "\n\n" + source - for node in tree.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - last_import_line = max(last_import_line, node.end_lineno) - insert_after = last_import_line - if insert_after == 0 and tree.body: - first = tree.body[0] - if ( - isinstance(first, ast.Expr) - and isinstance(first.value, ast.Constant) - and isinstance(first.value.value, str) - ): - insert_after = first.end_lineno - import_lines = [imp + "\n" for imp in sorted(imports)] - return "".join(lines[:insert_after] + import_lines + lines[insert_after:]) - - -def _inject_type_checking_imports(source: str, imports: List[str]) -> str: - """Add *imports* under a module-level ``if TYPE_CHECKING:`` guard in *source*. - - If a TYPE_CHECKING block already exists, new imports are appended to it - (skipping any already present). Otherwise a new block is inserted after - the last top-level import statement, along with ``from typing import - TYPE_CHECKING`` when that name is not already imported. - """ - if not imports: - return source - try: - tree = ast.parse(source) - except SyntaxError: - return source - - # Determine which imports are not already in an existing TC block. - existing_tc = {i.source for i in _extract_import_info(source) if i.is_type_checking} - new_imports = [imp for imp in imports if imp not in existing_tc] - if not new_imports: - return source - - lines = source.splitlines(keepends=True) - - # Append to an existing TYPE_CHECKING block if one is present. - for node in tree.body: - if ( - isinstance(node, ast.If) - and isinstance(node.test, ast.Name) - and node.test.id == "TYPE_CHECKING" - ): - insert_line = node.end_lineno - new_lines = [" " + imp + "\n" for imp in sorted(new_imports)] - return "".join(lines[:insert_line] + new_lines + lines[insert_line:]) - - # No existing block: insert one after the last top-level import. - last_import_line = 0 - for node in tree.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - last_import_line = max(last_import_line, node.end_lineno) - insert_after = last_import_line - - tc_already_imported = any( - isinstance(n, ast.ImportFrom) - and n.module == "typing" - and any((a.asname or a.name) == "TYPE_CHECKING" for a in n.names) - for n in tree.body - ) - new_lines = [] - if not tc_already_imported: - new_lines.append("from typing import TYPE_CHECKING\n") - new_lines.append("if TYPE_CHECKING:\n") - for imp in sorted(new_imports): - new_lines.append(" " + imp + "\n") - new_lines.append("\n") - return "".join(lines[:insert_after] + new_lines + lines[insert_after:]) - - -def _test_names_in_decorators(source: str, names: Set[str]) -> Set[str]: - """Return the subset of *names* that appear as Name loads inside a decorator. - - Decorators are evaluated before function bodies run, so a symbol that - only reaches a file via an inline import (injected into the function body) - will not be in scope when the decorator is evaluated. This helper detects - that situation so callers can abort the split rather than generate broken - code. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - found: Set[str] = set() - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - for dec in node.decorator_list: - for child in ast.walk(dec): - if ( - isinstance(child, ast.Name) - and isinstance(child.ctx, ast.Load) - and child.id in names - ): - found.add(child.id) - return found - - -def _extract_import_info(source: str) -> List[ImportInfo]: - """Return :class:`ImportInfo` for each top-level import in *source*. - - Also includes imports found inside module-level ``if TYPE_CHECKING:`` - blocks, marked with ``is_type_checking=True``. These are used by - :func:`_find_type_checking_needed_imports` to distribute forward-reference - imports to the correct sub-files after a split. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return [] - - lines = source.splitlines(keepends=True) - result: List[ImportInfo] = [] - - for node in tree.body: - if isinstance(node, ast.Import): - names = [ - alias.asname if alias.asname else alias.name.split(".")[0] - for alias in node.names - ] - src = "".join(lines[node.lineno - 1 : node.end_lineno]).rstrip() - result.append(ImportInfo(names=names, source=src, is_future=False)) - elif isinstance(node, ast.ImportFrom): - names = [ - alias.asname if alias.asname else alias.name for alias in node.names - ] - # Reconstruct as a normalized single-line import so that - # multi-line parenthesized imports (e.g. ``from X import (\n - # Y,\n Z,\n)``) don't break _merge_from_imports, whose regex - # only matches the first line. - dots = "." * (node.level or 0) - mod = node.module or "" - alias_strs = [ - f"{a.name} as {a.asname}" if a.asname else a.name for a in node.names - ] - src = f"from {dots}{mod} import {', '.join(alias_strs)}" - is_future = node.module == "__future__" - result.append(ImportInfo(names=names, source=src, is_future=is_future)) - elif ( - isinstance(node, ast.If) - and isinstance(node.test, ast.Name) - and node.test.id == "TYPE_CHECKING" - ): - for child in node.body: - if isinstance(child, ast.Import): - tc_names = [ - alias.asname if alias.asname else alias.name.split(".")[0] - for alias in child.names - ] - tc_src = "".join( - lines[child.lineno - 1 : child.end_lineno] - ).rstrip() - result.append( - ImportInfo( - names=tc_names, - source=tc_src, - is_future=False, - is_type_checking=True, - ) - ) - elif isinstance(child, ast.ImportFrom): - tc_names = [ - alias.asname if alias.asname else alias.name - for alias in child.names - ] - tc_dots = "." * (child.level or 0) - tc_mod = child.module or "" - tc_alias_strs = [ - f"{a.name} as {a.asname}" if a.asname else a.name - for a in child.names - ] - tc_src = f"from {tc_dots}{tc_mod} import {', '.join(tc_alias_strs)}" - result.append( - ImportInfo( - names=tc_names, - source=tc_src, - is_future=False, - is_type_checking=True, - ) - ) - - return result - - -def _find_needed_imports( - entity_names: List[str], - entity_source_map: Dict[str, str], - import_infos: List[ImportInfo], - all_entity_names: Set[str], -) -> List[str]: - """Return import statements needed by the given entities. - - Always includes ``from __future__`` imports. Other imports are included - when any of the names they introduce appear in the entities' source. - Duplicate import source strings are deduplicated. - """ - referenced: Set[str] = set() - for name in entity_names: - src = entity_source_map.get(name, "") - referenced |= _collect_name_loads(src) - - needed: List[str] = [] - seen: Set[str] = set() - for info in import_infos: - if info.source in seen: - continue - if info.is_type_checking: - continue # handled by _find_type_checking_needed_imports - if info.is_future or any(n in referenced for n in info.names): - needed.append(info.source) - seen.add(info.source) - - return needed - - -def _narrow_import_source(import_src: str, keep_names: Set[str]) -> str: - """Return a copy of *import_src* keeping only the exposed names in *keep_names*. - - For ``from X import A, B, C`` with ``keep_names={A}``, returns - ``from X import A``. Non-ImportFrom statements are returned unchanged. - """ - try: - node = ast.parse(import_src).body[0] - except (SyntaxError, IndexError): - return import_src - if not isinstance(node, ast.ImportFrom): - return import_src - dots = "." * (node.level or 0) - mod = node.module or "" - alias_strs = [ - f"{a.name} as {a.asname}" if a.asname else a.name - for a in node.names - if (a.asname or a.name) in keep_names - ] - if not alias_strs: - return import_src - return f"from {dots}{mod} import {', '.join(alias_strs)}" - - -def _find_type_checking_needed_imports( - entity_names: List[str], - entity_source_map: Dict[str, str], - import_infos: List[ImportInfo], -) -> List[str]: - """Return import statements needed only for quoted type annotations. - - These should be placed under ``if TYPE_CHECKING:`` because the names are - only referenced inside string-valued annotations (forward references) and - are not needed at runtime. Names that appear in regular (non-annotation) - loads are excluded via ``annotation_only = quoted - runtime``, which - guarantees that any name emitted here will be pruned from regular imports - by ``_prune_unused_imports`` — so no duplicate imports can arise. - ``__future__`` imports are always excluded since they are handled by - ``_find_needed_imports``. - """ - runtime: Set[str] = set() - quoted: Set[str] = set() - for name in entity_names: - src = entity_source_map.get(name, "") - runtime |= _collect_name_loads(src) - quoted |= _collect_quoted_annotation_names(src) - - annotation_only = quoted - runtime - if not annotation_only: - return [] - - needed: List[str] = [] - seen: Set[str] = set() - for info in import_infos: - if info.source in seen: - continue - if info.is_future: - continue - tc_names = {n for n in info.names if n in annotation_only} - if not tc_names: - continue - # Narrow the import to only the names actually needed for type checking, - # avoiding unused-import warnings for names from multi-name imports that - # are not referenced in this file. - src = ( - info.source - if len(tc_names) == len(info.names) - else _narrow_import_source(info.source, tc_names) - ) - if src in seen: - continue - needed.append(src) - seen.add(src) - return needed - - -def _bump_relative_imports(source: str, n: int = 1) -> str: - """Increment the level of every relative import in *source* by *n*. - - Used when file content is moved directory levels deeper, e.g. when the - source originally written for ``pkg/module.py`` becomes the content of - ``pkg/module/__init__.py``, or when new files go into a subdirectory - package instead of sitting next to the original file. - - With n=1: ``from .foo`` → ``from ..foo``, ``from ..bar`` → ``from ...bar``. - With n=2: ``from .foo`` → ``from ...foo``, etc. - Absolute imports are not affected. - """ - for _ in range(n): - source = _REL_IMPORT_RE.sub(lambda m: f"from .{m.group(1)}", source) - return source - - -def _relative_import_prefix(from_file: str, to_file: str) -> str: - """Return the Python relative-import prefix for *to_file* as seen from *from_file*. - - Both paths are relative to the same base directory (the original file's - directory). Examples:: - - _relative_import_prefix("utils.py", "helpers.py") → ".helpers" - _relative_import_prefix("sub/a.py", "helpers/b.py") → "..helpers.b" - _relative_import_prefix("sub/a.py", "sub/b.py") → ".b" - _relative_import_prefix("a.py", "__init__.py") → "." - _relative_import_prefix("sub/a.py", "sub/__init__.py") → "." - """ - to_path = Path(to_file) - from_parts = Path(from_file).parent.parts # () for top-level files - # __init__.py represents the package itself, not a submodule named "__init__". - if to_path.stem == "__init__": - to_module_parts = to_path.parent.parts - else: - to_module_parts = to_path.with_suffix("").parts # ("helpers", "b") - to_dir_parts = to_path.parent.parts # ("helpers",) - - # Length of the common directory prefix between from_dir and to_dir. - common_len = 0 - for fp, tp in zip(from_parts, to_dir_parts): - if fp == tp: - common_len += 1 - else: - break - - levels_up = len(from_parts) - common_len - module = ".".join(to_module_parts[common_len:]) - return "." * (levels_up + 1) + module - - -def _module_import_stmt( - current_target: str, - source_file: str, - abs_pkg: Optional[str], -) -> Tuple[str, str]: - """Return ``(import_statement, local_name)`` for a module-level import. - - Produces ``from . import conversion`` instead of - ``from .conversion import SAFE_MODE`` so callers can reference - ``conversion.SAFE_MODE`` for a live lookup rather than a value snapshot. - This preserves the original single-file behaviour where module globals are - looked up dynamically rather than captured at import time. - """ - local_name = _target_module_name(source_file).split(".")[-1] - if abs_pkg is not None: - mod = _target_module_name(source_file) - # Use "import full.module.path as local_name" for absolute contexts. - # This avoids "from pkg import test_module" patterns that are - # misidentified as test-name imports by _split_cross_imports_by_test. - full_mod = f"{abs_pkg}.{mod}" if abs_pkg else mod - stmt = ( - f"import {full_mod} as {local_name}" - if full_mod != local_name - else f"import {local_name}" - ) - else: - prefix = _relative_import_prefix(current_target, source_file) - # prefix looks like ".conversion", "..test_svc", or "..helpers.io". - # Decompose into leading dots + module path, then extract the last - # segment as local_name and the rest as the parent package prefix. - # ".conversion" → dots="..", path="conversion" → "from . import conversion" - # "..test_svc" → dots="..", path="test_svc" → "from .. import test_svc" - # "..helpers.io" → dots="..", path="helpers.io" → "from ..helpers import io" - dot_end = 0 - while dot_end < len(prefix) and prefix[dot_end] == ".": - dot_end += 1 - dots = prefix[:dot_end] - path = prefix[dot_end:] - last_dot = path.rfind(".") - if last_dot == -1: - parent = dots or "." - else: - parent = dots + path[:last_dot] - stmt = f"from {parent} import {local_name}" - return stmt, local_name - - -def _find_cross_file_imports( - entity_names: List[str], - entity_source_map: Dict[str, str], - name_to_target_file: Dict[str, str], - current_target: str, - abs_pkg: Optional[str] = None, - top_level_var_names: Optional[Set[str]] = None, -) -> Tuple[List[str], List[str], Dict[str, str]]: - """Return ``(from_imports, module_imports, name_rewrites)`` for other-file - dependencies. - - When an entity being moved to *current_target* references a name that is - defined by another entity being moved to a different target file, the new - file needs an explicit import for that name. - - *from_imports* are ``from .module import Name`` statements for - function/class references. These may be subject to test-name inline - injection by the caller (to avoid pytest collecting imported test functions - as duplicate tests). - - *module_imports* are ``from . import module`` (or ``import pkg.module as - module``) statements for names defined by ``TOP_LEVEL`` entities - (module-level variables such as ``SAFE_MODE = True``). These must always - be placed at module level — never injected inline — because they are - required by decorator expressions that are evaluated before any function - body runs. The returned *name_rewrites* dict maps each such bare name - (e.g. ``"SAFE_MODE"``) to its qualified form (e.g. - ``"conversion.SAFE_MODE"``); callers must rewrite the entity source - accordingly. - - When *abs_pkg* is ``None`` the import prefix is relative (e.g. - ``from .constants import _CONST``). When *abs_pkg* is set the import is - absolute (e.g. ``from tests.constants import _CONST``), which is required - for test files that pytest loads as top-level modules. - """ - referenced: Set[str] = set() - for name in entity_names: - src = entity_source_map.get(name, "") - referenced |= _collect_name_loads(src) - from_files: Dict[str, List[str]] = {} # source_file → regular names - mod_files: Dict[str, List[str]] = {} # source_file → top-level var names - for ref_name in sorted(referenced): - source_file = name_to_target_file.get(ref_name) - if source_file and source_file != current_target: - if top_level_var_names and ref_name in top_level_var_names: - mod_files.setdefault(source_file, []).append(ref_name) - else: - from_files.setdefault(source_file, []).append(ref_name) - - from_result: List[str] = [] - mod_result: List[str] = [] - rewrites: Dict[str, str] = {} - for source_file, names in sorted(from_files.items()): - if abs_pkg is not None: - mod = _target_module_name(source_file) - prefix = f"{abs_pkg}.{mod}" if abs_pkg else mod - else: - prefix = _relative_import_prefix(current_target, source_file) - from_result.append(f"from {prefix} import {', '.join(sorted(names))}") - for source_file, names in sorted(mod_files.items()): - stmt, local_name = _module_import_stmt(current_target, source_file, abs_pkg) - mod_result.append(stmt) - for name in names: - rewrites[name] = f"{local_name}.{name}" - return from_result, mod_result, rewrites - - -def _find_cross_file_type_checking_imports( - entity_names: List[str], - entity_source_map: Dict[str, str], - name_to_target_file: Dict[str, str], - current_target: str, - abs_pkg: Optional[str] = None, - top_level_var_names: Optional[Set[str]] = None, -) -> List[str]: - """Return cross-file imports for names only referenced in quoted annotations. - - When an entity uses a name only inside a quoted type annotation (e.g. - ``Optional["_LLMAccumulator"]``) and that name is defined in another new - file produced by the same split, a ``from .other import Name`` statement - is generated here. These should be placed under ``if TYPE_CHECKING:`` - because they are not needed at runtime. - - Names that also appear in regular (non-annotation) loads are excluded — - they already get a normal cross-file import from - ``_find_cross_file_imports``. Top-level variable names (which require - module-alias imports) are also skipped here. - """ - runtime_referenced: Set[str] = set() - quoted_referenced: Set[str] = set() - for name in entity_names: - src = entity_source_map.get(name, "") - runtime_referenced |= _collect_name_loads(src) - quoted_referenced |= _collect_quoted_annotation_names(src) - - annotation_only = quoted_referenced - runtime_referenced - if not annotation_only: - return [] - - tc_files: Dict[str, List[str]] = {} - for ref_name in sorted(annotation_only): - source_file = name_to_target_file.get(ref_name) - if source_file and source_file != current_target: - # Top-level var names need module-alias imports, not handled here. - if top_level_var_names and ref_name in top_level_var_names: - continue - tc_files.setdefault(source_file, []).append(ref_name) - - result: List[str] = [] - for source_file, names in sorted(tc_files.items()): - if abs_pkg is not None: - mod = _target_module_name(source_file) - prefix = f"{abs_pkg}.{mod}" if abs_pkg else mod - else: - prefix = _relative_import_prefix(current_target, source_file) - result.append(f"from {prefix} import {', '.join(sorted(names))}") - return result - - -_FROM_IMPORT_RE = re.compile(r"^(from\s+\S+)\s+import\s+(.*)") - - -def _merge_from_imports(imports: List[str]) -> List[str]: - """Merge ``from X import …`` lines that share the same module prefix. - - When multiple entities each contribute a ``from X import`` for the same - module but with different name subsets, the naive per-entity approach - produces duplicate imports such as:: - - from .conversion import lua_to_python, python_to_lua - from .conversion import lua_to_python_preserve_wrapped, python_to_lua - - This function collapses them into a single statement per prefix, with - names sorted and deduplicated:: - - from .conversion import lua_to_python, lua_to_python_preserve_wrapped, python_to_lua # noqa: E501 - - Plain ``import X`` statements are preserved unchanged and appended after - the merged from-imports. - """ - from_map: Dict[str, List[str]] = {} - order: List[str] = [] # first-seen order of prefixes - plain: List[str] = [] - for imp in imports: - m = _FROM_IMPORT_RE.match(imp) - if not m: - plain.append(imp) - continue - prefix = m.group(1) - names = [n.strip() for n in m.group(2).split(",") if n.strip()] - if prefix not in from_map: - from_map[prefix] = [] - order.append(prefix) - from_map[prefix].extend(names) - result = [] - for prefix in order: - unique = sorted(dict.fromkeys(from_map[prefix])) - result.append(f"{prefix} import {', '.join(unique)}") - return result + plain - - -def _target_module_name(target_file: str) -> str: - """Convert a relative target filename to a dotted module name. - - ``"utils.py"`` → ``"utils"``, ``"helpers/io.py"`` → ``"helpers.io"``, - ``"pkg/__init__.py"`` → ``"pkg"`` (package, not ``"pkg.__init__"``). - """ - path = Path(target_file) - if path.stem == "__init__": - parts = list(path.parent.parts) - else: - parts = list(path.with_suffix("").parts) - return ".".join(parts) - - -def _import_line_numbers(entity: Entity, entity_src: str) -> Set[int]: - """Return absolute 1-based line numbers of import statements in *entity*. - - Used to preserve import lines in the original file when a TOP_LEVEL - entity that mixes imports and assignments is migrated. - """ - try: - tree = ast.parse(entity_src) - except SyntaxError: - return set() - result: Set[int] = set() - for node in tree.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - for rel_ln in range(node.lineno, node.end_lineno + 1): - result.add(entity.start_line + rel_ln - 1) - return result - - -def _remove_entity_lines( - source: str, - migrated_names: Set[str], - entity_map: Dict[str, Entity], - entity_source_map: Dict[str, str], -) -> str: - """Return *source* with lines belonging to migrated entities removed. - - For TOP_LEVEL entities, import statement lines are preserved in the - original file even when the entity is migrated: the remaining code may - still reference those imported names, and stdlib/third-party names - cannot be safely re-exported from a new module. - """ - remove: Set[int] = set() - preserve: Set[int] = set() - for name in migrated_names: - entity = entity_map.get(name) - if entity is None: - continue - for ln in range(entity.start_line, entity.end_line + 1): - remove.add(ln) - if entity.kind == EntityKind.TOP_LEVEL: - preserve |= _import_line_numbers(entity, entity_source_map.get(name, "")) - - lines = source.splitlines(keepends=True) - return "".join( - line for i, line in enumerate(lines, 1) if i not in remove or i in preserve - ) - - -def _find_project_root(path: Path) -> Optional[Path]: - """Walk up from *path* to find the project root directory. - - Returns the first directory containing ``pyproject.toml``, ``setup.py``, - ``setup.cfg``, or ``.git``. Returns ``None`` when the filesystem root is - reached without finding any of these markers. - """ - markers = {"pyproject.toml", "setup.py", "setup.cfg", ".git"} - current = path if path.is_dir() else path.parent - while True: - if any((current / m).exists() for m in markers): - return current - parent = current.parent - if parent == current: - return None - current = parent - - -def _module_path_from_file(project_root: Path, file_path: Path) -> Optional[str]: - """Return the dotted Python module path of *file_path* relative to *project_root*. - - Returns ``None`` when *file_path* is not under *project_root*. - """ - try: - rel = file_path.relative_to(project_root) - except ValueError: - return None - return ".".join(rel.with_suffix("").parts) - - -def _abs_package_for_dir(file_path: str) -> Optional[str]: - """Return the dotted package path of the directory containing *file_path*. - - Used to generate absolute imports for test files so that pytest's default - import mode (which loads test files as top-level modules, not package - members) does not choke on ``from .module import …`` syntax. - - Returns an empty string for files sitting directly in the project root, - ``None`` when the project root cannot be determined. - """ - orig = Path(file_path).resolve() - project_root = _find_project_root(orig.parent) - if project_root is None: - return None - try: - rel = orig.parent.relative_to(project_root) - except ValueError: - return None - return ".".join(rel.parts) - - -def _collect_external_imported_names(original_path: str) -> Set[str]: - """Return names imported from *original_path* by other Python files. - - Scans all Python files under the project root for ``from import`` - statements targeting the module corresponding to *original_path*, and - returns the union of all imported original names (before any ``as`` alias). - - Returns an empty set when *original_path* does not resolve to an existing - file, the project root cannot be determined, or the path cannot be mapped - to a module. Both absolute and relative paths are accepted; relative paths - are resolved against the current working directory (the repo root when - crispen is invoked as ``git diff | crispen``). - """ - orig = Path(original_path).resolve() - if not orig.exists(): - return set() - project_root = _find_project_root(orig.parent) - if project_root is None: - return set() - # project_root is an ancestor of orig (derived by walking up from orig.parent), - # so _module_path_from_file always returns a non-None string here. - target_module = _module_path_from_file(project_root, orig) - # __init__.py defines the package itself; external callers import from the - # package path (e.g. "pkg.sub"), not "pkg.sub.__init__". - if orig.name == "__init__.py": - dot = target_module.rfind(".") - if dot == -1: - return set() # bare __init__.py at project root; no external callers - target_module = target_module[:dot] - result: Set[str] = set() - for py_file in project_root.rglob("*.py"): - if py_file.resolve() == orig: - continue - try: - source = py_file.read_text(encoding="utf-8", errors="replace") - tree = ast.parse(source, filename=str(py_file)) - except Exception: - continue - # Compute this file's dotted module path for relative-import resolution. - file_module = _module_path_from_file(project_root, py_file) - file_pkg_parts = file_module.split(".")[:-1] if file_module else [] - for node in ast.walk(tree): - if not isinstance(node, ast.ImportFrom): - continue - if node.level == 0: - imported_from = node.module or "" - else: - # Relative import: go up (level - 1) packages from file_pkg_parts. - up = node.level - 1 - if up > len(file_pkg_parts): - continue - base = file_pkg_parts[: len(file_pkg_parts) - up] - sub = node.module or "" - imported_from = ".".join(base + ([sub] if sub else [])) - if imported_from != target_module: - continue - for alias in node.names: - result.add(alias.name) - return result - - -def _class_has_test_methods(entity_src: str) -> bool: - """Return True if *entity_src* defines a class with any ``test_`` methods. - - Used to suppress re-exports of test classes: pytest discovers test classes - by scanning the filesystem, so re-exporting them from the original file - causes every test inside to run twice. - """ - try: - tree = ast.parse(entity_src) - except SyntaxError: - return False - for node in tree.body: - if isinstance(node, ast.ClassDef): - for item in node.body: - if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): - if item.name.startswith("test_"): - return True - return False - - -def _add_re_exports( - source: str, - placements: List[GroupPlacement], - entity_map: Dict[str, Entity], - entity_source_map: Dict[str, str], - external_loads: Set[str] = frozenset(), - abs_pkg: Optional[str] = None, - relative_from: Optional[str] = None, - is_test_file: bool = False, - reexport_mode: str = "always", -) -> str: - """Add ``from .module import name`` imports for migrated entities. - - *reexport_mode* controls when public (non-underscore) names get a - re-export stub: - - * ``"always"`` — always re-export every public name (default; most - conservative, preserves the full public API regardless of usage). - * ``"application"`` — re-export public names in non-test files only. - * ``"imported"`` — re-export a public name only when it appears in - *external_loads* (imported from the original module by another file in - the project) or is still referenced in the remaining *source*. - - Private names (starting with ``_``) are always re-exported when the - remaining *source* still references them, or when they appear in - *external_loads*, regardless of *reexport_mode*. - - When *relative_from* is set (e.g. ``"service/__init__.py"``), import - prefixes are computed via :func:`_relative_import_prefix` so that - re-exports from a package ``__init__.py`` reference sibling modules - correctly (e.g. ``from .utils import Foo`` instead of - ``from .service.utils import Foo``). - - Import-derived names (names introduced by ``import`` / ``from … import`` - statements inside a TOP_LEVEL entity) are never re-exported: they were - kept in the original file by :func:`_remove_entity_lines` and cannot - meaningfully be re-exported from a new module. - - Inserts after the last import line in *source*. Returns *source* unchanged - when there are no names to import. - """ - still_loaded = _collect_name_loads(source) - re_exports: Dict[str, List[str]] = {} - # Names added solely for external re-export (not referenced in remaining source). - # These need "# fmt: skip # noqa: F401, E501" to suppress flake8 false positives. - noqa_names: Set[str] = set() - for placement in placements: - # Compute the import prefix for this placement's target file. - if relative_from is not None: - import_prefix = _relative_import_prefix( - relative_from, placement.target_file - ) - elif abs_pkg is not None: - module = _target_module_name(placement.target_file) - import_prefix = f"{abs_pkg}.{module}" if abs_pkg else module - else: - module = _target_module_name(placement.target_file) - import_prefix = f".{module}" - to_import: List[str] = [] - for entity_name in placement.group: - if entity_name in entity_map: - entity = entity_map[entity_name] - defined = entity.names_defined - if entity.kind == EntityKind.TOP_LEVEL: - skip = _import_derived_names(entity_source_map.get(entity_name, "")) - defined = [n for n in defined if n not in skip] - else: - defined = [entity_name] - is_test_class = entity_name in entity_map and _class_has_test_methods( - entity_source_map.get(entity_name, "") - ) - for defined_name in defined: - # Test-named symbols (Test* / test_*) are never re-exported at - # module level: _inject_inline_test_imports_original injects - # them inside function/class bodies to prevent pytest from - # discovering the same test twice. - if _is_test_name(defined_name): - continue - # Unconditional public re-export: only when reexport_mode - # permits it for this file type. - reexport_unconditionally = ( - not defined_name.startswith("_") - and not defined_name.startswith("test_") - and not is_test_class - and ( - reexport_mode == "always" - or (reexport_mode == "application" and not is_test_file) - ) - ) - if ( - reexport_unconditionally - or defined_name in still_loaded - or defined_name in external_loads - ): - to_import.append(defined_name) - # Add noqa when the name is not referenced in the remaining - # source (pure re-export stub), OR when it is in external_loads - # — in the latter case a non-migrated entity may currently use - # the name, but if that entity is itself migrated in a later - # recursive split the stub would become unreferenced and - # _prune_unused_imports would silently drop it, breaking the - # external caller. The noqa marker protects against that. - if ( - defined_name not in still_loaded - or defined_name in external_loads - ): - noqa_names.add(defined_name) - if to_import: - re_exports.setdefault(import_prefix, []).extend(to_import) - - if not re_exports: - return source - - # Build export statements. When a name is only there for external re-export - # (not referenced in the remaining source), add "# fmt: skip # noqa: F401, E501" - # so flake8 does not flag it as an unused import and Black does not reformat - # the line (which would break the noqa directive). Split mixed imports into - # two lines so that the noqa comment does not suppress warnings for used names. - export_stmts: List[str] = [] - for prefix, names in sorted(re_exports.items()): - sorted_names = sorted(names) - used = [n for n in sorted_names if n not in noqa_names] - noqa = [n for n in sorted_names if n in noqa_names] - if used: - export_stmts.append(f"from {prefix} import {', '.join(used)}\n") - for name in noqa: - export_stmts.append( - f"from {prefix} import {name} # fmt: skip # noqa: F401, E501\n" - ) - - # In test files, add a single explanatory comment before the first F401 import. - if is_test_file and noqa_names: - first_noqa = next(i for i, s in enumerate(export_stmts) if "# noqa: F401" in s) - export_stmts.insert( - first_noqa, - "# Re-exported for backwards compatibility with external callers.\n", - ) - - lines = source.splitlines(keepends=True) - last_import_line = 0 - try: - tree = ast.parse(source) - except SyntaxError: - return source - for node in tree.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - last_import_line = max(last_import_line, node.end_lineno) - - insert_after = last_import_line - if insert_after == 0 and tree.body: - first = tree.body[0] - if ( - isinstance(first, ast.Expr) - and isinstance(first.value, ast.Constant) - and isinstance(first.value.value, str) - ): - insert_after = first.end_lineno - - return "".join(lines[:insert_after] + export_stmts + lines[insert_after:]) - - -def _topo_depth(graph: Dict[str, Set[str]]) -> Dict[str, int]: - """Return topological depth for each node in a DAG. - - Depth 0 = leaf (no outgoing edges). A node's depth is 1 + the maximum - depth of its dependencies. All dependency nodes must be keys in *graph*. - On non-DAG inputs (cycles detected), returns 0 for every node as a safe - fallback so that callers degrade to arbitrary candidate ordering. - """ - if any(len(s) > 1 for s in find_sccs(graph)): - return {node: 0 for node in graph} - depths: Dict[str, int] = {} - - def dfs(node: str) -> int: - if node in depths: - return depths[node] - depths[node] = 1 + max((dfs(dep) for dep in graph[node]), default=-1) - return depths[node] - - for node in graph: - dfs(node) - return depths - - -def _extract_shared_helpers( - file_entity_names: Dict[str, List[str]], - entity_source_map: Dict[str, str], - entity_map: Dict[str, Entity], - classified: ClassifiedEntities, - name_to_target_file: Dict[str, str], - migrated_names: Set[str], - original_basename: str, -) -> List[GroupPlacement]: - """Extract non-migrated functions/classes referenced by migrated entities. - - When a migrated entity in new file F references a non-migrated function X - from the original O, the generated ``from .O import X`` combined with O's - re-export ``from .F import …`` creates a cycle O→F→O. - - Fix: pull X (and all helpers X transitively depends on) into a new file - that uses them. The destination is chosen using topological depth ordering: - the inter-file dependency graph is built from migrated-entity cross-references - first, then for each helper SCC the candidates (all files wanting the - helpers) are sorted by topological depth (deepest / most-downstream first). - For a DAG the deepest wanting file is always cycle-free on the first try; - for non-DAG inputs (pre-existing cycles) _topo_depth falls back to 0 for - all nodes and the loop exhausts all candidates via trial SCC analysis. - If no cycle-free placement exists the SCC is left in the original file and - the safety-net in :func:`generate_file_splits` will abort if the result is - unloadable. - - Mutates *file_entity_names*, *migrated_names*, and *name_to_target_file* - in place. Returns synthetic :class:`GroupPlacement` objects for the - extracted entities so that :func:`_add_re_exports` can re-import them from - their new location in the updated original source. - """ - # Build defined-name → entity-name map for non-migrated FUNCTION/CLASS entities. - defined_to_entity: Dict[str, str] = {} - for entity in classified.entities: - if entity.name in migrated_names: - continue - if entity.kind not in (EntityKind.FUNCTION, EntityKind.CLASS): - continue - for defined_name in entity.names_defined: - if name_to_target_file.get(defined_name) == original_basename: - defined_to_entity[defined_name] = entity.name - - # Collect directly-wanted helpers: entity_name → set of target_files that want it. - wanting: Dict[str, Set[str]] = {} - for target_file, ent_names in list(file_entity_names.items()): - for ent_name in ent_names: - src = entity_source_map.get(ent_name, "") - for ref_name in _collect_name_loads(src): - entity_name = defined_to_entity.get(ref_name) - if entity_name is not None: - wanting.setdefault(entity_name, set()).add(target_file) - - if not wanting: - return [] - - # Transitively expand wanting-sets to cover helpers referenced by - # already-wanted helpers, preventing O→new-file→O cycles. - # Re-queue a helper whenever its wanting-set gains new target files so that - # the propagation reaches all transitive dependents. - queue = list(wanting.keys()) - idx = 0 - while idx < len(queue): - entity_name = queue[idx] - idx += 1 - src = entity_source_map.get(entity_name, "") - for ref_name in _collect_name_loads(src): - dep_name = defined_to_entity.get(ref_name) - if dep_name and wanting[entity_name] - wanting.get(dep_name, set()): - wanting.setdefault(dep_name, set()).update(wanting[entity_name]) - queue.append(dep_name) - - # SCC analysis on the sub-graph of wanted helpers to co-locate - # mutually-dependent helpers. - sub_graph: Dict[str, Set[str]] = { - name: {d for d in classified.graph.get(name, set()) if d in wanting} - for name in wanting - } - sccs = find_sccs(sub_graph) - - # Build the initial inter-file dependency graph from migrated-entity - # cross-references (before any helper placement). This is the baseline for - # the cycle-aware candidate selection below. - file_deps: Dict[str, Set[str]] = {f: set() for f in file_entity_names} - for target_file, ent_names in file_entity_names.items(): - for ent_name in ent_names: - src = entity_source_map.get(ent_name, "") - for ref_name in _collect_name_loads(src): - dep_file = name_to_target_file.get(ref_name) - if ( - dep_file - and dep_file != target_file - and dep_file in file_entity_names - ): - file_deps[target_file].add(dep_file) - - synthetic_placements: List[GroupPlacement] = [] - for scc in sccs: - # Union of wanting-sets across this helper SCC. - scc_wanting: Set[str] = set() - for name in scc: - scc_wanting.update(wanting.get(name, set())) - - # Sort candidates by topological depth (deepest / most-downstream first). - # For a DAG the deepest wanting file is always cycle-free on the first try, - # eliminating trial-and-error. Depths are recomputed after each placement - # because file_deps grows as helpers are extracted. - topo_depth = _topo_depth(file_deps) - candidates = sorted(scc_wanting, key=lambda t: topo_depth.get(t, 0)) - chosen: Optional[str] = None - for candidate in candidates: - trial_deps: Dict[str, Set[str]] = { - f: set(deps) for f, deps in file_deps.items() - } - for wanting_file in scc_wanting: - if wanting_file != candidate: - trial_deps[wanting_file].add(candidate) - for helper_name in scc: - src = entity_source_map.get(helper_name, "") - for ref_name in _collect_name_loads(src): - dep_file = name_to_target_file.get(ref_name) - if ( - dep_file - and dep_file != candidate - and dep_file in file_entity_names - ): - trial_deps[candidate].add(dep_file) - if not any(len(s) > 1 for s in find_sccs(trial_deps)): - chosen = candidate - break - - if chosen is None: - continue # No cycle-free placement — leave helpers in original file. - - # Apply the chosen placement: update file_deps for subsequent SCC decisions. - for wanting_file in scc_wanting: - if wanting_file != chosen: - file_deps[wanting_file].add(chosen) - for helper_name in scc: - src = entity_source_map.get(helper_name, "") - for ref_name in _collect_name_loads(src): - dep_file = name_to_target_file.get(ref_name) - if dep_file and dep_file != chosen and dep_file in file_entity_names: - file_deps[chosen].add(dep_file) - - # Prepend extracted helpers so they appear before the functions that use them. - file_entity_names[chosen] = list(scc) + file_entity_names[chosen] - for entity_name in scc: - migrated_names.add(entity_name) - entity = entity_map[entity_name] - for defined_name in entity.names_defined: - name_to_target_file[defined_name] = chosen - synthetic_placements.append(GroupPlacement(group=list(scc), target_file=chosen)) - return synthetic_placements - - -def _prune_inline_redundant_imports(source: str) -> str: - """Remove function-body imports that duplicate module-level imports. - - When a function-local ``from x import y`` re-imports a name that is - already provided by a top-level import, flake8 reports an F811 - redefinition warning. This function removes such redundant inner imports - (or narrows them when only some names are redundant). - - Returns *source* unchanged when it cannot be parsed or nothing needs - pruning. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return source - - # Names already available from top-level (module-level) imports. - top_level_names: Set[str] = set() - top_level_node_ids: Set[int] = set() - for node in tree.body: - if isinstance(node, ast.Import): - top_level_node_ids.add(id(node)) - for alias in node.names: - top_level_names.add( - alias.asname if alias.asname else alias.name.split(".")[0] - ) - elif isinstance(node, ast.ImportFrom): - top_level_node_ids.add(id(node)) - for alias in node.names: - top_level_names.add(alias.asname if alias.asname else alias.name) - - if not top_level_names: - return source - - # Collect import node IDs inside module-level 'if TYPE_CHECKING:' blocks. - # These are intentional type-checking guards and must not be treated as - # redundant even when the same name is already imported at module level — - # removing them would leave an empty (and therefore invalid) if-block. - tc_guard_import_ids: Set[int] = set() - for node in tree.body: - if ( - isinstance(node, ast.If) - and isinstance(node.test, ast.Name) - and node.test.id == "TYPE_CHECKING" - ): - for child in ast.walk(node): - if isinstance(child, (ast.Import, ast.ImportFrom)): - tc_guard_import_ids.add(id(child)) - - # Find all import nodes that are NOT at module level and NOT inside a - # module-level 'if TYPE_CHECKING:' guard. - inner_imports = [ - node - for node in ast.walk(tree) - if isinstance(node, (ast.Import, ast.ImportFrom)) - and id(node) not in top_level_node_ids - and id(node) not in tc_guard_import_ids - ] - - if not inner_imports: - return source - - lines = source.splitlines(keepends=True) - # Maps 1-based line number → replacement line (None = remove that line). - line_ops: Dict[int, Optional[str]] = {} - - for stmt in inner_imports: - if isinstance(stmt, ast.Import): - kept = [ - a - for a in stmt.names - if (a.asname if a.asname else a.name.split(".")[0]) - not in top_level_names - ] - else: - kept = [ - a - for a in stmt.names - if (a.asname if a.asname else a.name) not in top_level_names - ] - - if len(kept) == len(stmt.names): - continue # no redundancy — nothing to remove - - # Mark every line of this import for removal. - for ln in range(stmt.lineno, stmt.end_lineno + 1): - line_ops[ln] = None - - if kept: - # Rebuild a narrowed import preserving original indentation. - alias_strs = [ - f"{a.name} as {a.asname}" if a.asname else a.name for a in kept - ] - orig_line = lines[stmt.lineno - 1] - indent = orig_line[: len(orig_line) - len(orig_line.lstrip())] - if isinstance(stmt, ast.ImportFrom): - dots = "." * (stmt.level or 0) - mod = stmt.module or "" - new_line = f"{indent}from {dots}{mod} import {', '.join(alias_strs)}\n" - else: - new_line = f"{indent}import {', '.join(alias_strs)}\n" - line_ops[stmt.lineno] = new_line - - if not line_ops: - return source - - result: List[str] = [] - for i, line in enumerate(lines, 1): - if i in line_ops: - repl = line_ops[i] - if repl is not None: - result.append(repl) - # else: None → line is removed - else: - result.append(line) - return "".join(result) - - -def _prune_unused_imports(source: str) -> str: - """Remove or narrow unused imports in a generated file. - - ``from __future__`` and star imports are always preserved. Multi-name - imports are narrowed to only the names actually referenced in *source* - rather than dropped wholesale. Fully-unused imports are removed entirely. - - Returns *source* unchanged when it cannot be parsed or nothing needs - pruning. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return source - - used = _collect_name_loads(source) - lines = source.splitlines(keepends=True) - # Maps 1-based line number → replacement line (None = remove that line). - replacements: Dict[int, Optional[str]] = {} - - for node in tree.body: - if not isinstance(node, (ast.Import, ast.ImportFrom)): - continue - - # Always preserve __future__ imports. - if isinstance(node, ast.ImportFrom) and node.module == "__future__": - continue - - # Always preserve star imports. - if isinstance(node, ast.ImportFrom) and any(a.name == "*" for a in node.names): - continue - - # Preserve intentional re-export stubs added by _add_re_exports. - # These carry "# noqa: F401" and must not be pruned even when the - # name is no longer referenced in the file body — they exist solely - # to keep the module's public/private API intact for external callers. - import_lines = lines[node.lineno - 1 : node.end_lineno] - if any("noqa: F401" in line for line in import_lines): - continue - - kept = [ - a - for a in node.names - if (a.asname if a.asname else a.name.split(".")[0]) in used - ] - - if len(kept) == len(node.names): - continue # nothing to prune - - # Mark every line of this import for removal. - for ln in range(node.lineno, node.end_lineno + 1): - replacements[ln] = None - - if not kept: - continue # fully unused — all lines already removed - - # Rebuild a single-line import with only the kept aliases. - alias_strs = [f"{a.name} as {a.asname}" if a.asname else a.name for a in kept] - if isinstance(node, ast.ImportFrom): - level_dots = "." * (node.level or 0) - module = node.module or "" - new_line = f"from {level_dots}{module} import {', '.join(alias_strs)}\n" - else: - new_line = f"import {', '.join(alias_strs)}\n" - replacements[node.lineno] = new_line - - if not replacements: - return source - - result: List[str] = [] - for i, line in enumerate(lines, 1): - if i not in replacements: - result.append(line) - elif replacements[i] is not None: - result.append(replacements[i]) - # else: line is removed — skip it - return "".join(result) - - -def _strip_top_level_import_lines(src: str) -> str: - """Return *src* with all top-level import statements removed. - - Also removes module-level ``if TYPE_CHECKING:`` blocks, since their - imports are now redistributed to each sub-file via the import-info - system and emitting the block verbatim would produce the wrong relative - import path and/or an unused import in the wrong sub-file. - - Uses AST to locate the exact line range of each import node, correctly - handling multi-line imports. Returns *src* unchanged when it cannot be - parsed as Python. - """ - try: - tree = ast.parse(src) - except SyntaxError: - return src - remove: Set[int] = set() - for node in tree.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - for ln in range(node.lineno, node.end_lineno + 1): - remove.add(ln) - elif ( - isinstance(node, ast.If) - and isinstance(node.test, ast.Name) - and node.test.id == "TYPE_CHECKING" - ): - for ln in range(node.lineno, node.end_lineno + 1): - remove.add(ln) - if not remove: - return src - lines = src.splitlines(keepends=True) - return "".join(line for i, line in enumerate(lines, 1) if i not in remove) - - -def _extract_module_docstring(source: str) -> Optional[str]: - """Return the module-level docstring source text, or None if absent.""" - try: - tree = ast.parse(source) - except SyntaxError: - return None - if not ( - tree.body - and isinstance(tree.body[0], ast.Expr) - and isinstance(tree.body[0].value, ast.Constant) - and isinstance(tree.body[0].value.value, str) - ): - return None - node = tree.body[0] - lines = source.splitlines(keepends=True) - return "".join(lines[node.lineno - 1 : node.end_lineno]).rstrip() - - -def _strip_module_docstring(src: str) -> str: - """Return *src* with the leading module-level docstring removed.""" - try: - tree = ast.parse(src) - except SyntaxError: - return src - if not ( - tree.body - and isinstance(tree.body[0], ast.Expr) - and isinstance(tree.body[0].value, ast.Constant) - and isinstance(tree.body[0].value.value, str) - ): - return src - node = tree.body[0] - remove = set(range(node.lineno, node.end_lineno + 1)) - lines = src.splitlines(keepends=True) - return "".join(line for i, line in enumerate(lines, 1) if i not in remove) - - -def _source_is_only_docstring(source: str) -> bool: - """Return True if *source* contains only a module-level docstring.""" - try: - tree = ast.parse(source) - except SyntaxError: - return False - return ( - len(tree.body) == 1 - and isinstance(tree.body[0], ast.Expr) - and isinstance(tree.body[0].value, ast.Constant) - and isinstance(tree.body[0].value.value, str) - ) - - -# --------------------------------------------------------------------------- -# __main__ handling -# --------------------------------------------------------------------------- - - -def _is_test_name(name: str) -> bool: - """Return True if *name* matches pytest's test-discovery patterns. - - Pytest collects classes named ``Test*`` and functions named ``test_*``. - Importing such names at module level in a test file causes every test - inside to be discovered — and run — a second time. - """ - return name.startswith("Test") or name.startswith("test_") - - -def _is_pytest_fixture(entity_src: str) -> bool: - """Return True if *entity_src* defines a function with a @pytest.fixture decorator. - - Handles all common forms: ``@fixture``, ``@fixture()``, ``@pytest.fixture``, - and ``@pytest.fixture(scope=...)``. - """ - try: - tree = ast.parse(entity_src) - except SyntaxError: - return False - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - for dec in node.decorator_list: - # Unwrap call forms like @pytest.fixture(...) to get the base reference. - ref = dec.func if isinstance(dec, ast.Call) else dec - if isinstance(ref, ast.Name) and ref.id == "fixture": - return True - if isinstance(ref, ast.Attribute) and ref.attr == "fixture": - return True - return False - - -def _file_has_only_fixtures(source: str) -> bool: - """Return True if *source* has at least one @pytest.fixture and nothing else. - - "Nothing else" means no test functions (``def test_*``), no test classes - (``class Test*``), no other function/class definitions, and no non-import - module-level statements other than a module docstring. Import statements - and a leading docstring are allowed because they are needed to support the - fixture definitions. - - Returns False on syntax errors (be conservative). - """ - try: - tree = ast.parse(source) - except SyntaxError: - return False - lines = source.splitlines(keepends=True) - has_fixture = False - for node in tree.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - continue - if isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant): - continue # module docstring or standalone string literal - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - first_line = ( - node.decorator_list[0].lineno if node.decorator_list else node.lineno - ) - fn_src = "".join(lines[first_line - 1 : node.end_lineno]).rstrip() - if _is_pytest_fixture(fn_src): - has_fixture = True - continue - return False # non-fixture function (including test_ functions) - return False # class, assignment, or other statement - return has_fixture - - -def _split_cross_imports_by_test( - imports: List[str], -) -> Tuple[List[str], List[str]]: - """Split cross-file import statements into (non_test, test_named) groups. - - Import statements that name pytest-discoverable symbols (``Test*`` or - ``test_*``) are returned as inline imports so callers can inject them - into function/class bodies rather than emitting them at module level. - Mixed imports (some test, some non-test names) are split into two - separate statements. - """ - non_test: List[str] = [] - test_named: List[str] = [] - for imp in imports: - m = re.match(r"^(from\s+\S+\s+import\s+)(.*)", imp) - if not m: - non_test.append(imp) - continue - prefix = m.group(1) - names = [n.strip() for n in m.group(2).split(",")] - t_names = sorted(n for n in names if _is_test_name(n)) - nt_names = sorted(n for n in names if not _is_test_name(n)) - if t_names: - test_named.append(f"{prefix}{', '.join(t_names)}") - if nt_names: - non_test.append(f"{prefix}{', '.join(nt_names)}") - return non_test, test_named - - -def _inject_inline_imports(entity_src: str, imports: List[str]) -> str: - """Inject *imports* at the top of a function or class body in *entity_src*. - - The imports are inserted after any leading docstring. Returns - *entity_src* unchanged when it cannot be parsed or the top-level node - is not a function or class (TOP_LEVEL entities have no body scope). - """ - if not imports: - return entity_src - try: - tree = ast.parse(entity_src) - except SyntaxError: - return entity_src - if not tree.body: - return entity_src - top = tree.body[0] - if not isinstance(top, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - return entity_src - first_stmt = top.body[0] - insert_line = first_stmt.lineno - if ( - isinstance(first_stmt, ast.Expr) - and isinstance(first_stmt.value, ast.Constant) - and isinstance(first_stmt.value.value, str) - and len(top.body) > 1 - ): - insert_line = top.body[1].lineno - lines = entity_src.splitlines(keepends=True) - body_line = lines[insert_line - 1] - indent = body_line[: len(body_line) - len(body_line.lstrip())] - import_lines = [f"{indent}{imp}\n" for imp in imports] - return "".join(lines[: insert_line - 1] + import_lines + lines[insert_line - 1 :]) - - -def _find_main_block_entity( - entities: List[Entity], - entity_source_map: Dict[str, str], -) -> Optional[str]: - """Return the entity name of the ``if __name__ == '__main__':`` block. - - Returns ``None`` when no such block is present. - """ - for entity in entities: - if entity.kind != EntityKind.TOP_LEVEL: - continue - src = entity_source_map.get(entity.name, "") - try: - tree = ast.parse(src) - except SyntaxError: - continue - for node in tree.body: - if ( - isinstance(node, ast.If) - and isinstance(node.test, ast.Compare) - and isinstance(node.test.left, ast.Name) - and node.test.left.id == "__name__" - and len(node.test.ops) == 1 - and isinstance(node.test.ops[0], ast.Eq) - and len(node.test.comparators) == 1 - and isinstance(node.test.comparators[0], ast.Constant) - and node.test.comparators[0].value == "__main__" - ): - return entity.name - return None - - -def _find_main_direct_callees( - main_src: str, function_entity_names: Set[str] -) -> Set[str]: - """Return function entity names called directly in the ``__main__`` block. - - Only names that appear in *function_entity_names* (i.e. are defined as - top-level FUNCTION entities in the same file) are returned, so the - caller can keep those functions sticky to the original file alongside - the ``__main__`` block. - """ - try: - tree = ast.parse(main_src) - except SyntaxError: - return set() - callees: Set[str] = set() - for node in tree.body: - if not ( - isinstance(node, ast.If) - and isinstance(node.test, ast.Compare) - and isinstance(node.test.left, ast.Name) - and node.test.left.id == "__name__" - and len(node.test.ops) == 1 - and isinstance(node.test.ops[0], ast.Eq) - and len(node.test.comparators) == 1 - and isinstance(node.test.comparators[0], ast.Constant) - and node.test.comparators[0].value == "__main__" - ): - continue - for subnode in ast.walk(node): - if ( - isinstance(subnode, ast.Call) - and isinstance(subnode.func, ast.Name) - and subnode.func.id in function_entity_names - ): - callees.add(subnode.func.id) - return callees - - -def _inject_inline_test_imports_original( - source: str, - migrated_test_symbols: Dict[str, str], - abs_pkg: Optional[str], - original_basename: str, -) -> str: - """Inject inline imports for migrated test-named symbols into function/class bodies. - - After a split, test-named symbols (``Test*`` / ``test_*``) that were - migrated to new files are not re-exported at module level (to avoid - pytest double-discovery). This function finds every top-level - function or class in *source* that still references such symbols and - injects the required ``from … import …`` statement at the top of - each body, after any docstring. - - *migrated_test_symbols* maps each migrated test name to its target - file (relative path). *abs_pkg* and *original_basename* are used to - build the correct import prefix (absolute for test files, relative - otherwise). - """ - if not migrated_test_symbols: - return source - try: - tree = ast.parse(source) - except SyntaxError: - return source - - lines = source.splitlines(keepends=True) - # Maps 1-based line number → list of import lines to insert before it. - insertions: Dict[int, List[str]] = {} - - for node in tree.body: - if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - continue - body_names: Set[str] = set() - for subnode in ast.walk(node): - if isinstance(subnode, ast.Name) and isinstance(subnode.ctx, ast.Load): - body_names.add(subnode.id) - needed: Dict[str, List[str]] = {} - for name in body_names: - tfile = migrated_test_symbols.get(name) - if tfile: - needed.setdefault(tfile, []).append(name) - if not needed: - continue - import_stmts: List[str] = [] - for tfile, names in sorted(needed.items()): - if abs_pkg is not None: - mod = _target_module_name(tfile) - prefix = f"{abs_pkg}.{mod}" if abs_pkg else mod - else: - prefix = _relative_import_prefix(original_basename, tfile) - import_stmts.append(f"from {prefix} import {', '.join(sorted(names))}") - first_stmt = node.body[0] - insert_line = first_stmt.lineno - if ( - isinstance(first_stmt, ast.Expr) - and isinstance(first_stmt.value, ast.Constant) - and isinstance(first_stmt.value.value, str) - and len(node.body) > 1 - ): - insert_line = node.body[1].lineno - body_line = lines[insert_line - 1] - indent = body_line[: len(body_line) - len(body_line.lstrip())] - insertions.setdefault(insert_line, []) - insertions[insert_line] = [f"{indent}{s}\n" for s in import_stmts] + insertions[ - insert_line - ] - - if not insertions: - return source - result: List[str] = [] - for i, line in enumerate(lines, 1): - if i in insertions: - result.extend(insertions[i]) - result.append(line) - return "".join(result) - - -# --------------------------------------------------------------------------- -# Conftest merging -# --------------------------------------------------------------------------- - - -def _merge_conftest_sources(existing: str, new_content: str) -> str: - """Merge *new_content* into an existing conftest.py without duplicating anything. - - When multiple file splits each contribute fixtures to the same conftest.py, - naively appending produces duplicate import statements, duplicate function - definitions, and E402 errors (imports after function definitions). - - This function avoids all three: - - Duplicate import statements (same module + same names) are skipped. - - Function/class definitions whose names already appear in *existing* are skipped. - - Non-duplicate imports from *new_content* are inserted after the last existing - import (before any existing function definitions), preventing E402. - - Non-duplicate definitions are appended at the end. - - Falls back to simple concatenation when either source cannot be parsed. - """ - try: - existing_tree = ast.parse(existing) - new_tree = ast.parse(new_content) - except SyntaxError: - return existing.rstrip() + "\n\n\n" + new_content - - existing_lines = existing.splitlines(keepends=True) - new_lines = new_content.splitlines(keepends=True) - - def _import_key(node: ast.stmt) -> str: - if isinstance(node, ast.Import): - return "I:" + ",".join( - sorted(f"{a.name}:{a.asname or ''}" for a in node.names) - ) - assert isinstance(node, ast.ImportFrom) - dots = "." * (node.level or 0) - mod = node.module or "" - return ( - "F:" - + dots - + mod - + ":" - + ",".join(sorted(f"{a.name}:{a.asname or ''}" for a in node.names)) - ) - - # What is already in existing? - existing_import_keys: Set[str] = { - _import_key(n) - for n in existing_tree.body - if isinstance(n, (ast.Import, ast.ImportFrom)) - } - existing_defined_names: Set[str] = { - n.name - for n in existing_tree.body - if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) - } - - # Last import line in existing (0-indexed insertion point). - last_import_lineno: int = 0 - for n in existing_tree.body: - if isinstance(n, (ast.Import, ast.ImportFrom)): - last_import_lineno = max(last_import_lineno, n.end_lineno) - - # Collect new, non-duplicate imports and definitions from new_content. - imports_to_insert: List[str] = [] - defs_to_append: List[str] = [] - - for node in new_tree.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - if _import_key(node) not in existing_import_keys: - src = "".join(new_lines[node.lineno - 1 : node.end_lineno]).rstrip() - imports_to_insert.append(src) - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - if node.name not in existing_defined_names: - first_line = ( - node.decorator_list[0].lineno - if node.decorator_list - else node.lineno - ) - src = "".join(new_lines[first_line - 1 : node.end_lineno]).rstrip() - defs_to_append.append(src) - - if not imports_to_insert and not defs_to_append: - return existing - - result_lines = list(existing_lines) - if imports_to_insert: - # Insert new imports directly after the last existing import line. - insert_at = last_import_lineno # 0-indexed position after last import - new_import_lines = [imp + "\n" for imp in imports_to_insert] - result_lines = ( - result_lines[:insert_at] + new_import_lines + result_lines[insert_at:] - ) - - result = "".join(result_lines).rstrip() - if defs_to_append: - result = result + "\n\n\n" + "\n\n\n".join(defs_to_append) + "\n" - else: - result = result + "\n" - return result - - -def _rewrite_module_var_names(src: str, rewrites: Dict[str, str]) -> str: - """Replace bare ``Name`` loads with ``module.name`` attribute accesses. - - Uses the AST to locate exact positions of ``Name`` load nodes whose - ``id`` is in *rewrites*, replacing each with its qualified form (e.g. - ``"SAFE_MODE"`` → ``"conversion.SAFE_MODE"``). - - Because ``ast.Name`` nodes **never** represent the attribute part of an - ``Attribute`` node (which stores ``attr`` as a plain string), this - approach is immune to the corruption that a regex would cause on - ``obj.SAFE_MODE`` and naturally skips string literals and comments. - - After rewriting, the result is re-parsed and every original ``Name`` - load for each rewritten identifier is verified to be absent. If - verification fails the original source is returned unchanged so that - callers can fall back to direct-import semantics rather than corrupt - the output. - """ - if not rewrites: - return src - try: - tree = ast.parse(src) - except SyntaxError: - return src - - lines = src.splitlines(keepends=True) - - # Collect (lineno, col_offset, end_col_offset, new_text). - # ast uses 1-indexed lineno and 0-indexed col_offset / end_col_offset. - edits: List[Tuple[int, int, int, str]] = [] - for node in ast.walk(tree): - if ( - isinstance(node, ast.Name) - and isinstance(node.ctx, ast.Load) - and node.id in rewrites - ): - edits.append( - (node.lineno, node.col_offset, node.end_col_offset, rewrites[node.id]) - ) - - if not edits: - return src - - # Apply edits from last to first within each line to keep earlier offsets valid. - edits.sort(key=lambda e: (e[0], e[1]), reverse=True) - for lineno, col_start, col_end, new_text in edits: - line = lines[lineno - 1] - lines[lineno - 1] = line[:col_start] + new_text + line[col_end:] - - result = "".join(lines) - - # Verification: re-parse and confirm no bare Name loads remain for any - # rewritten identifier. If the result is unparseable or a bare name - # survives, return the original source to avoid corrupting the output. - try: - new_tree = ast.parse(result) - except SyntaxError: - return src - for node in ast.walk(new_tree): - if ( - isinstance(node, ast.Name) - and isinstance(node.ctx, ast.Load) - and node.id in rewrites - ): - return src - - return result - - -def _rewrite_module_level_stores(src: str, rewrites: Dict[str, str]) -> str: - """Rewrite module-level Name store targets to ``module.name`` attribute stores. - - Only statements at the top level of the module are affected - (``ast.Module.body``). Assignments inside function or class bodies are - left unchanged so that local variable bindings are not corrupted. - - Used for the non-migrated home file: when a non-migrated entity reassigns - a TOP_LEVEL constant that was moved to a sub-file, the assignment must be - rewritten as ``module.CONST = expr`` so that the mutation updates the - canonical value in the sub-file rather than creating an orphaned local - binding. - """ - if not rewrites: - return src - try: - tree = ast.parse(src) - except SyntaxError: - return src - lines = src.splitlines(keepends=True) - edits: List[Tuple[int, int, int, str]] = [] - for node in tree.body: - if isinstance(node, ast.Assign): - for target in node.targets: - if isinstance(target, ast.Name) and target.id in rewrites: - edits.append( - ( - target.lineno, - target.col_offset, - target.end_col_offset, - rewrites[target.id], - ) - ) - elif isinstance(node, ast.AugAssign): - if isinstance(node.target, ast.Name) and node.target.id in rewrites: - edits.append( - ( - node.target.lineno, - node.target.col_offset, - node.target.end_col_offset, - rewrites[node.target.id], - ) - ) - elif isinstance(node, ast.AnnAssign): - if ( - node.value is not None - and isinstance(node.target, ast.Name) - and node.target.id in rewrites - ): - edits.append( - ( - node.target.lineno, - node.target.col_offset, - node.target.end_col_offset, - rewrites[node.target.id], - ) - ) - if not edits: - return src - edits.sort(key=lambda e: (e[0], e[1]), reverse=True) - for lineno, col_start, col_end, new_text in edits: - line = lines[lineno - 1] - lines[lineno - 1] = line[:col_start] + new_text + line[col_end:] - return "".join(lines) - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - - -def generate_file_splits( - classified: ClassifiedEntities, - plan: FileLimiterPlan, - post_source: str, - original_path: str, - subdir_name: Optional[str] = None, - pytest_conftest: bool = False, - has_main: bool = False, - reexport_mode: str = "always", -) -> SplitResult: - """Generate new file contents and the updated original source. - - When *plan* is aborted or has no placements, returns :class:`SplitResult` - with the original source unchanged (``abort`` mirrors ``plan.abort``). - - When *subdir_name* is set (e.g. ``"service"``), the file is being split - into a package subdirectory. The "original" file is treated as - ``service/__init__.py`` for dependency-graph and import-prefix purposes, - so cross-file imports within the package use correct relative paths. - - When *pytest_conftest* is True, any entity decorated with - ``@pytest.fixture`` (or ``@fixture``) is redirected to ``conftest.py`` - instead of the LLM-assigned target file. pytest auto-discovers fixtures - from ``conftest.py``, so no re-export import is added to the original - file — eliminating both the F401 and F811 flake8 warnings that arise - when a fixture name is used as a test function parameter. - """ - if plan.abort: - return SplitResult( - new_files={}, - original_source=post_source, - abort=True, - abort_reason=plan.abort_reason, - ) - - if not plan.placements: - return SplitResult(new_files={}, original_source=post_source, abort=False) - - # Detect shebang on line 1 so it can be stripped from new files and - # preserved (or restored) at the top of the original. - shebang: Optional[str] = None - if post_source.startswith("#!"): - nl = post_source.find("\n") - shebang = post_source[: nl + 1] if nl != -1 else post_source + "\n" - - lines = post_source.splitlines(keepends=True) - entity_map = {e.name: e for e in classified.entities} - - # Build entity source map (name → stripped source string). - entity_source_map: Dict[str, str] = {} - for entity in classified.entities: - entity_source_map[entity.name] = "".join( - lines[entity.start_line - 1 : entity.end_line] - ).rstrip() - - # All entity-defined names (used to limit import matching scope). - all_entity_names: Set[str] = { - name for e in classified.entities for name in e.names_defined - } - - # Extract import info from post-refactor source. - import_infos = _extract_import_info(post_source) - - # Placements whose target_file matches the original filename would create a - # self-referential import (e.g. `from .duplicate_extractor import Foo` inside - # duplicate_extractor.py). Drop them — entities stay in the original file. - # In subdir-split mode the "original" is the package __init__.py; use that - # name throughout so dependency-graph edges and import prefixes are correct. - original_basename = ( - f"{subdir_name}/__init__.py" - if subdir_name and not has_main - else Path(original_path).name - ) - is_test_file = Path(original_path).name.startswith("test_") - # For test-file subdir splits the original test file stays on disk (runner.py - # does not redirect it to __init__.py), so non-migrated names still live in - # the original file (e.g. "test_runner.py"), not in the package __init__.py. - non_migrated_home = ( - Path(original_path).name - if (subdir_name and is_test_file) - else original_basename - ) - # Identify the __main__ block and any functions it calls directly. - # These stay in the original file unconditionally: the __main__ block - # is an entry point the user expects to keep working, and its direct - # callees must live in the same file to avoid module-level test-class - # imports that would cause pytest double-discovery. - main_entity = _find_main_block_entity(classified.entities, entity_source_map) - main_sticky: Set[str] = set() - if main_entity is not None: - main_sticky.add(main_entity) - function_entity_names = { - e.name for e in classified.entities if e.kind == EntityKind.FUNCTION - } - main_sticky.update( - _find_main_direct_callees( - entity_source_map.get(main_entity, ""), function_entity_names - ) - ) - - valid_placements = [ - p - for p in plan.placements - if p.target_file != original_basename - and not any(name in main_sticky for name in p.group) - ] - - # --- Pytest conftest routing --- - # When enabled, entities decorated with @pytest.fixture are redirected to - # conftest.py instead of the LLM-assigned target file. pytest discovers - # fixtures from conftest.py automatically, so no re-export import is added - # to the original file — eliminating the F401/F811 flake8 false positives. - # - # Exception: if conftest.py already defines a function with the same name - # (e.g. a default fixture that this test file overrides), routing the entity - # there would cause _merge_conftest_sources to silently drop the new version - # (keeping the old one), so the entity would disappear from the split output - # entirely and verification would fail. In that case the fixture stays in - # its LLM-assigned target file; re-exports are still suppressed to avoid - # F401/F811 (the fixture is injected by pytest name-lookup, not by import). - # For subdir splits, route fixtures into the subdirectory's own conftest.py - # so that multiple test files in the same parent directory each get an - # isolated conftest scope and cannot overwrite each other's fixtures. - # Exception to the exception: if the fixture is still referenced in entities - # that remain in the original file (i.e. tests that were not migrated), route - # it to the parent conftest.py instead so those tests can find it. Tests in - # the subdirectory also inherit from the parent conftest, so this is safe. - # Further exception: if the fixture is referenced in remaining source AND the - # parent conftest already has a fixture with the same name (the module was - # overriding it), merging into parent conftest would silently keep the old - # version — the original test would get the wrong fixture and the migrated - # subdir tests would also inherit the wrong version. In that case, copy - # (don't move) the fixture to the subdir conftest so migrated tests get the - # override; the entity is also kept in the original file so the original - # test can discover it directly from its own module. - conftest_target = f"{subdir_name}/conftest.py" if subdir_name else "conftest.py" - parent_conftest_target = "conftest.py" - existing_conftest_path = ( - Path(original_path).parent / subdir_name / "conftest.py" - if subdir_name - else Path(original_path).parent / "conftest.py" - ) - fixture_entity_names: Set[str] = set() - # Names of fixtures kept in their LLM-assigned file because the target - # conftest.py already defines a symbol with that name. Fixtures injected - # by pytest name-lookup need no re-export import in the original file. - conftest_conflict_names: Set[str] = set() - # Names of fixtures that are copied to the subdir conftest but also kept in - # the original file (not removed). This handles the case where the fixture - # overrides a parent conftest fixture and is also referenced in remaining - # entities: the subdir copy ensures migrated tests see the override; the - # original file copy ensures the original test's module-level fixture takes - # precedence over the stale parent conftest version. - copy_not_migrate: Set[str] = set() - if pytest_conftest: - existing_conftest_names: Set[str] = set() - if existing_conftest_path.exists(): - try: - _ec_tree = ast.parse(existing_conftest_path.read_text(encoding="utf-8")) - for _ec_node in _ec_tree.body: - if isinstance( - _ec_node, - (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef), - ): - existing_conftest_names.add(_ec_node.name) - except (SyntaxError, OSError): - pass - - # For subdir splits: build source of entities that will remain in the - # original file so we can detect fixtures still needed by those entities. - # Also load parent conftest names to detect fixtures that override a - # parent-level fixture (used below to avoid silently keeping the old - # parent version when merging would drop the new override). - _migrating_names: Set[str] = { - name for p in valid_placements for name in p.group - } - _remaining_src = "\n".join( - entity_source_map[e.name] - for e in classified.entities - if e.name not in _migrating_names and e.name in entity_source_map - ) - _parent_conftest_names: Set[str] = set() - if subdir_name: - _parent_conftest_path = Path(original_path).parent / "conftest.py" - if _parent_conftest_path.exists(): - try: - _pc_tree = ast.parse( - _parent_conftest_path.read_text(encoding="utf-8") - ) - for _pc_node in _pc_tree.body: - if isinstance( - _pc_node, - (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef), - ): - _parent_conftest_names.add(_pc_node.name) - except (SyntaxError, OSError): - pass - - conftest_group: List[str] = [] - conftest_group_parent: List[str] = [] - new_valid: List[GroupPlacement] = [] - for p in valid_placements: - non_fixture: List[str] = [] - for name in p.group: - src = entity_source_map.get(name, "") - if src and _is_pytest_fixture(src): - fixture_entity_names.add(name) - if name in existing_conftest_names: - # conftest already has this name: keep in the - # LLM-assigned file to avoid losing the entity. - # No re-export needed — pytest discovers fixtures - # by name-lookup, not by import. - non_fixture.append(name) - conftest_conflict_names.add(name) - elif subdir_name and re.search( - r"\b" + re.escape(name) + r"\b", _remaining_src - ): - # Fixture still used by non-migrated tests. - if name in _parent_conftest_names: - # Parent conftest already has this name: the module - # was overriding it. Merging into parent conftest - # would silently keep the old version. Instead, - # copy (don't move) to subdir conftest — migrated - # tests get the override via the subdir conftest; - # the entity stays in the original file so the - # original test discovers the override from its own - # module rather than the stale parent conftest entry. - conftest_group.append(name) - copy_not_migrate.add(name) - else: - # No conflict: route to the parent conftest.py so - # both original and subdir tests can find it. - conftest_group_parent.append(name) - else: - conftest_group.append(name) - else: - non_fixture.append(name) - if non_fixture: - new_valid.append( - GroupPlacement(group=non_fixture, target_file=p.target_file) - ) - if conftest_group: - new_valid.append( - GroupPlacement(group=conftest_group, target_file=conftest_target) - ) - if conftest_group_parent: - new_valid.append( - GroupPlacement( - group=conftest_group_parent, target_file=parent_conftest_target - ) - ) - valid_placements = new_valid - - # Group placements by target file (preserving order for topo sort). - file_entity_names: Dict[str, List[str]] = {} - for placement in valid_placements: - file_entity_names.setdefault(placement.target_file, []).extend(placement.group) - - # All migrated entity names. - migrated_names: Set[str] = {name for p in valid_placements for name in p.group} - - # Build name → target-file map for cross-file import detection. - # Exclude import-derived names (_find_needed_imports handles those). - import_defined_names = {name for info in import_infos for name in info.names} - name_to_target_file: Dict[str, str] = {} - for target_file, ent_names in file_entity_names.items(): - for ent_name in ent_names: - entity = entity_map.get(ent_name) - if entity: - for defined_name in entity.names_defined: - if defined_name not in import_defined_names: - name_to_target_file[defined_name] = target_file - - # Also map names from non-migrated entities to the original file so that - # split files can import helpers (e.g. _run) that stayed behind. - for entity in classified.entities: - if entity.name not in migrated_names: - for defined_name in entity.names_defined: - if defined_name not in import_defined_names: - name_to_target_file.setdefault(defined_name, non_migrated_home) - - # For TOP_LEVEL names that are reassigned (stored) by a *different* entity, - # cross-file references must use ``module.NAME`` so that the mutation is - # visible to all importers. Names that are only ever defined once and never - # mutated elsewhere use a plain ``from .module import NAME`` — the idiomatic - # Python form — since the value is stable after import. - _top_level_def_entity: Dict[str, str] = { - defined_name: entity.name - for entity in classified.entities - if entity.kind == EntityKind.TOP_LEVEL - for defined_name in entity.names_defined - if defined_name not in import_defined_names - } - top_level_var_names: Set[str] = { - name - for name, def_ent in _top_level_def_entity.items() - for ent_name, src in entity_source_map.items() - if ent_name != def_ent and name in _collect_name_stores(src) - } - - # Extract non-migrated FUNCTION/CLASS entities referenced by migrated ones - # into the new files that use them, breaking O→F→O import cycles. - synthetic_placements = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - non_migrated_home, - ) - - # Collect names that external files (outside the module being split) import - # from the original file. Private symbols in this set must get a re-export - # proxy even though they are no longer referenced by the remaining source. - external_loads = _collect_external_imported_names(original_path) - - # Detect circular imports. Cycles can pass through the original file: - # a new file that imports a non-migrated name from the original can form a - # chain back to the original via the re-exports the original adds. - # Model the original as an explicit node in the dependency graph. - # - # Original's outgoing edges: it will re-export a migrated name when the - # name is public (no _/test_ prefix), referenced by a non-migrated entity, - # or imported by an external file (external_loads). - # - # In a test-file subdir split, non_migrated_home ("test_runner.py") differs - # from original_basename ("runner/__init__.py"). Re-exports are injected - # into the original test file, so it—not the __init__.py—gains outgoing - # import edges and must be a separate node in the dependency graph. - non_migrated_loads: Set[str] = set() - for ent_name, src in entity_source_map.items(): - if ent_name not in migrated_names: - non_migrated_loads |= _collect_name_loads(src) - - reexport_home = non_migrated_home - all_dep_nodes = set(file_entity_names.keys()) | {original_basename, reexport_home} - file_deps: Dict[str, Set[str]] = {node: set() for node in all_dep_nodes} - for target_file, ent_names in file_entity_names.items(): - for ent_name in ent_names: - src = entity_source_map.get(ent_name, "") - for ref_name in _collect_name_loads(src): - dep_file = name_to_target_file.get(ref_name) - if dep_file and dep_file != target_file and dep_file in file_deps: - file_deps[target_file].add(dep_file) - for placement in valid_placements + synthetic_placements: - for ent_name in placement.group: - entity = entity_map.get(ent_name) - if entity: - for defined_name in entity.names_defined: - reexport_unconditionally = ( - not defined_name.startswith("_") - and not defined_name.startswith("test_") - and ( - reexport_mode == "always" - or (reexport_mode == "application" and not is_test_file) - ) - ) - if ( - reexport_unconditionally - or defined_name in non_migrated_loads - or defined_name in external_loads - ): - file_deps[reexport_home].add(placement.target_file) - break - if any(len(scc) > 1 for scc in find_sccs(file_deps)): - return SplitResult( - new_files={}, - original_source=post_source, - abort=True, - abort_reason="proposed split would create circular file imports", - ) - - # Use absolute imports when the original file is a test file. Pytest's - # default import mode loads test files as top-level modules (not package - # members), so relative imports like `from .helpers import foo` would - # raise ImportError at collection time. - abs_pkg: Optional[str] = None - if Path(original_path).name.startswith("test_"): - abs_pkg = _abs_package_for_dir(original_path) - - # In subdir-split mode, new files live inside a package subdirectory and - # can use relative imports for cross-file references within that package. - abs_pkg_for_new_files: Optional[str] = None if subdir_name else abs_pkg - - # Generate new file contents. - new_files: Dict[str, str] = {} - entity_name_rewrites: Dict[str, Dict[str, str]] = {} # per-entity rewrites - for target_file, ent_names in file_entity_names.items(): - needed = _find_needed_imports( - ent_names, entity_source_map, import_infos, all_entity_names - ) - needed_tc = _find_type_checking_needed_imports( - ent_names, entity_source_map, import_infos - ) - if subdir_name is not None: - depth = len(Path(target_file).parts) - 1 - needed = [_bump_relative_imports(s, depth) for s in needed] - needed_tc = [_bump_relative_imports(s, depth) for s in needed_tc] - entity_srcs = [] - top_cross: List[str] = [] - seen_top_cross: Set[str] = set() - all_tc_imports: List[str] = list(needed_tc) - seen_tc: Set[str] = set(needed_tc) - for _ent_name in ent_names: - _src = entity_source_map.get(_ent_name) - if _src is None: - continue - _entity = entity_map.get(_ent_name) - if _entity and _entity.kind == EntityKind.TOP_LEVEL: - # Imports are emitted separately by _find_needed_imports; strip - # them from the entity body to prevent duplicate import stmts. - _src = _strip_top_level_import_lines(_src) - if subdir_name is not None: - # In subdir-split mode the module docstring belongs in - # __init__.py rather than in one of the child modules. - _src = _strip_module_docstring(_src) - else: - _src = _FUTURE_IMPORT_LINE_RE.sub("", _src) - # Strip shebang from any entity that begins on line 1 of the - # original source — it must not appear in generated new files. - if shebang and _entity and _entity.start_line == 1: - nl = _src.find("\n") - _src = _src[nl + 1 :] if nl != -1 else "" - _src = _src.rstrip() - # Compute cross-file imports for this entity and split off any - # test-named symbols (Test* / test_*) to be injected inline. - entity_from, entity_mod, entity_rewrites = _find_cross_file_imports( - [_ent_name], - entity_source_map, - name_to_target_file, - target_file, - abs_pkg=abs_pkg_for_new_files, - top_level_var_names=top_level_var_names, - ) - for _tc_imp in _find_cross_file_type_checking_imports( - [_ent_name], - entity_source_map, - name_to_target_file, - target_file, - abs_pkg=abs_pkg_for_new_files, - top_level_var_names=top_level_var_names, - ): - if _tc_imp not in seen_tc: - seen_tc.add(_tc_imp) - all_tc_imports.append(_tc_imp) - if entity_rewrites: - _src = _rewrite_module_var_names(_src, entity_rewrites) - entity_name_rewrites[_ent_name] = entity_rewrites - # Module imports for TOP_LEVEL vars must always be at module level - # (never inlined) — decorators are evaluated before function bodies run. - for imp in entity_mod: - if imp not in seen_top_cross: - seen_top_cross.add(imp) - top_cross.append(imp) - ent_top_cross, ent_test_imports = _split_cross_imports_by_test(entity_from) - for imp in ent_top_cross: - if imp not in seen_top_cross: - seen_top_cross.add(imp) - top_cross.append(imp) - if ent_test_imports and _entity and _entity.kind != EntityKind.TOP_LEVEL: - # Extract the test-named symbols that would be inlined. - # All items from _split_cross_imports_by_test are "from X import Y" - # form, so partitioning on " import " is safe. - inlined_names: Set[str] = set() - for _imp in ent_test_imports: - _, _, _names_part = _imp.partition(" import ") - for _n in _names_part.split(","): - inlined_names.add(_n.strip()) - # Decorators are evaluated before function bodies, so a symbol - # that only arrives via an inline import will not be in scope. - dec_conflicts = _test_names_in_decorators(_src, inlined_names) - if dec_conflicts: - _names_str = ", ".join(f"'{n}'" for n in sorted(dec_conflicts)) - return SplitResult( - new_files={}, - original_source=post_source, - abort=True, - abort_reason=( - f"cannot split '{_ent_name}': {_names_str} appear(s) " - f"in a decorator but would need to be imported inline " - f"to avoid pytest collecting them as duplicate tests — " - f"keep the dependent test classes in the same file" - ), - ) - _src = _inject_inline_imports(_src, ent_test_imports) - else: - # TOP_LEVEL entity: no body scope, fall back to module level. - for imp in ent_test_imports: - if imp not in seen_top_cross: - seen_top_cross.add(imp) - top_cross.append(imp) - entity_srcs.append(_src) - entity_srcs = [s for s in entity_srcs if s] - # Dedup: remove TC imports for names already covered by regular imports. - # This can happen when one entity uses a name at runtime (→ top_cross) - # while another entity in the same file only uses it in a quoted - # annotation (→ all_tc_imports), producing duplicate import statements. - if all_tc_imports and (needed or top_cross): - _regular_names: Set[str] = set() - for _imp in needed + top_cross: - _m = _FROM_IMPORT_RE.match(_imp) - if _m: - _regular_names.update( - n.strip() for n in _m.group(2).split(",") if n.strip() - ) - _deduped_tc: List[str] = [] - for _tc in all_tc_imports: - _m = _FROM_IMPORT_RE.match(_tc) - if _m: - _tc_names = { - _n.strip() for _n in _m.group(2).split(",") if _n.strip() - } - _leftover = _tc_names - _regular_names - if _leftover: - _deduped_tc.append( - _tc - if _leftover == _tc_names - else _narrow_import_source(_tc, _leftover) - ) - else: - _deduped_tc.append(_tc) - all_tc_imports = _deduped_tc - parts: List[str] = [] - imports_for_sort = list(needed + top_cross) - if all_tc_imports: - imports_for_sort.append("from typing import TYPE_CHECKING") - all_imports = _sort_imports_pep8(_merge_from_imports(imports_for_sort)) - if all_imports: - parts.append("\n".join(all_imports)) - if all_tc_imports: - tc_sorted = _sort_imports_pep8(_merge_from_imports(all_tc_imports)) - tc_block = "if TYPE_CHECKING:\n" + "\n".join(" " + s for s in tc_sorted) - parts.append(tc_block) - parts.extend(entity_srcs) - pruned = _prune_unused_imports("\n\n\n".join(parts) + "\n") - new_files[target_file] = _prune_inline_redundant_imports(pruned) - - # If an existing conftest.py is present on disk, merge intelligently so - # that duplicate imports and fixture definitions are not repeated (which - # would cause flake8 F811/E402 errors when multiple splits write to the - # same conftest.py file). - if "conftest.py" in new_files: - existing_conftest = Path(original_path).parent / "conftest.py" - if existing_conftest.exists(): - prior = existing_conftest.read_text(encoding="utf-8") - new_files["conftest.py"] = _merge_conftest_sources( - prior, new_files["conftest.py"] - ) - - # Build updated original source. - # copy_not_migrate fixtures are written to the subdir conftest (so they - # appear in migrated_names / file_entity_names) but must NOT be removed - # from the original file — the original test discovers them via the test - # module itself, which takes precedence over the parent conftest's stale - # base version. - updated = _remove_entity_lines( - post_source, migrated_names - copy_not_migrate, entity_map, entity_source_map - ) - updated = _prune_unused_imports(updated) - # Compute TYPE_CHECKING imports needed by non-migrated entities that had - # their import guard block removed as part of a migrated TOP_LEVEL entity. - # Injection happens AFTER the relative-import bump below so that bumped - # import strings are passed to _inject_type_checking_imports rather than - # relying on the bump (which only matches unindented ``from .`` lines and - # therefore misses indented imports inside an ``if TYPE_CHECKING:`` block). - _non_migrated_names = [ - e.name for e in classified.entities if e.name not in migrated_names - ] - _tc_to_inject: List[str] = [] - if _non_migrated_names: - _tc_to_inject = _find_type_checking_needed_imports( - _non_migrated_names, entity_source_map, import_infos - ) - # For non-test subdir splits, re-exports from the __init__.py use relative - # import prefixes computed from inside the package (e.g. ".utils" not - # ".service.utils"). For test files the original keeps existing abs_pkg - # behaviour so pytest can find the re-exported symbols. - # In a non-test subdir split the updated source becomes subdir/__init__.py, - # which sits one directory level deeper than the original file. Any - # relative imports it still contains (e.g. ``from .. import llm_client`` - # or ``from .base import Foo``) therefore need one extra dot so they keep - # pointing at the same modules. Re-exports added by _add_re_exports below - # are already computed from the __init__.py's perspective and are correct. - if subdir_name is not None and not is_test_file and not has_main: - updated = _bump_relative_imports(updated) - _tc_to_inject = [_bump_relative_imports(imp) for imp in _tc_to_inject] - if _tc_to_inject: - updated = _inject_type_checking_imports(updated, _tc_to_inject) - if subdir_name is not None: - # If the original file had a module docstring and it was migrated away, - # place it in subdir/__init__.py in both cases: for non-test splits - # the docstring is prepended to `updated` which runner.py redirects to - # __init__.py; for test splits it is written directly to __init__.py - # (runner.py does not redirect `updated` for test files). - _module_doc = _extract_module_docstring(post_source) - if _module_doc and not _extract_module_docstring(updated): - if is_test_file: - new_files[f"{subdir_name}/__init__.py"] = _module_doc + "\n" - else: - updated = _module_doc + "\n\n" + updated - elif is_test_file and _source_is_only_docstring(updated): - # All entities migrated; the only thing remaining in the original - # is the module docstring (a TOP_LEVEL entity that was never - # removed by _remove_entity_lines). Route it to __init__.py and - # clear the original so the engine deletes it. - new_files[f"{subdir_name}/__init__.py"] = ( - _extract_module_docstring(updated) + "\n" - ) - updated = _strip_module_docstring(updated) - relative_from: Optional[str] = ( - f"{subdir_name}/__init__.py" - if (subdir_name and not is_test_file and not has_main) - else None - ) - # Apply module-qualified rewrites to the original file for any non-migrated - # entity that reassigns a TOP_LEVEL name that was moved to a sub-file. - # This is symmetric with the same treatment for new sub-files: when a name - # is in top_level_var_names (i.e. reassigned somewhere), all files that - # reference it — including the non-migrated home — use ``module.NAME``. - if top_level_var_names: - non_migrated_entity_names = [ - e.name for e in classified.entities if e.name not in migrated_names - ] - if non_migrated_entity_names: - _orig_from, orig_mod_imports, orig_rewrites = _find_cross_file_imports( - non_migrated_entity_names, - entity_source_map, - name_to_target_file, - non_migrated_home, - abs_pkg=abs_pkg, - top_level_var_names=top_level_var_names, - ) - if orig_rewrites: - updated = _rewrite_module_var_names(updated, orig_rewrites) - updated = _rewrite_module_level_stores(updated, orig_rewrites) - if orig_mod_imports: - updated = _inject_module_level_imports(updated, orig_mod_imports) - - # Exclude conftest.py from re-exports: fixtures there are auto-discovered - # by pytest and must not be imported back into the original test file. - # Also exclude conftest-conflict fixtures (kept in LLM-assigned file because - # conftest already has that name): pytest discovers them by name-lookup, so - # re-exporting them from the original file would be dead code and prevent - # the original from being cleaned up / deleted. - _conftest_files = {conftest_target, parent_conftest_target} - re_export_placements = [] - for p in valid_placements + synthetic_placements: - if p.target_file in _conftest_files: - continue - if conftest_conflict_names: - filtered_group = [n for n in p.group if n not in conftest_conflict_names] - if not filtered_group: - continue - p = GroupPlacement(group=filtered_group, target_file=p.target_file) - re_export_placements.append(p) - updated = _add_re_exports( - updated, - re_export_placements, - entity_map, - entity_source_map, - external_loads=external_loads, - abs_pkg=abs_pkg, - relative_from=relative_from, - is_test_file=is_test_file, - reexport_mode=reexport_mode, - ) - - # For non-migrated entities that reference test-named symbols now living - # in new files: _add_re_exports intentionally skips re-exporting them - # (to avoid double-discovery), so inject those imports inline instead. - migrated_test_symbols = { - name: tfile - for name, tfile in name_to_target_file.items() - if tfile != original_basename and _is_test_name(name) - } - updated = _inject_inline_test_imports_original( - updated, migrated_test_symbols, abs_pkg, original_basename - ) - - # Restore shebang at line 1 of the original. It may have been removed - # by _remove_entity_lines if the entity owning line 1 was migrated. - if shebang and not updated.startswith("#!"): - updated = shebang + updated - - # Remove section header comment blocks that became orphaned after entity - # removal (nothing substantive remains beneath them). - new_files = {f: _strip_orphaned_section_headers(s) for f, s in new_files.items()} - updated = _strip_orphaned_section_headers(updated) - - # Remove indented comment lines that ended up at module level after entity - # migration (flake8 E116: unexpected indentation: comment). - new_files = {f: _strip_orphaned_indented_comments(s) for f, s in new_files.items()} - updated = _strip_orphaned_indented_comments(updated) - - # When pytest_conftest is active and the test file's remaining content - # contains only fixtures (no test functions, no other definitions), the - # file has become dead code — all tests migrated away and the fixture is - # stranded. Route the remaining content to conftest.py (merging to avoid - # duplicates if it is already there) and empty the original so the engine - # deletes it. - if ( - is_test_file - and pytest_conftest - and updated - and _file_has_only_fixtures(updated) - ): - if conftest_target in new_files: - new_files[conftest_target] = _merge_conftest_sources( - new_files[conftest_target], updated - ) - elif existing_conftest_path.exists(): - prior = existing_conftest_path.read_text(encoding="utf-8") - new_files[conftest_target] = _merge_conftest_sources(prior, updated) - else: - new_files[conftest_target] = updated - updated = "" - - # Normalize blank lines: collapse 3+ consecutive blank lines to 2 and - # ensure exactly one trailing newline in every generated file. - new_files = {f: _normalize_blank_lines(s) for f, s in new_files.items()} - updated = _normalize_blank_lines(updated) - - return SplitResult( - new_files=new_files, - original_source=updated, - abort=False, - entity_name_rewrites=entity_name_rewrites, - actual_placements=valid_placements + synthetic_placements, - ) diff --git a/crispen/file_limiter/code_gen/__init__.py b/crispen/file_limiter/code_gen/__init__.py new file mode 100644 index 0000000..e44b27d --- /dev/null +++ b/crispen/file_limiter/code_gen/__init__.py @@ -0,0 +1,807 @@ +"""Code generation for FileLimiter: build new files and update original source.""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path +from typing import Dict, List, Optional, Set + +from ...import_sort import _sort_imports_pep8 +from ..advisor import FileLimiterPlan, GroupPlacement +from ..classifier import ClassifiedEntities +from ..dep_graph import find_sccs +from ..entity_parser import EntityKind +from .cross_file_deps import _abs_package_for_dir # fmt: skip # noqa: F401, E501 +from .cross_file_deps import _bump_relative_imports # fmt: skip # noqa: F401, E501 +from .cross_file_deps import _collect_external_imported_names # fmt: skip # noqa: F401, E501 +from .cross_file_deps import _find_cross_file_imports # fmt: skip # noqa: F401, E501 +from .cross_file_deps import _find_cross_file_type_checking_imports # fmt: skip # noqa: F401, E501 +from .cross_file_deps import _find_project_root # fmt: skip # noqa: F401, E501 +from .cross_file_deps import _module_import_stmt # fmt: skip # noqa: F401, E501 +from .cross_file_deps import _module_path_from_file # fmt: skip # noqa: F401, E501 +from .cross_file_deps import _relative_import_prefix # fmt: skip # noqa: F401, E501 +from .cross_file_deps import _target_module_name # fmt: skip # noqa: F401, E501 +from .import_analysis import ImportInfo # fmt: skip # noqa: F401, E501 +from .import_analysis import _collect_name_loads # fmt: skip # noqa: F401, E501 +from .import_analysis import _collect_name_stores # fmt: skip # noqa: F401, E501 +from .import_analysis import _collect_quoted_annotation_names # fmt: skip # noqa: F401, E501 +from .import_analysis import _extract_import_info # fmt: skip # noqa: F401, E501 +from .import_analysis import _find_needed_imports # fmt: skip # noqa: F401, E501 +from .import_analysis import _find_type_checking_needed_imports # fmt: skip # noqa: F401, E501 +from .import_analysis import _import_derived_names # fmt: skip # noqa: F401, E501 +from .import_analysis import _inject_module_level_imports # fmt: skip # noqa: F401, E501 +from .import_analysis import _inject_type_checking_imports # fmt: skip # noqa: F401, E501 +from .import_analysis import _narrow_import_source # fmt: skip # noqa: F401, E501 +from .import_analysis import _test_names_in_decorators # fmt: skip # noqa: F401, E501 +from .source_utils import _FUTURE_IMPORT_LINE_RE +from .source_utils import _EXCESS_BLANK_BODY_RE # fmt: skip # noqa: F401, E501 +from .source_utils import _EXCESS_BLANK_RE # fmt: skip # noqa: F401, E501 +from .source_utils import _extract_module_docstring # fmt: skip # noqa: F401, E501 +from .source_utils import _multiline_string_ranges # fmt: skip # noqa: F401, E501 +from .source_utils import _normalize_blank_lines # fmt: skip # noqa: F401, E501 +from .source_utils import _source_is_only_docstring # fmt: skip # noqa: F401, E501 +from .source_utils import _strip_module_docstring # fmt: skip # noqa: F401, E501 +from .source_utils import _strip_orphaned_indented_comments # fmt: skip # noqa: F401, E501 +from .source_utils import _strip_orphaned_section_headers # fmt: skip # noqa: F401, E501 +from .source_utils import _sub_skip_strings # fmt: skip # noqa: F401, E501 +from .test_support import _class_has_test_methods # fmt: skip # noqa: F401, E501 +from .test_support import _file_has_only_fixtures # fmt: skip # noqa: F401, E501 +from .test_support import _find_main_block_entity # fmt: skip # noqa: F401, E501 +from .test_support import _find_main_direct_callees # fmt: skip # noqa: F401, E501 +from .test_support import _inject_inline_imports # fmt: skip # noqa: F401, E501 +from .test_support import _inject_inline_test_imports_original # fmt: skip # noqa: F401, E501 +from .test_support import _is_pytest_fixture # fmt: skip # noqa: F401, E501 +from .test_support import _is_test_name # fmt: skip # noqa: F401, E501 +from .test_support import _merge_conftest_sources # fmt: skip # noqa: F401, E501 +from .test_support import _rewrite_module_level_stores # fmt: skip # noqa: F401, E501 +from .test_support import _rewrite_module_var_names # fmt: skip # noqa: F401, E501 +from .test_support import _split_cross_imports_by_test # fmt: skip # noqa: F401, E501 +from .transforms import _FROM_IMPORT_RE +from .transforms import SplitResult # fmt: skip # noqa: F401, E501 +from .transforms import _add_re_exports # fmt: skip # noqa: F401, E501 +from .transforms import _extract_shared_helpers # fmt: skip # noqa: F401, E501 +from .transforms import _import_line_numbers # fmt: skip # noqa: F401, E501 +from .transforms import _merge_from_imports # fmt: skip # noqa: F401, E501 +from .transforms import _prune_inline_redundant_imports # fmt: skip # noqa: F401, E501 +from .transforms import _prune_unused_imports # fmt: skip # noqa: F401, E501 +from .transforms import _remove_entity_lines # fmt: skip # noqa: F401, E501 +from .transforms import _strip_top_level_import_lines # fmt: skip # noqa: F401, E501 +from .transforms import _topo_depth # fmt: skip # noqa: F401, E501 + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def generate_file_splits( + classified: ClassifiedEntities, + plan: FileLimiterPlan, + post_source: str, + original_path: str, + subdir_name: Optional[str] = None, + pytest_conftest: bool = False, + has_main: bool = False, + reexport_mode: str = "always", +) -> SplitResult: + """Generate new file contents and the updated original source. + + When *plan* is aborted or has no placements, returns :class:`SplitResult` + with the original source unchanged (``abort`` mirrors ``plan.abort``). + + When *subdir_name* is set (e.g. ``"service"``), the file is being split + into a package subdirectory. The "original" file is treated as + ``service/__init__.py`` for dependency-graph and import-prefix purposes, + so cross-file imports within the package use correct relative paths. + + When *pytest_conftest* is True, any entity decorated with + ``@pytest.fixture`` (or ``@fixture``) is redirected to ``conftest.py`` + instead of the LLM-assigned target file. pytest auto-discovers fixtures + from ``conftest.py``, so no re-export import is added to the original + file — eliminating both the F401 and F811 flake8 warnings that arise + when a fixture name is used as a test function parameter. + """ + if plan.abort: + return SplitResult( + new_files={}, + original_source=post_source, + abort=True, + abort_reason=plan.abort_reason, + ) + + if not plan.placements: + return SplitResult(new_files={}, original_source=post_source, abort=False) + + # Detect shebang on line 1 so it can be stripped from new files and + # preserved (or restored) at the top of the original. + shebang: Optional[str] = None + if post_source.startswith("#!"): + nl = post_source.find("\n") + shebang = post_source[: nl + 1] if nl != -1 else post_source + "\n" + + lines = post_source.splitlines(keepends=True) + entity_map = {e.name: e for e in classified.entities} + + # Build entity source map (name → stripped source string). + entity_source_map: Dict[str, str] = {} + for entity in classified.entities: + entity_source_map[entity.name] = "".join( + lines[entity.start_line - 1 : entity.end_line] + ).rstrip() + + # All entity-defined names (used to limit import matching scope). + all_entity_names: Set[str] = { + name for e in classified.entities for name in e.names_defined + } + + # Extract import info from post-refactor source. + import_infos = _extract_import_info(post_source) + + # Placements whose target_file matches the original filename would create a + # self-referential import (e.g. `from .duplicate_extractor import Foo` inside + # duplicate_extractor.py). Drop them — entities stay in the original file. + # In subdir-split mode the "original" is the package __init__.py; use that + # name throughout so dependency-graph edges and import prefixes are correct. + original_basename = ( + f"{subdir_name}/__init__.py" + if subdir_name and not has_main + else Path(original_path).name + ) + is_test_file = Path(original_path).name.startswith("test_") + # For test-file subdir splits the original test file stays on disk (runner.py + # does not redirect it to __init__.py), so non-migrated names still live in + # the original file (e.g. "test_runner.py"), not in the package __init__.py. + non_migrated_home = ( + Path(original_path).name + if (subdir_name and is_test_file) + else original_basename + ) + # Identify the __main__ block and any functions it calls directly. + # These stay in the original file unconditionally: the __main__ block + # is an entry point the user expects to keep working, and its direct + # callees must live in the same file to avoid module-level test-class + # imports that would cause pytest double-discovery. + main_entity = _find_main_block_entity(classified.entities, entity_source_map) + main_sticky: Set[str] = set() + if main_entity is not None: + main_sticky.add(main_entity) + function_entity_names = { + e.name for e in classified.entities if e.kind == EntityKind.FUNCTION + } + main_sticky.update( + _find_main_direct_callees( + entity_source_map.get(main_entity, ""), function_entity_names + ) + ) + + valid_placements = [ + p + for p in plan.placements + if p.target_file != original_basename + and not any(name in main_sticky for name in p.group) + ] + + # --- Pytest conftest routing --- + # When enabled, entities decorated with @pytest.fixture are redirected to + # conftest.py instead of the LLM-assigned target file. pytest discovers + # fixtures from conftest.py automatically, so no re-export import is added + # to the original file — eliminating the F401/F811 flake8 false positives. + # + # Exception: if conftest.py already defines a function with the same name + # (e.g. a default fixture that this test file overrides), routing the entity + # there would cause _merge_conftest_sources to silently drop the new version + # (keeping the old one), so the entity would disappear from the split output + # entirely and verification would fail. In that case the fixture stays in + # its LLM-assigned target file; re-exports are still suppressed to avoid + # F401/F811 (the fixture is injected by pytest name-lookup, not by import). + # For subdir splits, route fixtures into the subdirectory's own conftest.py + # so that multiple test files in the same parent directory each get an + # isolated conftest scope and cannot overwrite each other's fixtures. + # Exception to the exception: if the fixture is still referenced in entities + # that remain in the original file (i.e. tests that were not migrated), route + # it to the parent conftest.py instead so those tests can find it. Tests in + # the subdirectory also inherit from the parent conftest, so this is safe. + # Further exception: if the fixture is referenced in remaining source AND the + # parent conftest already has a fixture with the same name (the module was + # overriding it), merging into parent conftest would silently keep the old + # version — the original test would get the wrong fixture and the migrated + # subdir tests would also inherit the wrong version. In that case, copy + # (don't move) the fixture to the subdir conftest so migrated tests get the + # override; the entity is also kept in the original file so the original + # test can discover it directly from its own module. + conftest_target = f"{subdir_name}/conftest.py" if subdir_name else "conftest.py" + parent_conftest_target = "conftest.py" + existing_conftest_path = ( + Path(original_path).parent / subdir_name / "conftest.py" + if subdir_name + else Path(original_path).parent / "conftest.py" + ) + fixture_entity_names: Set[str] = set() + # Names of fixtures kept in their LLM-assigned file because the target + # conftest.py already defines a symbol with that name. Fixtures injected + # by pytest name-lookup need no re-export import in the original file. + conftest_conflict_names: Set[str] = set() + # Names of fixtures that are copied to the subdir conftest but also kept in + # the original file (not removed). This handles the case where the fixture + # overrides a parent conftest fixture and is also referenced in remaining + # entities: the subdir copy ensures migrated tests see the override; the + # original file copy ensures the original test's module-level fixture takes + # precedence over the stale parent conftest version. + copy_not_migrate: Set[str] = set() + if pytest_conftest: + existing_conftest_names: Set[str] = set() + if existing_conftest_path.exists(): + try: + _ec_tree = ast.parse(existing_conftest_path.read_text(encoding="utf-8")) + for _ec_node in _ec_tree.body: + if isinstance( + _ec_node, + (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef), + ): + existing_conftest_names.add(_ec_node.name) + except (SyntaxError, OSError): + pass + + # For subdir splits: build source of entities that will remain in the + # original file so we can detect fixtures still needed by those entities. + # Also load parent conftest names to detect fixtures that override a + # parent-level fixture (used below to avoid silently keeping the old + # parent version when merging would drop the new override). + _migrating_names: Set[str] = { + name for p in valid_placements for name in p.group + } + _remaining_src = "\n".join( + entity_source_map[e.name] + for e in classified.entities + if e.name not in _migrating_names and e.name in entity_source_map + ) + _parent_conftest_names: Set[str] = set() + if subdir_name: + _parent_conftest_path = Path(original_path).parent / "conftest.py" + if _parent_conftest_path.exists(): + try: + _pc_tree = ast.parse( + _parent_conftest_path.read_text(encoding="utf-8") + ) + for _pc_node in _pc_tree.body: + if isinstance( + _pc_node, + (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef), + ): + _parent_conftest_names.add(_pc_node.name) + except (SyntaxError, OSError): + pass + + conftest_group: List[str] = [] + conftest_group_parent: List[str] = [] + new_valid: List[GroupPlacement] = [] + for p in valid_placements: + non_fixture: List[str] = [] + for name in p.group: + src = entity_source_map.get(name, "") + if src and _is_pytest_fixture(src): + fixture_entity_names.add(name) + if name in existing_conftest_names: + # conftest already has this name: keep in the + # LLM-assigned file to avoid losing the entity. + # No re-export needed — pytest discovers fixtures + # by name-lookup, not by import. + non_fixture.append(name) + conftest_conflict_names.add(name) + elif subdir_name and re.search( + r"\b" + re.escape(name) + r"\b", _remaining_src + ): + # Fixture still used by non-migrated tests. + if name in _parent_conftest_names: + # Parent conftest already has this name: the module + # was overriding it. Merging into parent conftest + # would silently keep the old version. Instead, + # copy (don't move) to subdir conftest — migrated + # tests get the override via the subdir conftest; + # the entity stays in the original file so the + # original test discovers the override from its own + # module rather than the stale parent conftest entry. + conftest_group.append(name) + copy_not_migrate.add(name) + else: + # No conflict: route to the parent conftest.py so + # both original and subdir tests can find it. + conftest_group_parent.append(name) + else: + conftest_group.append(name) + else: + non_fixture.append(name) + if non_fixture: + new_valid.append( + GroupPlacement(group=non_fixture, target_file=p.target_file) + ) + if conftest_group: + new_valid.append( + GroupPlacement(group=conftest_group, target_file=conftest_target) + ) + if conftest_group_parent: + new_valid.append( + GroupPlacement( + group=conftest_group_parent, target_file=parent_conftest_target + ) + ) + valid_placements = new_valid + + # Group placements by target file (preserving order for topo sort). + file_entity_names: Dict[str, List[str]] = {} + for placement in valid_placements: + file_entity_names.setdefault(placement.target_file, []).extend(placement.group) + + # All migrated entity names. + migrated_names: Set[str] = {name for p in valid_placements for name in p.group} + + # Build name → target-file map for cross-file import detection. + # Exclude import-derived names (_find_needed_imports handles those). + import_defined_names = {name for info in import_infos for name in info.names} + name_to_target_file: Dict[str, str] = {} + for target_file, ent_names in file_entity_names.items(): + for ent_name in ent_names: + entity = entity_map.get(ent_name) + if entity: + for defined_name in entity.names_defined: + if defined_name not in import_defined_names: + name_to_target_file[defined_name] = target_file + + # Also map names from non-migrated entities to the original file so that + # split files can import helpers (e.g. _run) that stayed behind. + for entity in classified.entities: + if entity.name not in migrated_names: + for defined_name in entity.names_defined: + if defined_name not in import_defined_names: + name_to_target_file.setdefault(defined_name, non_migrated_home) + + # For TOP_LEVEL names that are reassigned (stored) by a *different* entity, + # cross-file references must use ``module.NAME`` so that the mutation is + # visible to all importers. Names that are only ever defined once and never + # mutated elsewhere use a plain ``from .module import NAME`` — the idiomatic + # Python form — since the value is stable after import. + _top_level_def_entity: Dict[str, str] = { + defined_name: entity.name + for entity in classified.entities + if entity.kind == EntityKind.TOP_LEVEL + for defined_name in entity.names_defined + if defined_name not in import_defined_names + } + top_level_var_names: Set[str] = { + name + for name, def_ent in _top_level_def_entity.items() + for ent_name, src in entity_source_map.items() + if ent_name != def_ent and name in _collect_name_stores(src) + } + + # Extract non-migrated FUNCTION/CLASS entities referenced by migrated ones + # into the new files that use them, breaking O→F→O import cycles. + synthetic_placements = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + non_migrated_home, + ) + + # Collect names that external files (outside the module being split) import + # from the original file. Private symbols in this set must get a re-export + # proxy even though they are no longer referenced by the remaining source. + external_loads = _collect_external_imported_names(original_path) + + # Detect circular imports. Cycles can pass through the original file: + # a new file that imports a non-migrated name from the original can form a + # chain back to the original via the re-exports the original adds. + # Model the original as an explicit node in the dependency graph. + # + # Original's outgoing edges: it will re-export a migrated name when the + # name is public (no _/test_ prefix), referenced by a non-migrated entity, + # or imported by an external file (external_loads). + # + # In a test-file subdir split, non_migrated_home ("test_runner.py") differs + # from original_basename ("runner/__init__.py"). Re-exports are injected + # into the original test file, so it—not the __init__.py—gains outgoing + # import edges and must be a separate node in the dependency graph. + non_migrated_loads: Set[str] = set() + for ent_name, src in entity_source_map.items(): + if ent_name not in migrated_names: + non_migrated_loads |= _collect_name_loads(src) + + reexport_home = non_migrated_home + all_dep_nodes = set(file_entity_names.keys()) | {original_basename, reexport_home} + file_deps: Dict[str, Set[str]] = {node: set() for node in all_dep_nodes} + for target_file, ent_names in file_entity_names.items(): + for ent_name in ent_names: + src = entity_source_map.get(ent_name, "") + for ref_name in _collect_name_loads(src): + dep_file = name_to_target_file.get(ref_name) + if dep_file and dep_file != target_file and dep_file in file_deps: + file_deps[target_file].add(dep_file) + for placement in valid_placements + synthetic_placements: + for ent_name in placement.group: + entity = entity_map.get(ent_name) + if entity: + for defined_name in entity.names_defined: + reexport_unconditionally = ( + not defined_name.startswith("_") + and not defined_name.startswith("test_") + and ( + reexport_mode == "always" + or (reexport_mode == "application" and not is_test_file) + ) + ) + if ( + reexport_unconditionally + or defined_name in non_migrated_loads + or defined_name in external_loads + ): + file_deps[reexport_home].add(placement.target_file) + break + if any(len(scc) > 1 for scc in find_sccs(file_deps)): + return SplitResult( + new_files={}, + original_source=post_source, + abort=True, + abort_reason="proposed split would create circular file imports", + ) + + # Use absolute imports when the original file is a test file. Pytest's + # default import mode loads test files as top-level modules (not package + # members), so relative imports like `from .helpers import foo` would + # raise ImportError at collection time. + abs_pkg: Optional[str] = None + if Path(original_path).name.startswith("test_"): + abs_pkg = _abs_package_for_dir(original_path) + + # In subdir-split mode, new files live inside a package subdirectory and + # can use relative imports for cross-file references within that package. + abs_pkg_for_new_files: Optional[str] = None if subdir_name else abs_pkg + + # Generate new file contents. + new_files: Dict[str, str] = {} + entity_name_rewrites: Dict[str, Dict[str, str]] = {} # per-entity rewrites + for target_file, ent_names in file_entity_names.items(): + needed = _find_needed_imports( + ent_names, entity_source_map, import_infos, all_entity_names + ) + needed_tc = _find_type_checking_needed_imports( + ent_names, entity_source_map, import_infos + ) + if subdir_name is not None: + depth = len(Path(target_file).parts) - 1 + needed = [_bump_relative_imports(s, depth) for s in needed] + needed_tc = [_bump_relative_imports(s, depth) for s in needed_tc] + entity_srcs = [] + top_cross: List[str] = [] + seen_top_cross: Set[str] = set() + all_tc_imports: List[str] = list(needed_tc) + seen_tc: Set[str] = set(needed_tc) + for _ent_name in ent_names: + _src = entity_source_map.get(_ent_name) + if _src is None: + continue + _entity = entity_map.get(_ent_name) + if _entity and _entity.kind == EntityKind.TOP_LEVEL: + # Imports are emitted separately by _find_needed_imports; strip + # them from the entity body to prevent duplicate import stmts. + _src = _strip_top_level_import_lines(_src) + if subdir_name is not None: + # In subdir-split mode the module docstring belongs in + # __init__.py rather than in one of the child modules. + _src = _strip_module_docstring(_src) + else: + _src = _FUTURE_IMPORT_LINE_RE.sub("", _src) + # Strip shebang from any entity that begins on line 1 of the + # original source — it must not appear in generated new files. + if shebang and _entity and _entity.start_line == 1: + nl = _src.find("\n") + _src = _src[nl + 1 :] if nl != -1 else "" + _src = _src.rstrip() + # Compute cross-file imports for this entity and split off any + # test-named symbols (Test* / test_*) to be injected inline. + entity_from, entity_mod, entity_rewrites = _find_cross_file_imports( + [_ent_name], + entity_source_map, + name_to_target_file, + target_file, + abs_pkg=abs_pkg_for_new_files, + top_level_var_names=top_level_var_names, + ) + for _tc_imp in _find_cross_file_type_checking_imports( + [_ent_name], + entity_source_map, + name_to_target_file, + target_file, + abs_pkg=abs_pkg_for_new_files, + top_level_var_names=top_level_var_names, + ): + if _tc_imp not in seen_tc: + seen_tc.add(_tc_imp) + all_tc_imports.append(_tc_imp) + if entity_rewrites: + _src = _rewrite_module_var_names(_src, entity_rewrites) + entity_name_rewrites[_ent_name] = entity_rewrites + # Module imports for TOP_LEVEL vars must always be at module level + # (never inlined) — decorators are evaluated before function bodies run. + for imp in entity_mod: + if imp not in seen_top_cross: + seen_top_cross.add(imp) + top_cross.append(imp) + ent_top_cross, ent_test_imports = _split_cross_imports_by_test(entity_from) + for imp in ent_top_cross: + if imp not in seen_top_cross: + seen_top_cross.add(imp) + top_cross.append(imp) + if ent_test_imports and _entity and _entity.kind != EntityKind.TOP_LEVEL: + # Extract the test-named symbols that would be inlined. + # All items from _split_cross_imports_by_test are "from X import Y" + # form, so partitioning on " import " is safe. + inlined_names: Set[str] = set() + for _imp in ent_test_imports: + _, _, _names_part = _imp.partition(" import ") + for _n in _names_part.split(","): + inlined_names.add(_n.strip()) + # Decorators are evaluated before function bodies, so a symbol + # that only arrives via an inline import will not be in scope. + dec_conflicts = _test_names_in_decorators(_src, inlined_names) + if dec_conflicts: + _names_str = ", ".join(f"'{n}'" for n in sorted(dec_conflicts)) + return SplitResult( + new_files={}, + original_source=post_source, + abort=True, + abort_reason=( + f"cannot split '{_ent_name}': {_names_str} appear(s) " + f"in a decorator but would need to be imported inline " + f"to avoid pytest collecting them as duplicate tests — " + f"keep the dependent test classes in the same file" + ), + ) + _src = _inject_inline_imports(_src, ent_test_imports) + else: + # TOP_LEVEL entity: no body scope, fall back to module level. + for imp in ent_test_imports: + if imp not in seen_top_cross: + seen_top_cross.add(imp) + top_cross.append(imp) + entity_srcs.append(_src) + entity_srcs = [s for s in entity_srcs if s] + # Dedup: remove TC imports for names already covered by regular imports. + # This can happen when one entity uses a name at runtime (→ top_cross) + # while another entity in the same file only uses it in a quoted + # annotation (→ all_tc_imports), producing duplicate import statements. + if all_tc_imports and (needed or top_cross): + _regular_names: Set[str] = set() + for _imp in needed + top_cross: + _m = _FROM_IMPORT_RE.match(_imp) + if _m: + _regular_names.update( + n.strip() for n in _m.group(2).split(",") if n.strip() + ) + _deduped_tc: List[str] = [] + for _tc in all_tc_imports: + _m = _FROM_IMPORT_RE.match(_tc) + if _m: + _tc_names = { + _n.strip() for _n in _m.group(2).split(",") if _n.strip() + } + _leftover = _tc_names - _regular_names + if _leftover: + _deduped_tc.append( + _tc + if _leftover == _tc_names + else _narrow_import_source(_tc, _leftover) + ) + else: + _deduped_tc.append(_tc) + all_tc_imports = _deduped_tc + parts: List[str] = [] + imports_for_sort = list(needed + top_cross) + if all_tc_imports: + imports_for_sort.append("from typing import TYPE_CHECKING") + all_imports = _sort_imports_pep8(_merge_from_imports(imports_for_sort)) + if all_imports: + parts.append("\n".join(all_imports)) + if all_tc_imports: + tc_sorted = _sort_imports_pep8(_merge_from_imports(all_tc_imports)) + tc_block = "if TYPE_CHECKING:\n" + "\n".join(" " + s for s in tc_sorted) + parts.append(tc_block) + parts.extend(entity_srcs) + pruned = _prune_unused_imports("\n\n\n".join(parts) + "\n") + new_files[target_file] = _prune_inline_redundant_imports(pruned) + + # If an existing conftest.py is present on disk, merge intelligently so + # that duplicate imports and fixture definitions are not repeated (which + # would cause flake8 F811/E402 errors when multiple splits write to the + # same conftest.py file). + if "conftest.py" in new_files: + existing_conftest = Path(original_path).parent / "conftest.py" + if existing_conftest.exists(): + prior = existing_conftest.read_text(encoding="utf-8") + new_files["conftest.py"] = _merge_conftest_sources( + prior, new_files["conftest.py"] + ) + + # Build updated original source. + # copy_not_migrate fixtures are written to the subdir conftest (so they + # appear in migrated_names / file_entity_names) but must NOT be removed + # from the original file — the original test discovers them via the test + # module itself, which takes precedence over the parent conftest's stale + # base version. + updated = _remove_entity_lines( + post_source, migrated_names - copy_not_migrate, entity_map, entity_source_map + ) + updated = _prune_unused_imports(updated) + # Compute TYPE_CHECKING imports needed by non-migrated entities that had + # their import guard block removed as part of a migrated TOP_LEVEL entity. + # Injection happens AFTER the relative-import bump below so that bumped + # import strings are passed to _inject_type_checking_imports rather than + # relying on the bump (which only matches unindented ``from .`` lines and + # therefore misses indented imports inside an ``if TYPE_CHECKING:`` block). + _non_migrated_names = [ + e.name for e in classified.entities if e.name not in migrated_names + ] + _tc_to_inject: List[str] = [] + if _non_migrated_names: + _tc_to_inject = _find_type_checking_needed_imports( + _non_migrated_names, entity_source_map, import_infos + ) + # For non-test subdir splits, re-exports from the __init__.py use relative + # import prefixes computed from inside the package (e.g. ".utils" not + # ".service.utils"). For test files the original keeps existing abs_pkg + # behaviour so pytest can find the re-exported symbols. + # In a non-test subdir split the updated source becomes subdir/__init__.py, + # which sits one directory level deeper than the original file. Any + # relative imports it still contains (e.g. ``from .. import llm_client`` + # or ``from .base import Foo``) therefore need one extra dot so they keep + # pointing at the same modules. Re-exports added by _add_re_exports below + # are already computed from the __init__.py's perspective and are correct. + if subdir_name is not None and not is_test_file and not has_main: + updated = _bump_relative_imports(updated) + _tc_to_inject = [_bump_relative_imports(imp) for imp in _tc_to_inject] + if _tc_to_inject: + updated = _inject_type_checking_imports(updated, _tc_to_inject) + if subdir_name is not None: + # If the original file had a module docstring and it was migrated away, + # place it in subdir/__init__.py in both cases: for non-test splits + # the docstring is prepended to `updated` which runner.py redirects to + # __init__.py; for test splits it is written directly to __init__.py + # (runner.py does not redirect `updated` for test files). + _module_doc = _extract_module_docstring(post_source) + if _module_doc and not _extract_module_docstring(updated): + if is_test_file: + new_files[f"{subdir_name}/__init__.py"] = _module_doc + "\n" + else: + updated = _module_doc + "\n\n" + updated + elif is_test_file and _source_is_only_docstring(updated): + # All entities migrated; the only thing remaining in the original + # is the module docstring (a TOP_LEVEL entity that was never + # removed by _remove_entity_lines). Route it to __init__.py and + # clear the original so the engine deletes it. + new_files[f"{subdir_name}/__init__.py"] = ( + _extract_module_docstring(updated) + "\n" + ) + updated = _strip_module_docstring(updated) + relative_from: Optional[str] = ( + f"{subdir_name}/__init__.py" + if (subdir_name and not is_test_file and not has_main) + else None + ) + # Apply module-qualified rewrites to the original file for any non-migrated + # entity that reassigns a TOP_LEVEL name that was moved to a sub-file. + # This is symmetric with the same treatment for new sub-files: when a name + # is in top_level_var_names (i.e. reassigned somewhere), all files that + # reference it — including the non-migrated home — use ``module.NAME``. + if top_level_var_names: + non_migrated_entity_names = [ + e.name for e in classified.entities if e.name not in migrated_names + ] + if non_migrated_entity_names: + _orig_from, orig_mod_imports, orig_rewrites = _find_cross_file_imports( + non_migrated_entity_names, + entity_source_map, + name_to_target_file, + non_migrated_home, + abs_pkg=abs_pkg, + top_level_var_names=top_level_var_names, + ) + if orig_rewrites: + updated = _rewrite_module_var_names(updated, orig_rewrites) + updated = _rewrite_module_level_stores(updated, orig_rewrites) + if orig_mod_imports: + updated = _inject_module_level_imports(updated, orig_mod_imports) + + # Exclude conftest.py from re-exports: fixtures there are auto-discovered + # by pytest and must not be imported back into the original test file. + # Also exclude conftest-conflict fixtures (kept in LLM-assigned file because + # conftest already has that name): pytest discovers them by name-lookup, so + # re-exporting them from the original file would be dead code and prevent + # the original from being cleaned up / deleted. + _conftest_files = {conftest_target, parent_conftest_target} + re_export_placements = [] + for p in valid_placements + synthetic_placements: + if p.target_file in _conftest_files: + continue + if conftest_conflict_names: + filtered_group = [n for n in p.group if n not in conftest_conflict_names] + if not filtered_group: + continue + p = GroupPlacement(group=filtered_group, target_file=p.target_file) + re_export_placements.append(p) + updated = _add_re_exports( + updated, + re_export_placements, + entity_map, + entity_source_map, + external_loads=external_loads, + abs_pkg=abs_pkg, + relative_from=relative_from, + is_test_file=is_test_file, + reexport_mode=reexport_mode, + ) + + # For non-migrated entities that reference test-named symbols now living + # in new files: _add_re_exports intentionally skips re-exporting them + # (to avoid double-discovery), so inject those imports inline instead. + migrated_test_symbols = { + name: tfile + for name, tfile in name_to_target_file.items() + if tfile != original_basename and _is_test_name(name) + } + updated = _inject_inline_test_imports_original( + updated, migrated_test_symbols, abs_pkg, original_basename + ) + + # Restore shebang at line 1 of the original. It may have been removed + # by _remove_entity_lines if the entity owning line 1 was migrated. + if shebang and not updated.startswith("#!"): + updated = shebang + updated + + # Remove section header comment blocks that became orphaned after entity + # removal (nothing substantive remains beneath them). + new_files = {f: _strip_orphaned_section_headers(s) for f, s in new_files.items()} + updated = _strip_orphaned_section_headers(updated) + + # Remove indented comment lines that ended up at module level after entity + # migration (flake8 E116: unexpected indentation: comment). + new_files = {f: _strip_orphaned_indented_comments(s) for f, s in new_files.items()} + updated = _strip_orphaned_indented_comments(updated) + + # When pytest_conftest is active and the test file's remaining content + # contains only fixtures (no test functions, no other definitions), the + # file has become dead code — all tests migrated away and the fixture is + # stranded. Route the remaining content to conftest.py (merging to avoid + # duplicates if it is already there) and empty the original so the engine + # deletes it. + if ( + is_test_file + and pytest_conftest + and updated + and _file_has_only_fixtures(updated) + ): + if conftest_target in new_files: + new_files[conftest_target] = _merge_conftest_sources( + new_files[conftest_target], updated + ) + elif existing_conftest_path.exists(): + prior = existing_conftest_path.read_text(encoding="utf-8") + new_files[conftest_target] = _merge_conftest_sources(prior, updated) + else: + new_files[conftest_target] = updated + updated = "" + + # Normalize blank lines: collapse 3+ consecutive blank lines to 2 and + # ensure exactly one trailing newline in every generated file. + new_files = {f: _normalize_blank_lines(s) for f, s in new_files.items()} + updated = _normalize_blank_lines(updated) + + return SplitResult( + new_files=new_files, + original_source=updated, + abort=False, + entity_name_rewrites=entity_name_rewrites, + actual_placements=valid_placements + synthetic_placements, + ) diff --git a/crispen/file_limiter/code_gen/cross_file_deps.py b/crispen/file_limiter/code_gen/cross_file_deps.py new file mode 100644 index 0000000..bfccb23 --- /dev/null +++ b/crispen/file_limiter/code_gen/cross_file_deps.py @@ -0,0 +1,349 @@ +from __future__ import annotations +from pathlib import Path +from typing import Dict, List, Optional, Set, Tuple +import ast +from .source_utils import _REL_IMPORT_RE +from .import_analysis import _collect_name_loads, _collect_quoted_annotation_names + + +def _bump_relative_imports(source: str, n: int = 1) -> str: + """Increment the level of every relative import in *source* by *n*. + + Used when file content is moved directory levels deeper, e.g. when the + source originally written for ``pkg/module.py`` becomes the content of + ``pkg/module/__init__.py``, or when new files go into a subdirectory + package instead of sitting next to the original file. + + With n=1: ``from .foo`` → ``from ..foo``, ``from ..bar`` → ``from ...bar``. + With n=2: ``from .foo`` → ``from ...foo``, etc. + Absolute imports are not affected. + """ + for _ in range(n): + source = _REL_IMPORT_RE.sub(lambda m: f"from .{m.group(1)}", source) + return source + + +def _relative_import_prefix(from_file: str, to_file: str) -> str: + """Return the Python relative-import prefix for *to_file* as seen from *from_file*. + + Both paths are relative to the same base directory (the original file's + directory). Examples:: + + _relative_import_prefix("utils.py", "helpers.py") → ".helpers" + _relative_import_prefix("sub/a.py", "helpers/b.py") → "..helpers.b" + _relative_import_prefix("sub/a.py", "sub/b.py") → ".b" + _relative_import_prefix("a.py", "__init__.py") → "." + _relative_import_prefix("sub/a.py", "sub/__init__.py") → "." + """ + to_path = Path(to_file) + from_parts = Path(from_file).parent.parts # () for top-level files + # __init__.py represents the package itself, not a submodule named "__init__". + if to_path.stem == "__init__": + to_module_parts = to_path.parent.parts + else: + to_module_parts = to_path.with_suffix("").parts # ("helpers", "b") + to_dir_parts = to_path.parent.parts # ("helpers",) + + # Length of the common directory prefix between from_dir and to_dir. + common_len = 0 + for fp, tp in zip(from_parts, to_dir_parts): + if fp == tp: + common_len += 1 + else: + break + + levels_up = len(from_parts) - common_len + module = ".".join(to_module_parts[common_len:]) + return "." * (levels_up + 1) + module + + +def _target_module_name(target_file: str) -> str: + """Convert a relative target filename to a dotted module name. + + ``"utils.py"`` → ``"utils"``, ``"helpers/io.py"`` → ``"helpers.io"``, + ``"pkg/__init__.py"`` → ``"pkg"`` (package, not ``"pkg.__init__"``). + """ + path = Path(target_file) + if path.stem == "__init__": + parts = list(path.parent.parts) + else: + parts = list(path.with_suffix("").parts) + return ".".join(parts) + + +def _module_import_stmt( + current_target: str, + source_file: str, + abs_pkg: Optional[str], +) -> Tuple[str, str]: + """Return ``(import_statement, local_name)`` for a module-level import. + + Produces ``from . import conversion`` instead of + ``from .conversion import SAFE_MODE`` so callers can reference + ``conversion.SAFE_MODE`` for a live lookup rather than a value snapshot. + This preserves the original single-file behaviour where module globals are + looked up dynamically rather than captured at import time. + """ + local_name = _target_module_name(source_file).split(".")[-1] + if abs_pkg is not None: + mod = _target_module_name(source_file) + # Use "import full.module.path as local_name" for absolute contexts. + # This avoids "from pkg import test_module" patterns that are + # misidentified as test-name imports by _split_cross_imports_by_test. + full_mod = f"{abs_pkg}.{mod}" if abs_pkg else mod + stmt = ( + f"import {full_mod} as {local_name}" + if full_mod != local_name + else f"import {local_name}" + ) + else: + prefix = _relative_import_prefix(current_target, source_file) + # prefix looks like ".conversion", "..test_svc", or "..helpers.io". + # Decompose into leading dots + module path, then extract the last + # segment as local_name and the rest as the parent package prefix. + # ".conversion" → dots="..", path="conversion" → "from . import conversion" + # "..test_svc" → dots="..", path="test_svc" → "from .. import test_svc" + # "..helpers.io" → dots="..", path="helpers.io" → "from ..helpers import io" + dot_end = 0 + while dot_end < len(prefix) and prefix[dot_end] == ".": + dot_end += 1 + dots = prefix[:dot_end] + path = prefix[dot_end:] + last_dot = path.rfind(".") + if last_dot == -1: + parent = dots or "." + else: + parent = dots + path[:last_dot] + stmt = f"from {parent} import {local_name}" + return stmt, local_name + + +def _find_cross_file_imports( + entity_names: List[str], + entity_source_map: Dict[str, str], + name_to_target_file: Dict[str, str], + current_target: str, + abs_pkg: Optional[str] = None, + top_level_var_names: Optional[Set[str]] = None, +) -> Tuple[List[str], List[str], Dict[str, str]]: + """Return ``(from_imports, module_imports, name_rewrites)`` for other-file + dependencies. + + When an entity being moved to *current_target* references a name that is + defined by another entity being moved to a different target file, the new + file needs an explicit import for that name. + + *from_imports* are ``from .module import Name`` statements for + function/class references. These may be subject to test-name inline + injection by the caller (to avoid pytest collecting imported test functions + as duplicate tests). + + *module_imports* are ``from . import module`` (or ``import pkg.module as + module``) statements for names defined by ``TOP_LEVEL`` entities + (module-level variables such as ``SAFE_MODE = True``). These must always + be placed at module level — never injected inline — because they are + required by decorator expressions that are evaluated before any function + body runs. The returned *name_rewrites* dict maps each such bare name + (e.g. ``"SAFE_MODE"``) to its qualified form (e.g. + ``"conversion.SAFE_MODE"``); callers must rewrite the entity source + accordingly. + + When *abs_pkg* is ``None`` the import prefix is relative (e.g. + ``from .constants import _CONST``). When *abs_pkg* is set the import is + absolute (e.g. ``from tests.constants import _CONST``), which is required + for test files that pytest loads as top-level modules. + """ + referenced: Set[str] = set() + for name in entity_names: + src = entity_source_map.get(name, "") + referenced |= _collect_name_loads(src) + from_files: Dict[str, List[str]] = {} # source_file → regular names + mod_files: Dict[str, List[str]] = {} # source_file → top-level var names + for ref_name in sorted(referenced): + source_file = name_to_target_file.get(ref_name) + if source_file and source_file != current_target: + if top_level_var_names and ref_name in top_level_var_names: + mod_files.setdefault(source_file, []).append(ref_name) + else: + from_files.setdefault(source_file, []).append(ref_name) + + from_result: List[str] = [] + mod_result: List[str] = [] + rewrites: Dict[str, str] = {} + for source_file, names in sorted(from_files.items()): + if abs_pkg is not None: + mod = _target_module_name(source_file) + prefix = f"{abs_pkg}.{mod}" if abs_pkg else mod + else: + prefix = _relative_import_prefix(current_target, source_file) + from_result.append(f"from {prefix} import {', '.join(sorted(names))}") + for source_file, names in sorted(mod_files.items()): + stmt, local_name = _module_import_stmt(current_target, source_file, abs_pkg) + mod_result.append(stmt) + for name in names: + rewrites[name] = f"{local_name}.{name}" + return from_result, mod_result, rewrites + + +def _find_cross_file_type_checking_imports( + entity_names: List[str], + entity_source_map: Dict[str, str], + name_to_target_file: Dict[str, str], + current_target: str, + abs_pkg: Optional[str] = None, + top_level_var_names: Optional[Set[str]] = None, +) -> List[str]: + """Return cross-file imports for names only referenced in quoted annotations. + + When an entity uses a name only inside a quoted type annotation (e.g. + ``Optional["_LLMAccumulator"]``) and that name is defined in another new + file produced by the same split, a ``from .other import Name`` statement + is generated here. These should be placed under ``if TYPE_CHECKING:`` + because they are not needed at runtime. + + Names that also appear in regular (non-annotation) loads are excluded — + they already get a normal cross-file import from + ``_find_cross_file_imports``. Top-level variable names (which require + module-alias imports) are also skipped here. + """ + runtime_referenced: Set[str] = set() + quoted_referenced: Set[str] = set() + for name in entity_names: + src = entity_source_map.get(name, "") + runtime_referenced |= _collect_name_loads(src) + quoted_referenced |= _collect_quoted_annotation_names(src) + + annotation_only = quoted_referenced - runtime_referenced + if not annotation_only: + return [] + + tc_files: Dict[str, List[str]] = {} + for ref_name in sorted(annotation_only): + source_file = name_to_target_file.get(ref_name) + if source_file and source_file != current_target: + # Top-level var names need module-alias imports, not handled here. + if top_level_var_names and ref_name in top_level_var_names: + continue + tc_files.setdefault(source_file, []).append(ref_name) + + result: List[str] = [] + for source_file, names in sorted(tc_files.items()): + if abs_pkg is not None: + mod = _target_module_name(source_file) + prefix = f"{abs_pkg}.{mod}" if abs_pkg else mod + else: + prefix = _relative_import_prefix(current_target, source_file) + result.append(f"from {prefix} import {', '.join(sorted(names))}") + return result + + +def _find_project_root(path: Path) -> Optional[Path]: + """Walk up from *path* to find the project root directory. + + Returns the first directory containing ``pyproject.toml``, ``setup.py``, + ``setup.cfg``, or ``.git``. Returns ``None`` when the filesystem root is + reached without finding any of these markers. + """ + markers = {"pyproject.toml", "setup.py", "setup.cfg", ".git"} + current = path if path.is_dir() else path.parent + while True: + if any((current / m).exists() for m in markers): + return current + parent = current.parent + if parent == current: + return None + current = parent + + +def _module_path_from_file(project_root: Path, file_path: Path) -> Optional[str]: + """Return the dotted Python module path of *file_path* relative to *project_root*. + + Returns ``None`` when *file_path* is not under *project_root*. + """ + try: + rel = file_path.relative_to(project_root) + except ValueError: + return None + return ".".join(rel.with_suffix("").parts) + + +def _abs_package_for_dir(file_path: str) -> Optional[str]: + """Return the dotted package path of the directory containing *file_path*. + + Used to generate absolute imports for test files so that pytest's default + import mode (which loads test files as top-level modules, not package + members) does not choke on ``from .module import …`` syntax. + + Returns an empty string for files sitting directly in the project root, + ``None`` when the project root cannot be determined. + """ + orig = Path(file_path).resolve() + project_root = _find_project_root(orig.parent) + if project_root is None: + return None + try: + rel = orig.parent.relative_to(project_root) + except ValueError: + return None + return ".".join(rel.parts) + + +def _collect_external_imported_names(original_path: str) -> Set[str]: + """Return names imported from *original_path* by other Python files. + + Scans all Python files under the project root for ``from import`` + statements targeting the module corresponding to *original_path*, and + returns the union of all imported original names (before any ``as`` alias). + + Returns an empty set when *original_path* does not resolve to an existing + file, the project root cannot be determined, or the path cannot be mapped + to a module. Both absolute and relative paths are accepted; relative paths + are resolved against the current working directory (the repo root when + crispen is invoked as ``git diff | crispen``). + """ + orig = Path(original_path).resolve() + if not orig.exists(): + return set() + project_root = _find_project_root(orig.parent) + if project_root is None: + return set() + # project_root is an ancestor of orig (derived by walking up from orig.parent), + # so _module_path_from_file always returns a non-None string here. + target_module = _module_path_from_file(project_root, orig) + # __init__.py defines the package itself; external callers import from the + # package path (e.g. "pkg.sub"), not "pkg.sub.__init__". + if orig.name == "__init__.py": + dot = target_module.rfind(".") + if dot == -1: + return set() # bare __init__.py at project root; no external callers + target_module = target_module[:dot] + result: Set[str] = set() + for py_file in project_root.rglob("*.py"): + if py_file.resolve() == orig: + continue + try: + source = py_file.read_text(encoding="utf-8", errors="replace") + tree = ast.parse(source, filename=str(py_file)) + except Exception: + continue + # Compute this file's dotted module path for relative-import resolution. + file_module = _module_path_from_file(project_root, py_file) + file_pkg_parts = file_module.split(".")[:-1] if file_module else [] + for node in ast.walk(tree): + if not isinstance(node, ast.ImportFrom): + continue + if node.level == 0: + imported_from = node.module or "" + else: + # Relative import: go up (level - 1) packages from file_pkg_parts. + up = node.level - 1 + if up > len(file_pkg_parts): + continue + base = file_pkg_parts[: len(file_pkg_parts) - up] + sub = node.module or "" + imported_from = ".".join(base + ([sub] if sub else [])) + if imported_from != target_module: + continue + for alias in node.names: + result.add(alias.name) + return result diff --git a/crispen/file_limiter/code_gen/import_analysis.py b/crispen/file_limiter/code_gen/import_analysis.py new file mode 100644 index 0000000..f8d9953 --- /dev/null +++ b/crispen/file_limiter/code_gen/import_analysis.py @@ -0,0 +1,513 @@ +from __future__ import annotations +from dataclasses import dataclass +from typing import Dict, List, Set +import ast + + +@dataclass +class ImportInfo: + """A top-level import statement and the names it introduces.""" + + names: List[str] # names made available by this import + source: str # the import statement text (no trailing newline) + is_future: bool # True if `from __future__ import ...` + is_type_checking: bool = False # True if inside `if TYPE_CHECKING:` block + + +def _import_derived_names(source: str) -> Set[str]: + """Return names introduced solely by import statements in *source*. + + These names live in the original file's namespace via its import + statements and cannot be re-exported from a new module the way + assignment-defined names can. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + names: Set[str] = set() + for node in tree.body: + if isinstance(node, ast.Import): + for alias in node.names: + names.add(alias.asname if alias.asname else alias.name.split(".")[0]) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + names.add(alias.asname if alias.asname else alias.name) + return names + + +def _collect_name_loads(source: str) -> Set[str]: + """Return Name loads in *source* that are not shadowed by function parameters + or local variable assignments. + + For each function or async function, names that appear as parameters of that + function or are assigned anywhere in the function body are excluded from Name + loads within its body. This prevents generating spurious cross-file imports + for names that are satisfied locally (e.g. pytest fixture names that appear as + test function parameters, or local variables like ``helpers = tmp_path / ...``). + + Decorators, argument default values, and return/argument annotations are + always evaluated in the outer scope and are never excluded. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + names: Set[str] = set() + + def _body_stores(stmts) -> frozenset: + """Names stored/deleted at this scope level in *stmts*. + + Recurses into control-flow nodes (if/for/while/with/try) but stops at + nested FunctionDef/AsyncFunctionDef/ClassDef scopes so only names that + are local to the current function are returned. + """ + stores: Set[str] = set() + work = list(stmts) + while work: + node = work.pop() + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + continue + if isinstance(node, ast.Name) and isinstance( + node.ctx, (ast.Store, ast.Del) + ): + stores.add(node.id) + work.extend(ast.iter_child_nodes(node)) + return frozenset(stores) + + def _walk(node: ast.AST, excluded: frozenset) -> None: + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): + if node.id not in excluded: + names.add(node.id) + return + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = node.args + own_params: frozenset = frozenset( + a.arg + for a in ( + args.args + + args.posonlyargs + + args.kwonlyargs + + ([args.vararg] if args.vararg else []) + + ([args.kwarg] if args.kwarg else []) + ) + ) + # Decorators are evaluated in the outer scope. + for dec in node.decorator_list: + _walk(dec, excluded) + # Default values are evaluated in the outer scope. + for default in args.defaults + args.kw_defaults: + if default is not None: + _walk(default, excluded) + # Annotations are in the outer scope (PEP 563 / regular annotations). + for arg in args.args + args.posonlyargs + args.kwonlyargs: + if arg.annotation: + _walk(arg.annotation, excluded) + if args.vararg and args.vararg.annotation: + _walk(args.vararg.annotation, excluded) + if args.kwarg and args.kwarg.annotation: + _walk(args.kwarg.annotation, excluded) + if node.returns: + _walk(node.returns, excluded) + # Function body uses params + local stores as the excluded set. + own_locals = _body_stores(node.body) + new_excluded = excluded | own_params | own_locals + for child in node.body: + _walk(child, new_excluded) + return + for child in ast.iter_child_nodes(node): + _walk(child, excluded) + + _walk(tree, frozenset()) + return names + + +def _collect_quoted_annotation_names(source: str) -> Set[str]: + """Return names referenced inside quoted type annotations in *source*. + + Finds names like ``_LLMAccumulator`` in ``Optional["_LLMAccumulator"]`` + (string literals used as forward references in type annotations). These + names are only needed at type-checking time — not at runtime — and should + be imported under ``if TYPE_CHECKING:`` rather than as regular imports. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + names: Set[str] = set() + + def _scan_annotation(node: ast.AST) -> None: + """Recursively scan an annotation, extracting names from string constants.""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + try: + inner = ast.parse(node.value, mode="eval") + for n in ast.walk(inner): + if isinstance(n, ast.Name): + names.add(n.id) + except SyntaxError: + pass + return + for child in ast.iter_child_nodes(node): + _scan_annotation(child) + + def _walk(node: ast.AST) -> None: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = node.args + for arg in args.args + args.posonlyargs + args.kwonlyargs: + if arg.annotation: + _scan_annotation(arg.annotation) + if args.vararg and args.vararg.annotation: + _scan_annotation(args.vararg.annotation) + if args.kwarg and args.kwarg.annotation: + _scan_annotation(args.kwarg.annotation) + if node.returns: + _scan_annotation(node.returns) + for child in node.body: + _walk(child) + return + if isinstance(node, ast.AnnAssign): + _scan_annotation(node.annotation) + if node.value: + _walk(node.value) + return + for child in ast.iter_child_nodes(node): + _walk(child) + + _walk(tree) + return names + + +def _collect_name_stores(source: str) -> Set[str]: + """Return names assigned at module level in *source*. + + Detects ``x = ...``, ``x += ...``, and annotated assignments with a value + (``x: int = ...``) at the top level of the module. Used to identify + TOP_LEVEL constants that are mutated outside their defining entity so that + cross-file references must use ``module.NAME`` rather than a plain + ``from .module import NAME``. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + stores: Set[str] = set() + for node in tree.body: + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + stores.add(target.id) + elif isinstance(node, ast.AugAssign): + if isinstance(node.target, ast.Name): + stores.add(node.target.id) + elif isinstance(node, ast.AnnAssign): + if node.value is not None and isinstance(node.target, ast.Name): + stores.add(node.target.id) + return stores + + +def _inject_module_level_imports(source: str, imports: List[str]) -> str: + """Insert *imports* after the last existing import line in *source*. + + Uses the same insertion logic as :func:`_add_re_exports` so that module + imports for reassigned TOP_LEVEL variables land in the same position as + other imports added to the original file. + """ + if not imports: + return source + lines = source.splitlines(keepends=True) + last_import_line = 0 + try: + tree = ast.parse(source) + except SyntaxError: + return "\n".join(sorted(imports)) + "\n\n" + source + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + last_import_line = max(last_import_line, node.end_lineno) + insert_after = last_import_line + if insert_after == 0 and tree.body: + first = tree.body[0] + if ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + ): + insert_after = first.end_lineno + import_lines = [imp + "\n" for imp in sorted(imports)] + return "".join(lines[:insert_after] + import_lines + lines[insert_after:]) + + +def _extract_import_info(source: str) -> List[ImportInfo]: + """Return :class:`ImportInfo` for each top-level import in *source*. + + Also includes imports found inside module-level ``if TYPE_CHECKING:`` + blocks, marked with ``is_type_checking=True``. These are used by + :func:`_find_type_checking_needed_imports` to distribute forward-reference + imports to the correct sub-files after a split. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return [] + + lines = source.splitlines(keepends=True) + result: List[ImportInfo] = [] + + for node in tree.body: + if isinstance(node, ast.Import): + names = [ + alias.asname if alias.asname else alias.name.split(".")[0] + for alias in node.names + ] + src = "".join(lines[node.lineno - 1 : node.end_lineno]).rstrip() + result.append(ImportInfo(names=names, source=src, is_future=False)) + elif isinstance(node, ast.ImportFrom): + names = [ + alias.asname if alias.asname else alias.name for alias in node.names + ] + # Reconstruct as a normalized single-line import so that + # multi-line parenthesized imports (e.g. ``from X import (\n + # Y,\n Z,\n)``) don't break _merge_from_imports, whose regex + # only matches the first line. + dots = "." * (node.level or 0) + mod = node.module or "" + alias_strs = [ + f"{a.name} as {a.asname}" if a.asname else a.name for a in node.names + ] + src = f"from {dots}{mod} import {', '.join(alias_strs)}" + is_future = node.module == "__future__" + result.append(ImportInfo(names=names, source=src, is_future=is_future)) + elif ( + isinstance(node, ast.If) + and isinstance(node.test, ast.Name) + and node.test.id == "TYPE_CHECKING" + ): + for child in node.body: + if isinstance(child, ast.Import): + tc_names = [ + alias.asname if alias.asname else alias.name.split(".")[0] + for alias in child.names + ] + tc_src = "".join( + lines[child.lineno - 1 : child.end_lineno] + ).rstrip() + result.append( + ImportInfo( + names=tc_names, + source=tc_src, + is_future=False, + is_type_checking=True, + ) + ) + elif isinstance(child, ast.ImportFrom): + tc_names = [ + alias.asname if alias.asname else alias.name + for alias in child.names + ] + tc_dots = "." * (child.level or 0) + tc_mod = child.module or "" + tc_alias_strs = [ + f"{a.name} as {a.asname}" if a.asname else a.name + for a in child.names + ] + tc_src = f"from {tc_dots}{tc_mod} import {', '.join(tc_alias_strs)}" + result.append( + ImportInfo( + names=tc_names, + source=tc_src, + is_future=False, + is_type_checking=True, + ) + ) + + return result + + +def _inject_type_checking_imports(source: str, imports: List[str]) -> str: + """Add *imports* under a module-level ``if TYPE_CHECKING:`` guard in *source*. + + If a TYPE_CHECKING block already exists, new imports are appended to it + (skipping any already present). Otherwise a new block is inserted after + the last top-level import statement, along with ``from typing import + TYPE_CHECKING`` when that name is not already imported. + """ + if not imports: + return source + try: + tree = ast.parse(source) + except SyntaxError: + return source + + # Determine which imports are not already in an existing TC block. + existing_tc = {i.source for i in _extract_import_info(source) if i.is_type_checking} + new_imports = [imp for imp in imports if imp not in existing_tc] + if not new_imports: + return source + + lines = source.splitlines(keepends=True) + + # Append to an existing TYPE_CHECKING block if one is present. + for node in tree.body: + if ( + isinstance(node, ast.If) + and isinstance(node.test, ast.Name) + and node.test.id == "TYPE_CHECKING" + ): + insert_line = node.end_lineno + new_lines = [" " + imp + "\n" for imp in sorted(new_imports)] + return "".join(lines[:insert_line] + new_lines + lines[insert_line:]) + + # No existing block: insert one after the last top-level import. + last_import_line = 0 + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + last_import_line = max(last_import_line, node.end_lineno) + insert_after = last_import_line + + tc_already_imported = any( + isinstance(n, ast.ImportFrom) + and n.module == "typing" + and any((a.asname or a.name) == "TYPE_CHECKING" for a in n.names) + for n in tree.body + ) + new_lines = [] + if not tc_already_imported: + new_lines.append("from typing import TYPE_CHECKING\n") + new_lines.append("if TYPE_CHECKING:\n") + for imp in sorted(new_imports): + new_lines.append(" " + imp + "\n") + new_lines.append("\n") + return "".join(lines[:insert_after] + new_lines + lines[insert_after:]) + + +def _test_names_in_decorators(source: str, names: Set[str]) -> Set[str]: + """Return the subset of *names* that appear as Name loads inside a decorator. + + Decorators are evaluated before function bodies run, so a symbol that + only reaches a file via an inline import (injected into the function body) + will not be in scope when the decorator is evaluated. This helper detects + that situation so callers can abort the split rather than generate broken + code. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + found: Set[str] = set() + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + for dec in node.decorator_list: + for child in ast.walk(dec): + if ( + isinstance(child, ast.Name) + and isinstance(child.ctx, ast.Load) + and child.id in names + ): + found.add(child.id) + return found + + +def _find_needed_imports( + entity_names: List[str], + entity_source_map: Dict[str, str], + import_infos: List[ImportInfo], + all_entity_names: Set[str], +) -> List[str]: + """Return import statements needed by the given entities. + + Always includes ``from __future__`` imports. Other imports are included + when any of the names they introduce appear in the entities' source. + Duplicate import source strings are deduplicated. + """ + referenced: Set[str] = set() + for name in entity_names: + src = entity_source_map.get(name, "") + referenced |= _collect_name_loads(src) + + needed: List[str] = [] + seen: Set[str] = set() + for info in import_infos: + if info.source in seen: + continue + if info.is_type_checking: + continue # handled by _find_type_checking_needed_imports + if info.is_future or any(n in referenced for n in info.names): + needed.append(info.source) + seen.add(info.source) + + return needed + + +def _narrow_import_source(import_src: str, keep_names: Set[str]) -> str: + """Return a copy of *import_src* keeping only the exposed names in *keep_names*. + + For ``from X import A, B, C`` with ``keep_names={A}``, returns + ``from X import A``. Non-ImportFrom statements are returned unchanged. + """ + try: + node = ast.parse(import_src).body[0] + except (SyntaxError, IndexError): + return import_src + if not isinstance(node, ast.ImportFrom): + return import_src + dots = "." * (node.level or 0) + mod = node.module or "" + alias_strs = [ + f"{a.name} as {a.asname}" if a.asname else a.name + for a in node.names + if (a.asname or a.name) in keep_names + ] + if not alias_strs: + return import_src + return f"from {dots}{mod} import {', '.join(alias_strs)}" + + +def _find_type_checking_needed_imports( + entity_names: List[str], + entity_source_map: Dict[str, str], + import_infos: List[ImportInfo], +) -> List[str]: + """Return import statements needed only for quoted type annotations. + + These should be placed under ``if TYPE_CHECKING:`` because the names are + only referenced inside string-valued annotations (forward references) and + are not needed at runtime. Names that appear in regular (non-annotation) + loads are excluded via ``annotation_only = quoted - runtime``, which + guarantees that any name emitted here will be pruned from regular imports + by ``_prune_unused_imports`` — so no duplicate imports can arise. + ``__future__`` imports are always excluded since they are handled by + ``_find_needed_imports``. + """ + runtime: Set[str] = set() + quoted: Set[str] = set() + for name in entity_names: + src = entity_source_map.get(name, "") + runtime |= _collect_name_loads(src) + quoted |= _collect_quoted_annotation_names(src) + + annotation_only = quoted - runtime + if not annotation_only: + return [] + + needed: List[str] = [] + seen: Set[str] = set() + for info in import_infos: + if info.source in seen: + continue + if info.is_future: + continue + tc_names = {n for n in info.names if n in annotation_only} + if not tc_names: + continue + # Narrow the import to only the names actually needed for type checking, + # avoiding unused-import warnings for names from multi-name imports that + # are not referenced in this file. + src = ( + info.source + if len(tc_names) == len(info.names) + else _narrow_import_source(info.source, tc_names) + ) + if src in seen: + continue + needed.append(src) + seen.add(src) + return needed diff --git a/crispen/file_limiter/code_gen/source_utils.py b/crispen/file_limiter/code_gen/source_utils.py new file mode 100644 index 0000000..d5aca2b --- /dev/null +++ b/crispen/file_limiter/code_gen/source_utils.py @@ -0,0 +1,228 @@ +from __future__ import annotations +from typing import List, Optional, Set, Tuple +import ast +import io +import re +import tokenize +from ..entity_parser import _parse_section_headers + + +# Matches any line that is an import statement (plain or from-import). +_IMPORT_LINE_RE = re.compile(r"^(import\s+|from\s+\S.*\s+import\s+)") + +# Matches a `from __future__ import …` line (with optional trailing newline). +_FUTURE_IMPORT_LINE_RE = re.compile(r"^from __future__ import .*\n?", re.MULTILINE) + +# Matches the leading dots of a relative import (``from .foo`` or ``from ..``). +_REL_IMPORT_RE = re.compile(r"^from (\.+)", re.MULTILINE) + +# Matches four or more consecutive newlines (= 3+ blank lines between entities). +_EXCESS_BLANK_RE = re.compile(r"\n{4,}") +# Matches 3+ consecutive newlines followed by indented content (= 2+ blank lines +# inside a function/class body, where flake8 E303 allows at most one blank line). +_EXCESS_BLANK_BODY_RE = re.compile(r"\n{3,}(?=[ \t])") + + +def _multiline_string_ranges(source: str) -> List[Tuple[int, int]]: + """Return (start, end) character offsets for every multi-line string literal. + + Uses the tokenizer so that triple-quoted strings containing blank lines + followed by indented content are not mistakenly collapsed by blank-line + normalization regexes. Falls back to an empty list on tokenization error + (e.g. if the source is not yet valid Python), preserving original behavior. + """ + ranges: List[Tuple[int, int]] = [] + lines = source.splitlines(keepends=True) + # cumulative[i] = byte offset of the start of line i (0-indexed) + cumulative = [0] + for line in lines: + cumulative.append(cumulative[-1] + len(line)) + try: + tokens = tokenize.generate_tokens(io.StringIO(source).readline) + for tok_type, tok_string, tok_start, tok_end, _ in tokens: + if tok_type == tokenize.STRING and "\n" in tok_string: + start = cumulative[tok_start[0] - 1] + tok_start[1] + end = cumulative[tok_end[0] - 1] + tok_end[1] + ranges.append((start, end)) + except tokenize.TokenError: + pass + return ranges + + +def _sub_skip_strings(pattern: re.Pattern, repl: str, source: str) -> str: + """Apply *pattern*.sub(*repl*, ...) to *source*, skipping string literals. + + Blank-line normalization must not alter content inside string literals (e.g. + source code stored in a dedented triple-quoted string used in tests). + """ + ranges = _multiline_string_ranges(source) + if not ranges: + return pattern.sub(repl, source) + parts: List[str] = [] + last = 0 + for start, end in ranges: + parts.append(pattern.sub(repl, source[last:start])) + parts.append(source[start:end]) + last = end + parts.append(pattern.sub(repl, source[last:])) + return "".join(parts) + + +def _normalize_blank_lines(source: str) -> str: + """Collapse excess blank lines; ensure exactly one trailing newline. + + Removes blank-line artefacts produced by entity removal (original file) + and entity-source stripping (new files): + + - Strips leading blank lines at the start of the file (E303). + - Collapses 3+ consecutive blank lines between top-level definitions to 2 + (E303; PEP 8 allows at most two blank lines at module level). + - Collapses 2+ consecutive blank lines inside indented bodies to 1 + (E303; PEP 8 allows at most one blank line inside a function/class). + + Returns an empty string when *source* contains only whitespace, signalling + that the file should be deleted rather than written with a lone blank line. + + Multi-line string literals are protected: blank lines inside them are never + collapsed, so stored source-code snippets (e.g. in test fixtures) are not + mutated. + """ + source = _sub_skip_strings(_EXCESS_BLANK_RE, "\n\n\n", source) + source = _sub_skip_strings(_EXCESS_BLANK_BODY_RE, "\n\n", source) + source = source.lstrip("\n") + stripped = source.rstrip("\n") + if not stripped.strip(): + return "" + return stripped + "\n" + + +def _strip_orphaned_section_headers(source: str) -> str: + """Remove section header comment blocks with no substantive code after them. + + When entities are removed from the original file, section headers that + labelled a group of functions may be left with nothing beneath them. + This function detects both 3-line (``# ---...--- / # Label / # ---...---``) + and single-line (``# --- Label ---``, ``# === LABEL ===``) patterns and + removes any whose remaining content (non-blank, non-header lines) has + been entirely stripped away. + """ + lines = source.splitlines(keepends=True) + headers = _parse_section_headers(lines) + if not headers: + return source + + # 1-indexed set of lines that belong to any header block. + header_1idx: Set[int] = set() + for start, end, _ in headers: + header_1idx.update(range(start, end + 1)) + + # A header is orphaned when no substantive line (non-blank and not part of + # any header block) falls between it and the *next* header (or EOF). + orphaned_0idx: Set[int] = set() + for h_idx, (start_1, end_1, _) in enumerate(headers): + # Scan only up to the start of the next header so that content beneath + # a later header does not rescue an earlier, empty one. + if h_idx + 1 < len(headers): + scan_end_0 = headers[h_idx + 1][0] - 1 # 0-indexed exclusive + else: + scan_end_0 = len(lines) + has_content = False + for j0 in range(end_1, scan_end_0): # 0-indexed, past the header block + stripped = lines[j0].strip() + if stripped and (j0 + 1) not in header_1idx: + has_content = True + break + if not has_content: + for i1 in range(start_1, end_1 + 1): + orphaned_0idx.add(i1 - 1) # convert to 0-indexed + + if not orphaned_0idx: + return source + return "".join(line for i, line in enumerate(lines) if i not in orphaned_0idx) + + +def _strip_orphaned_indented_comments(source: str) -> str: + """Remove indented comment lines that appear at module level. + + After FileLimiter moves a function to a new file using AST line ranges, + trailing comments that were inside the function body may be left behind + in the original file. These comments retain their original indentation + (e.g. four spaces) even though they are now at module level, causing + flake8 E116 (unexpected indentation: comment). + + This function uses ``ast.parse`` to build the set of line numbers covered + by any AST node. Any comment line with leading whitespace whose line + number falls outside that set is considered orphaned and removed. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return source + + covered: Set[int] = set() + for node in ast.walk(tree): + if hasattr(node, "lineno") and hasattr(node, "end_lineno"): + for lineno in range(node.lineno, node.end_lineno + 1): + covered.add(lineno) + + lines = source.splitlines(keepends=True) + result = [] + for i, line in enumerate(lines): + lineno = i + 1 # 1-indexed + stripped = line.lstrip() + is_indented_comment = stripped.startswith("#") and len(line) > len(stripped) + if is_indented_comment and lineno not in covered: + continue + result.append(line) + return "".join(result) + + +def _extract_module_docstring(source: str) -> Optional[str]: + """Return the module-level docstring source text, or None if absent.""" + try: + tree = ast.parse(source) + except SyntaxError: + return None + if not ( + tree.body + and isinstance(tree.body[0], ast.Expr) + and isinstance(tree.body[0].value, ast.Constant) + and isinstance(tree.body[0].value.value, str) + ): + return None + node = tree.body[0] + lines = source.splitlines(keepends=True) + return "".join(lines[node.lineno - 1 : node.end_lineno]).rstrip() + + +def _strip_module_docstring(src: str) -> str: + """Return *src* with the leading module-level docstring removed.""" + try: + tree = ast.parse(src) + except SyntaxError: + return src + if not ( + tree.body + and isinstance(tree.body[0], ast.Expr) + and isinstance(tree.body[0].value, ast.Constant) + and isinstance(tree.body[0].value.value, str) + ): + return src + node = tree.body[0] + remove = set(range(node.lineno, node.end_lineno + 1)) + lines = src.splitlines(keepends=True) + return "".join(line for i, line in enumerate(lines, 1) if i not in remove) + + +def _source_is_only_docstring(source: str) -> bool: + """Return True if *source* contains only a module-level docstring.""" + try: + tree = ast.parse(source) + except SyntaxError: + return False + return ( + len(tree.body) == 1 + and isinstance(tree.body[0], ast.Expr) + and isinstance(tree.body[0].value, ast.Constant) + and isinstance(tree.body[0].value.value, str) + ) diff --git a/crispen/file_limiter/code_gen/test_support.py b/crispen/file_limiter/code_gen/test_support.py new file mode 100644 index 0000000..5689286 --- /dev/null +++ b/crispen/file_limiter/code_gen/test_support.py @@ -0,0 +1,538 @@ +from __future__ import annotations +from typing import Dict, List, Optional, Set, Tuple +import ast +import re +from ..entity_parser import Entity, EntityKind +from .cross_file_deps import _relative_import_prefix, _target_module_name + + +def _class_has_test_methods(entity_src: str) -> bool: + """Return True if *entity_src* defines a class with any ``test_`` methods. + + Used to suppress re-exports of test classes: pytest discovers test classes + by scanning the filesystem, so re-exporting them from the original file + causes every test inside to run twice. + """ + try: + tree = ast.parse(entity_src) + except SyntaxError: + return False + for node in tree.body: + if isinstance(node, ast.ClassDef): + for item in node.body: + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + if item.name.startswith("test_"): + return True + return False + + +def _is_test_name(name: str) -> bool: + """Return True if *name* matches pytest's test-discovery patterns. + + Pytest collects classes named ``Test*`` and functions named ``test_*``. + Importing such names at module level in a test file causes every test + inside to be discovered — and run — a second time. + """ + return name.startswith("Test") or name.startswith("test_") + + +def _is_pytest_fixture(entity_src: str) -> bool: + """Return True if *entity_src* defines a function with a @pytest.fixture decorator. + + Handles all common forms: ``@fixture``, ``@fixture()``, ``@pytest.fixture``, + and ``@pytest.fixture(scope=...)``. + """ + try: + tree = ast.parse(entity_src) + except SyntaxError: + return False + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for dec in node.decorator_list: + # Unwrap call forms like @pytest.fixture(...) to get the base reference. + ref = dec.func if isinstance(dec, ast.Call) else dec + if isinstance(ref, ast.Name) and ref.id == "fixture": + return True + if isinstance(ref, ast.Attribute) and ref.attr == "fixture": + return True + return False + + +def _file_has_only_fixtures(source: str) -> bool: + """Return True if *source* has at least one @pytest.fixture and nothing else. + + "Nothing else" means no test functions (``def test_*``), no test classes + (``class Test*``), no other function/class definitions, and no non-import + module-level statements other than a module docstring. Import statements + and a leading docstring are allowed because they are needed to support the + fixture definitions. + + Returns False on syntax errors (be conservative). + """ + try: + tree = ast.parse(source) + except SyntaxError: + return False + lines = source.splitlines(keepends=True) + has_fixture = False + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + continue + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant): + continue # module docstring or standalone string literal + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + first_line = ( + node.decorator_list[0].lineno if node.decorator_list else node.lineno + ) + fn_src = "".join(lines[first_line - 1 : node.end_lineno]).rstrip() + if _is_pytest_fixture(fn_src): + has_fixture = True + continue + return False # non-fixture function (including test_ functions) + return False # class, assignment, or other statement + return has_fixture + + +def _split_cross_imports_by_test( + imports: List[str], +) -> Tuple[List[str], List[str]]: + """Split cross-file import statements into (non_test, test_named) groups. + + Import statements that name pytest-discoverable symbols (``Test*`` or + ``test_*``) are returned as inline imports so callers can inject them + into function/class bodies rather than emitting them at module level. + Mixed imports (some test, some non-test names) are split into two + separate statements. + """ + non_test: List[str] = [] + test_named: List[str] = [] + for imp in imports: + m = re.match(r"^(from\s+\S+\s+import\s+)(.*)", imp) + if not m: + non_test.append(imp) + continue + prefix = m.group(1) + names = [n.strip() for n in m.group(2).split(",")] + t_names = sorted(n for n in names if _is_test_name(n)) + nt_names = sorted(n for n in names if not _is_test_name(n)) + if t_names: + test_named.append(f"{prefix}{', '.join(t_names)}") + if nt_names: + non_test.append(f"{prefix}{', '.join(nt_names)}") + return non_test, test_named + + +def _inject_inline_imports(entity_src: str, imports: List[str]) -> str: + """Inject *imports* at the top of a function or class body in *entity_src*. + + The imports are inserted after any leading docstring. Returns + *entity_src* unchanged when it cannot be parsed or the top-level node + is not a function or class (TOP_LEVEL entities have no body scope). + """ + if not imports: + return entity_src + try: + tree = ast.parse(entity_src) + except SyntaxError: + return entity_src + if not tree.body: + return entity_src + top = tree.body[0] + if not isinstance(top, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return entity_src + first_stmt = top.body[0] + insert_line = first_stmt.lineno + if ( + isinstance(first_stmt, ast.Expr) + and isinstance(first_stmt.value, ast.Constant) + and isinstance(first_stmt.value.value, str) + and len(top.body) > 1 + ): + insert_line = top.body[1].lineno + lines = entity_src.splitlines(keepends=True) + body_line = lines[insert_line - 1] + indent = body_line[: len(body_line) - len(body_line.lstrip())] + import_lines = [f"{indent}{imp}\n" for imp in imports] + return "".join(lines[: insert_line - 1] + import_lines + lines[insert_line - 1 :]) + + +def _find_main_block_entity( + entities: List[Entity], + entity_source_map: Dict[str, str], +) -> Optional[str]: + """Return the entity name of the ``if __name__ == '__main__':`` block. + + Returns ``None`` when no such block is present. + """ + for entity in entities: + if entity.kind != EntityKind.TOP_LEVEL: + continue + src = entity_source_map.get(entity.name, "") + try: + tree = ast.parse(src) + except SyntaxError: + continue + for node in tree.body: + if ( + isinstance(node, ast.If) + and isinstance(node.test, ast.Compare) + and isinstance(node.test.left, ast.Name) + and node.test.left.id == "__name__" + and len(node.test.ops) == 1 + and isinstance(node.test.ops[0], ast.Eq) + and len(node.test.comparators) == 1 + and isinstance(node.test.comparators[0], ast.Constant) + and node.test.comparators[0].value == "__main__" + ): + return entity.name + return None + + +def _find_main_direct_callees( + main_src: str, function_entity_names: Set[str] +) -> Set[str]: + """Return function entity names called directly in the ``__main__`` block. + + Only names that appear in *function_entity_names* (i.e. are defined as + top-level FUNCTION entities in the same file) are returned, so the + caller can keep those functions sticky to the original file alongside + the ``__main__`` block. + """ + try: + tree = ast.parse(main_src) + except SyntaxError: + return set() + callees: Set[str] = set() + for node in tree.body: + if not ( + isinstance(node, ast.If) + and isinstance(node.test, ast.Compare) + and isinstance(node.test.left, ast.Name) + and node.test.left.id == "__name__" + and len(node.test.ops) == 1 + and isinstance(node.test.ops[0], ast.Eq) + and len(node.test.comparators) == 1 + and isinstance(node.test.comparators[0], ast.Constant) + and node.test.comparators[0].value == "__main__" + ): + continue + for subnode in ast.walk(node): + if ( + isinstance(subnode, ast.Call) + and isinstance(subnode.func, ast.Name) + and subnode.func.id in function_entity_names + ): + callees.add(subnode.func.id) + return callees + + +def _inject_inline_test_imports_original( + source: str, + migrated_test_symbols: Dict[str, str], + abs_pkg: Optional[str], + original_basename: str, +) -> str: + """Inject inline imports for migrated test-named symbols into function/class bodies. + + After a split, test-named symbols (``Test*`` / ``test_*``) that were + migrated to new files are not re-exported at module level (to avoid + pytest double-discovery). This function finds every top-level + function or class in *source* that still references such symbols and + injects the required ``from … import …`` statement at the top of + each body, after any docstring. + + *migrated_test_symbols* maps each migrated test name to its target + file (relative path). *abs_pkg* and *original_basename* are used to + build the correct import prefix (absolute for test files, relative + otherwise). + """ + if not migrated_test_symbols: + return source + try: + tree = ast.parse(source) + except SyntaxError: + return source + + lines = source.splitlines(keepends=True) + # Maps 1-based line number → list of import lines to insert before it. + insertions: Dict[int, List[str]] = {} + + for node in tree.body: + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + continue + body_names: Set[str] = set() + for subnode in ast.walk(node): + if isinstance(subnode, ast.Name) and isinstance(subnode.ctx, ast.Load): + body_names.add(subnode.id) + needed: Dict[str, List[str]] = {} + for name in body_names: + tfile = migrated_test_symbols.get(name) + if tfile: + needed.setdefault(tfile, []).append(name) + if not needed: + continue + import_stmts: List[str] = [] + for tfile, names in sorted(needed.items()): + if abs_pkg is not None: + mod = _target_module_name(tfile) + prefix = f"{abs_pkg}.{mod}" if abs_pkg else mod + else: + prefix = _relative_import_prefix(original_basename, tfile) + import_stmts.append(f"from {prefix} import {', '.join(sorted(names))}") + first_stmt = node.body[0] + insert_line = first_stmt.lineno + if ( + isinstance(first_stmt, ast.Expr) + and isinstance(first_stmt.value, ast.Constant) + and isinstance(first_stmt.value.value, str) + and len(node.body) > 1 + ): + insert_line = node.body[1].lineno + body_line = lines[insert_line - 1] + indent = body_line[: len(body_line) - len(body_line.lstrip())] + insertions.setdefault(insert_line, []) + insertions[insert_line] = [f"{indent}{s}\n" for s in import_stmts] + insertions[ + insert_line + ] + + if not insertions: + return source + result: List[str] = [] + for i, line in enumerate(lines, 1): + if i in insertions: + result.extend(insertions[i]) + result.append(line) + return "".join(result) + + +def _merge_conftest_sources(existing: str, new_content: str) -> str: + """Merge *new_content* into an existing conftest.py without duplicating anything. + + When multiple file splits each contribute fixtures to the same conftest.py, + naively appending produces duplicate import statements, duplicate function + definitions, and E402 errors (imports after function definitions). + + This function avoids all three: + - Duplicate import statements (same module + same names) are skipped. + - Function/class definitions whose names already appear in *existing* are skipped. + - Non-duplicate imports from *new_content* are inserted after the last existing + import (before any existing function definitions), preventing E402. + - Non-duplicate definitions are appended at the end. + + Falls back to simple concatenation when either source cannot be parsed. + """ + try: + existing_tree = ast.parse(existing) + new_tree = ast.parse(new_content) + except SyntaxError: + return existing.rstrip() + "\n\n\n" + new_content + + existing_lines = existing.splitlines(keepends=True) + new_lines = new_content.splitlines(keepends=True) + + def _import_key(node: ast.stmt) -> str: + if isinstance(node, ast.Import): + return "I:" + ",".join( + sorted(f"{a.name}:{a.asname or ''}" for a in node.names) + ) + assert isinstance(node, ast.ImportFrom) + dots = "." * (node.level or 0) + mod = node.module or "" + return ( + "F:" + + dots + + mod + + ":" + + ",".join(sorted(f"{a.name}:{a.asname or ''}" for a in node.names)) + ) + + # What is already in existing? + existing_import_keys: Set[str] = { + _import_key(n) + for n in existing_tree.body + if isinstance(n, (ast.Import, ast.ImportFrom)) + } + existing_defined_names: Set[str] = { + n.name + for n in existing_tree.body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + } + + # Last import line in existing (0-indexed insertion point). + last_import_lineno: int = 0 + for n in existing_tree.body: + if isinstance(n, (ast.Import, ast.ImportFrom)): + last_import_lineno = max(last_import_lineno, n.end_lineno) + + # Collect new, non-duplicate imports and definitions from new_content. + imports_to_insert: List[str] = [] + defs_to_append: List[str] = [] + + for node in new_tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + if _import_key(node) not in existing_import_keys: + src = "".join(new_lines[node.lineno - 1 : node.end_lineno]).rstrip() + imports_to_insert.append(src) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + if node.name not in existing_defined_names: + first_line = ( + node.decorator_list[0].lineno + if node.decorator_list + else node.lineno + ) + src = "".join(new_lines[first_line - 1 : node.end_lineno]).rstrip() + defs_to_append.append(src) + + if not imports_to_insert and not defs_to_append: + return existing + + result_lines = list(existing_lines) + if imports_to_insert: + # Insert new imports directly after the last existing import line. + insert_at = last_import_lineno # 0-indexed position after last import + new_import_lines = [imp + "\n" for imp in imports_to_insert] + result_lines = ( + result_lines[:insert_at] + new_import_lines + result_lines[insert_at:] + ) + + result = "".join(result_lines).rstrip() + if defs_to_append: + result = result + "\n\n\n" + "\n\n\n".join(defs_to_append) + "\n" + else: + result = result + "\n" + return result + + +def _rewrite_module_var_names(src: str, rewrites: Dict[str, str]) -> str: + """Replace bare ``Name`` loads with ``module.name`` attribute accesses. + + Uses the AST to locate exact positions of ``Name`` load nodes whose + ``id`` is in *rewrites*, replacing each with its qualified form (e.g. + ``"SAFE_MODE"`` → ``"conversion.SAFE_MODE"``). + + Because ``ast.Name`` nodes **never** represent the attribute part of an + ``Attribute`` node (which stores ``attr`` as a plain string), this + approach is immune to the corruption that a regex would cause on + ``obj.SAFE_MODE`` and naturally skips string literals and comments. + + After rewriting, the result is re-parsed and every original ``Name`` + load for each rewritten identifier is verified to be absent. If + verification fails the original source is returned unchanged so that + callers can fall back to direct-import semantics rather than corrupt + the output. + """ + if not rewrites: + return src + try: + tree = ast.parse(src) + except SyntaxError: + return src + + lines = src.splitlines(keepends=True) + + # Collect (lineno, col_offset, end_col_offset, new_text). + # ast uses 1-indexed lineno and 0-indexed col_offset / end_col_offset. + edits: List[Tuple[int, int, int, str]] = [] + for node in ast.walk(tree): + if ( + isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Load) + and node.id in rewrites + ): + edits.append( + (node.lineno, node.col_offset, node.end_col_offset, rewrites[node.id]) + ) + + if not edits: + return src + + # Apply edits from last to first within each line to keep earlier offsets valid. + edits.sort(key=lambda e: (e[0], e[1]), reverse=True) + for lineno, col_start, col_end, new_text in edits: + line = lines[lineno - 1] + lines[lineno - 1] = line[:col_start] + new_text + line[col_end:] + + result = "".join(lines) + + # Verification: re-parse and confirm no bare Name loads remain for any + # rewritten identifier. If the result is unparseable or a bare name + # survives, return the original source to avoid corrupting the output. + try: + new_tree = ast.parse(result) + except SyntaxError: + return src + for node in ast.walk(new_tree): + if ( + isinstance(node, ast.Name) + and isinstance(node.ctx, ast.Load) + and node.id in rewrites + ): + return src + + return result + + +def _rewrite_module_level_stores(src: str, rewrites: Dict[str, str]) -> str: + """Rewrite module-level Name store targets to ``module.name`` attribute stores. + + Only statements at the top level of the module are affected + (``ast.Module.body``). Assignments inside function or class bodies are + left unchanged so that local variable bindings are not corrupted. + + Used for the non-migrated home file: when a non-migrated entity reassigns + a TOP_LEVEL constant that was moved to a sub-file, the assignment must be + rewritten as ``module.CONST = expr`` so that the mutation updates the + canonical value in the sub-file rather than creating an orphaned local + binding. + """ + if not rewrites: + return src + try: + tree = ast.parse(src) + except SyntaxError: + return src + lines = src.splitlines(keepends=True) + edits: List[Tuple[int, int, int, str]] = [] + for node in tree.body: + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id in rewrites: + edits.append( + ( + target.lineno, + target.col_offset, + target.end_col_offset, + rewrites[target.id], + ) + ) + elif isinstance(node, ast.AugAssign): + if isinstance(node.target, ast.Name) and node.target.id in rewrites: + edits.append( + ( + node.target.lineno, + node.target.col_offset, + node.target.end_col_offset, + rewrites[node.target.id], + ) + ) + elif isinstance(node, ast.AnnAssign): + if ( + node.value is not None + and isinstance(node.target, ast.Name) + and node.target.id in rewrites + ): + edits.append( + ( + node.target.lineno, + node.target.col_offset, + node.target.end_col_offset, + rewrites[node.target.id], + ) + ) + if not edits: + return src + edits.sort(key=lambda e: (e[0], e[1]), reverse=True) + for lineno, col_start, col_end, new_text in edits: + line = lines[lineno - 1] + lines[lineno - 1] = line[:col_start] + new_text + line[col_end:] + return "".join(lines) diff --git a/crispen/file_limiter/code_gen/transforms.py b/crispen/file_limiter/code_gen/transforms.py new file mode 100644 index 0000000..b17cb7b --- /dev/null +++ b/crispen/file_limiter/code_gen/transforms.py @@ -0,0 +1,689 @@ +from __future__ import annotations +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Set +import ast +import re +from ..advisor import GroupPlacement +from ..classifier import ClassifiedEntities +from ..dep_graph import find_sccs +from ..entity_parser import Entity, EntityKind +from .cross_file_deps import _relative_import_prefix, _target_module_name +from .import_analysis import _collect_name_loads, _import_derived_names +from .test_support import _class_has_test_methods, _is_test_name + + +@dataclass +class SplitResult: + """Output of :func:`generate_file_splits`.""" + + new_files: Dict[str, str] # {target_file: source_code} + original_source: str # updated original file source + abort: bool # True if generation failed / nothing to split + abort_reason: str = "" # human-readable explanation when abort=True + entity_name_rewrites: Dict[str, Dict[str, str]] = field( + default_factory=dict + ) # {entity_name: {old_name: new_qualified_name}} per migrated entity + actual_placements: List[GroupPlacement] = field( + default_factory=list + ) # final placements after conftest routing (for accurate output messages) + + +_FROM_IMPORT_RE = re.compile(r"^(from\s+\S+)\s+import\s+(.*)") + + +def _merge_from_imports(imports: List[str]) -> List[str]: + """Merge ``from X import …`` lines that share the same module prefix. + + When multiple entities each contribute a ``from X import`` for the same + module but with different name subsets, the naive per-entity approach + produces duplicate imports such as:: + + from .conversion import lua_to_python, python_to_lua + from .conversion import lua_to_python_preserve_wrapped, python_to_lua + + This function collapses them into a single statement per prefix, with + names sorted and deduplicated:: + + from .conversion import lua_to_python, lua_to_python_preserve_wrapped, python_to_lua # noqa: E501 + + Plain ``import X`` statements are preserved unchanged and appended after + the merged from-imports. + """ + from_map: Dict[str, List[str]] = {} + order: List[str] = [] # first-seen order of prefixes + plain: List[str] = [] + for imp in imports: + m = _FROM_IMPORT_RE.match(imp) + if not m: + plain.append(imp) + continue + prefix = m.group(1) + names = [n.strip() for n in m.group(2).split(",") if n.strip()] + if prefix not in from_map: + from_map[prefix] = [] + order.append(prefix) + from_map[prefix].extend(names) + result = [] + for prefix in order: + unique = sorted(dict.fromkeys(from_map[prefix])) + result.append(f"{prefix} import {', '.join(unique)}") + return result + plain + + +def _import_line_numbers(entity: Entity, entity_src: str) -> Set[int]: + """Return absolute 1-based line numbers of import statements in *entity*. + + Used to preserve import lines in the original file when a TOP_LEVEL + entity that mixes imports and assignments is migrated. + """ + try: + tree = ast.parse(entity_src) + except SyntaxError: + return set() + result: Set[int] = set() + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + for rel_ln in range(node.lineno, node.end_lineno + 1): + result.add(entity.start_line + rel_ln - 1) + return result + + +def _remove_entity_lines( + source: str, + migrated_names: Set[str], + entity_map: Dict[str, Entity], + entity_source_map: Dict[str, str], +) -> str: + """Return *source* with lines belonging to migrated entities removed. + + For TOP_LEVEL entities, import statement lines are preserved in the + original file even when the entity is migrated: the remaining code may + still reference those imported names, and stdlib/third-party names + cannot be safely re-exported from a new module. + """ + remove: Set[int] = set() + preserve: Set[int] = set() + for name in migrated_names: + entity = entity_map.get(name) + if entity is None: + continue + for ln in range(entity.start_line, entity.end_line + 1): + remove.add(ln) + if entity.kind == EntityKind.TOP_LEVEL: + preserve |= _import_line_numbers(entity, entity_source_map.get(name, "")) + + lines = source.splitlines(keepends=True) + return "".join( + line for i, line in enumerate(lines, 1) if i not in remove or i in preserve + ) + + +def _add_re_exports( + source: str, + placements: List[GroupPlacement], + entity_map: Dict[str, Entity], + entity_source_map: Dict[str, str], + external_loads: Set[str] = frozenset(), + abs_pkg: Optional[str] = None, + relative_from: Optional[str] = None, + is_test_file: bool = False, + reexport_mode: str = "always", +) -> str: + """Add ``from .module import name`` imports for migrated entities. + + *reexport_mode* controls when public (non-underscore) names get a + re-export stub: + + * ``"always"`` — always re-export every public name (default; most + conservative, preserves the full public API regardless of usage). + * ``"application"`` — re-export public names in non-test files only. + * ``"imported"`` — re-export a public name only when it appears in + *external_loads* (imported from the original module by another file in + the project) or is still referenced in the remaining *source*. + + Private names (starting with ``_``) are always re-exported when the + remaining *source* still references them, or when they appear in + *external_loads*, regardless of *reexport_mode*. + + When *relative_from* is set (e.g. ``"service/__init__.py"``), import + prefixes are computed via :func:`_relative_import_prefix` so that + re-exports from a package ``__init__.py`` reference sibling modules + correctly (e.g. ``from .utils import Foo`` instead of + ``from .service.utils import Foo``). + + Import-derived names (names introduced by ``import`` / ``from … import`` + statements inside a TOP_LEVEL entity) are never re-exported: they were + kept in the original file by :func:`_remove_entity_lines` and cannot + meaningfully be re-exported from a new module. + + Inserts after the last import line in *source*. Returns *source* unchanged + when there are no names to import. + """ + still_loaded = _collect_name_loads(source) + re_exports: Dict[str, List[str]] = {} + # Names added solely for external re-export (not referenced in remaining source). + # These need "# fmt: skip # noqa: F401, E501" to suppress flake8 false positives. + noqa_names: Set[str] = set() + for placement in placements: + # Compute the import prefix for this placement's target file. + if relative_from is not None: + import_prefix = _relative_import_prefix( + relative_from, placement.target_file + ) + elif abs_pkg is not None: + module = _target_module_name(placement.target_file) + import_prefix = f"{abs_pkg}.{module}" if abs_pkg else module + else: + module = _target_module_name(placement.target_file) + import_prefix = f".{module}" + to_import: List[str] = [] + for entity_name in placement.group: + if entity_name in entity_map: + entity = entity_map[entity_name] + defined = entity.names_defined + if entity.kind == EntityKind.TOP_LEVEL: + skip = _import_derived_names(entity_source_map.get(entity_name, "")) + defined = [n for n in defined if n not in skip] + else: + defined = [entity_name] + is_test_class = entity_name in entity_map and _class_has_test_methods( + entity_source_map.get(entity_name, "") + ) + for defined_name in defined: + # Test-named symbols (Test* / test_*) are never re-exported at + # module level: _inject_inline_test_imports_original injects + # them inside function/class bodies to prevent pytest from + # discovering the same test twice. + if _is_test_name(defined_name): + continue + # Unconditional public re-export: only when reexport_mode + # permits it for this file type. + reexport_unconditionally = ( + not defined_name.startswith("_") + and not defined_name.startswith("test_") + and not is_test_class + and ( + reexport_mode == "always" + or (reexport_mode == "application" and not is_test_file) + ) + ) + if ( + reexport_unconditionally + or defined_name in still_loaded + or defined_name in external_loads + ): + to_import.append(defined_name) + # Add noqa when the name is not referenced in the remaining + # source (pure re-export stub), OR when it is in external_loads + # — in the latter case a non-migrated entity may currently use + # the name, but if that entity is itself migrated in a later + # recursive split the stub would become unreferenced and + # _prune_unused_imports would silently drop it, breaking the + # external caller. The noqa marker protects against that. + if ( + defined_name not in still_loaded + or defined_name in external_loads + ): + noqa_names.add(defined_name) + if to_import: + re_exports.setdefault(import_prefix, []).extend(to_import) + + if not re_exports: + return source + + # Build export statements. When a name is only there for external re-export + # (not referenced in the remaining source), add "# fmt: skip # noqa: F401, E501" + # so flake8 does not flag it as an unused import and Black does not reformat + # the line (which would break the noqa directive). Split mixed imports into + # two lines so that the noqa comment does not suppress warnings for used names. + export_stmts: List[str] = [] + for prefix, names in sorted(re_exports.items()): + sorted_names = sorted(names) + used = [n for n in sorted_names if n not in noqa_names] + noqa = [n for n in sorted_names if n in noqa_names] + if used: + export_stmts.append(f"from {prefix} import {', '.join(used)}\n") + for name in noqa: + export_stmts.append( + f"from {prefix} import {name} # fmt: skip # noqa: F401, E501\n" + ) + + # In test files, add a single explanatory comment before the first F401 import. + if is_test_file and noqa_names: + first_noqa = next(i for i, s in enumerate(export_stmts) if "# noqa: F401" in s) + export_stmts.insert( + first_noqa, + "# Re-exported for backwards compatibility with external callers.\n", + ) + + lines = source.splitlines(keepends=True) + last_import_line = 0 + try: + tree = ast.parse(source) + except SyntaxError: + return source + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + last_import_line = max(last_import_line, node.end_lineno) + + insert_after = last_import_line + if insert_after == 0 and tree.body: + first = tree.body[0] + if ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + ): + insert_after = first.end_lineno + + return "".join(lines[:insert_after] + export_stmts + lines[insert_after:]) + + +def _topo_depth(graph: Dict[str, Set[str]]) -> Dict[str, int]: + """Return topological depth for each node in a DAG. + + Depth 0 = leaf (no outgoing edges). A node's depth is 1 + the maximum + depth of its dependencies. All dependency nodes must be keys in *graph*. + On non-DAG inputs (cycles detected), returns 0 for every node as a safe + fallback so that callers degrade to arbitrary candidate ordering. + """ + if any(len(s) > 1 for s in find_sccs(graph)): + return {node: 0 for node in graph} + depths: Dict[str, int] = {} + + def dfs(node: str) -> int: + if node in depths: + return depths[node] + depths[node] = 1 + max((dfs(dep) for dep in graph[node]), default=-1) + return depths[node] + + for node in graph: + dfs(node) + return depths + + +def _extract_shared_helpers( + file_entity_names: Dict[str, List[str]], + entity_source_map: Dict[str, str], + entity_map: Dict[str, Entity], + classified: ClassifiedEntities, + name_to_target_file: Dict[str, str], + migrated_names: Set[str], + original_basename: str, +) -> List[GroupPlacement]: + """Extract non-migrated functions/classes referenced by migrated entities. + + When a migrated entity in new file F references a non-migrated function X + from the original O, the generated ``from .O import X`` combined with O's + re-export ``from .F import …`` creates a cycle O→F→O. + + Fix: pull X (and all helpers X transitively depends on) into a new file + that uses them. The destination is chosen using topological depth ordering: + the inter-file dependency graph is built from migrated-entity cross-references + first, then for each helper SCC the candidates (all files wanting the + helpers) are sorted by topological depth (deepest / most-downstream first). + For a DAG the deepest wanting file is always cycle-free on the first try; + for non-DAG inputs (pre-existing cycles) _topo_depth falls back to 0 for + all nodes and the loop exhausts all candidates via trial SCC analysis. + If no cycle-free placement exists the SCC is left in the original file and + the safety-net in :func:`generate_file_splits` will abort if the result is + unloadable. + + Mutates *file_entity_names*, *migrated_names*, and *name_to_target_file* + in place. Returns synthetic :class:`GroupPlacement` objects for the + extracted entities so that :func:`_add_re_exports` can re-import them from + their new location in the updated original source. + """ + # Build defined-name → entity-name map for non-migrated FUNCTION/CLASS entities. + defined_to_entity: Dict[str, str] = {} + for entity in classified.entities: + if entity.name in migrated_names: + continue + if entity.kind not in (EntityKind.FUNCTION, EntityKind.CLASS): + continue + for defined_name in entity.names_defined: + if name_to_target_file.get(defined_name) == original_basename: + defined_to_entity[defined_name] = entity.name + + # Collect directly-wanted helpers: entity_name → set of target_files that want it. + wanting: Dict[str, Set[str]] = {} + for target_file, ent_names in list(file_entity_names.items()): + for ent_name in ent_names: + src = entity_source_map.get(ent_name, "") + for ref_name in _collect_name_loads(src): + entity_name = defined_to_entity.get(ref_name) + if entity_name is not None: + wanting.setdefault(entity_name, set()).add(target_file) + + if not wanting: + return [] + + # Transitively expand wanting-sets to cover helpers referenced by + # already-wanted helpers, preventing O→new-file→O cycles. + # Re-queue a helper whenever its wanting-set gains new target files so that + # the propagation reaches all transitive dependents. + queue = list(wanting.keys()) + idx = 0 + while idx < len(queue): + entity_name = queue[idx] + idx += 1 + src = entity_source_map.get(entity_name, "") + for ref_name in _collect_name_loads(src): + dep_name = defined_to_entity.get(ref_name) + if dep_name and wanting[entity_name] - wanting.get(dep_name, set()): + wanting.setdefault(dep_name, set()).update(wanting[entity_name]) + queue.append(dep_name) + + # SCC analysis on the sub-graph of wanted helpers to co-locate + # mutually-dependent helpers. + sub_graph: Dict[str, Set[str]] = { + name: {d for d in classified.graph.get(name, set()) if d in wanting} + for name in wanting + } + sccs = find_sccs(sub_graph) + + # Build the initial inter-file dependency graph from migrated-entity + # cross-references (before any helper placement). This is the baseline for + # the cycle-aware candidate selection below. + file_deps: Dict[str, Set[str]] = {f: set() for f in file_entity_names} + for target_file, ent_names in file_entity_names.items(): + for ent_name in ent_names: + src = entity_source_map.get(ent_name, "") + for ref_name in _collect_name_loads(src): + dep_file = name_to_target_file.get(ref_name) + if ( + dep_file + and dep_file != target_file + and dep_file in file_entity_names + ): + file_deps[target_file].add(dep_file) + + synthetic_placements: List[GroupPlacement] = [] + for scc in sccs: + # Union of wanting-sets across this helper SCC. + scc_wanting: Set[str] = set() + for name in scc: + scc_wanting.update(wanting.get(name, set())) + + # Sort candidates by topological depth (deepest / most-downstream first). + # For a DAG the deepest wanting file is always cycle-free on the first try, + # eliminating trial-and-error. Depths are recomputed after each placement + # because file_deps grows as helpers are extracted. + topo_depth = _topo_depth(file_deps) + candidates = sorted(scc_wanting, key=lambda t: topo_depth.get(t, 0)) + chosen: Optional[str] = None + for candidate in candidates: + trial_deps: Dict[str, Set[str]] = { + f: set(deps) for f, deps in file_deps.items() + } + for wanting_file in scc_wanting: + if wanting_file != candidate: + trial_deps[wanting_file].add(candidate) + for helper_name in scc: + src = entity_source_map.get(helper_name, "") + for ref_name in _collect_name_loads(src): + dep_file = name_to_target_file.get(ref_name) + if ( + dep_file + and dep_file != candidate + and dep_file in file_entity_names + ): + trial_deps[candidate].add(dep_file) + if not any(len(s) > 1 for s in find_sccs(trial_deps)): + chosen = candidate + break + + if chosen is None: + continue # No cycle-free placement — leave helpers in original file. + + # Apply the chosen placement: update file_deps for subsequent SCC decisions. + for wanting_file in scc_wanting: + if wanting_file != chosen: + file_deps[wanting_file].add(chosen) + for helper_name in scc: + src = entity_source_map.get(helper_name, "") + for ref_name in _collect_name_loads(src): + dep_file = name_to_target_file.get(ref_name) + if dep_file and dep_file != chosen and dep_file in file_entity_names: + file_deps[chosen].add(dep_file) + + # Prepend extracted helpers so they appear before the functions that use them. + file_entity_names[chosen] = list(scc) + file_entity_names[chosen] + for entity_name in scc: + migrated_names.add(entity_name) + entity = entity_map[entity_name] + for defined_name in entity.names_defined: + name_to_target_file[defined_name] = chosen + synthetic_placements.append(GroupPlacement(group=list(scc), target_file=chosen)) + return synthetic_placements + + +def _prune_inline_redundant_imports(source: str) -> str: + """Remove function-body imports that duplicate module-level imports. + + When a function-local ``from x import y`` re-imports a name that is + already provided by a top-level import, flake8 reports an F811 + redefinition warning. This function removes such redundant inner imports + (or narrows them when only some names are redundant). + + Returns *source* unchanged when it cannot be parsed or nothing needs + pruning. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return source + + # Names already available from top-level (module-level) imports. + top_level_names: Set[str] = set() + top_level_node_ids: Set[int] = set() + for node in tree.body: + if isinstance(node, ast.Import): + top_level_node_ids.add(id(node)) + for alias in node.names: + top_level_names.add( + alias.asname if alias.asname else alias.name.split(".")[0] + ) + elif isinstance(node, ast.ImportFrom): + top_level_node_ids.add(id(node)) + for alias in node.names: + top_level_names.add(alias.asname if alias.asname else alias.name) + + if not top_level_names: + return source + + # Collect import node IDs inside module-level 'if TYPE_CHECKING:' blocks. + # These are intentional type-checking guards and must not be treated as + # redundant even when the same name is already imported at module level — + # removing them would leave an empty (and therefore invalid) if-block. + tc_guard_import_ids: Set[int] = set() + for node in tree.body: + if ( + isinstance(node, ast.If) + and isinstance(node.test, ast.Name) + and node.test.id == "TYPE_CHECKING" + ): + for child in ast.walk(node): + if isinstance(child, (ast.Import, ast.ImportFrom)): + tc_guard_import_ids.add(id(child)) + + # Find all import nodes that are NOT at module level and NOT inside a + # module-level 'if TYPE_CHECKING:' guard. + inner_imports = [ + node + for node in ast.walk(tree) + if isinstance(node, (ast.Import, ast.ImportFrom)) + and id(node) not in top_level_node_ids + and id(node) not in tc_guard_import_ids + ] + + if not inner_imports: + return source + + lines = source.splitlines(keepends=True) + # Maps 1-based line number → replacement line (None = remove that line). + line_ops: Dict[int, Optional[str]] = {} + + for stmt in inner_imports: + if isinstance(stmt, ast.Import): + kept = [ + a + for a in stmt.names + if (a.asname if a.asname else a.name.split(".")[0]) + not in top_level_names + ] + else: + kept = [ + a + for a in stmt.names + if (a.asname if a.asname else a.name) not in top_level_names + ] + + if len(kept) == len(stmt.names): + continue # no redundancy — nothing to remove + + # Mark every line of this import for removal. + for ln in range(stmt.lineno, stmt.end_lineno + 1): + line_ops[ln] = None + + if kept: + # Rebuild a narrowed import preserving original indentation. + alias_strs = [ + f"{a.name} as {a.asname}" if a.asname else a.name for a in kept + ] + orig_line = lines[stmt.lineno - 1] + indent = orig_line[: len(orig_line) - len(orig_line.lstrip())] + if isinstance(stmt, ast.ImportFrom): + dots = "." * (stmt.level or 0) + mod = stmt.module or "" + new_line = f"{indent}from {dots}{mod} import {', '.join(alias_strs)}\n" + else: + new_line = f"{indent}import {', '.join(alias_strs)}\n" + line_ops[stmt.lineno] = new_line + + if not line_ops: + return source + + result: List[str] = [] + for i, line in enumerate(lines, 1): + if i in line_ops: + repl = line_ops[i] + if repl is not None: + result.append(repl) + # else: None → line is removed + else: + result.append(line) + return "".join(result) + + +def _prune_unused_imports(source: str) -> str: + """Remove or narrow unused imports in a generated file. + + ``from __future__`` and star imports are always preserved. Multi-name + imports are narrowed to only the names actually referenced in *source* + rather than dropped wholesale. Fully-unused imports are removed entirely. + + Returns *source* unchanged when it cannot be parsed or nothing needs + pruning. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return source + + used = _collect_name_loads(source) + lines = source.splitlines(keepends=True) + # Maps 1-based line number → replacement line (None = remove that line). + replacements: Dict[int, Optional[str]] = {} + + for node in tree.body: + if not isinstance(node, (ast.Import, ast.ImportFrom)): + continue + + # Always preserve __future__ imports. + if isinstance(node, ast.ImportFrom) and node.module == "__future__": + continue + + # Always preserve star imports. + if isinstance(node, ast.ImportFrom) and any(a.name == "*" for a in node.names): + continue + + # Preserve intentional re-export stubs added by _add_re_exports. + # These carry "# noqa: F401" and must not be pruned even when the + # name is no longer referenced in the file body — they exist solely + # to keep the module's public/private API intact for external callers. + import_lines = lines[node.lineno - 1 : node.end_lineno] + if any("noqa: F401" in line for line in import_lines): + continue + + kept = [ + a + for a in node.names + if (a.asname if a.asname else a.name.split(".")[0]) in used + ] + + if len(kept) == len(node.names): + continue # nothing to prune + + # Mark every line of this import for removal. + for ln in range(node.lineno, node.end_lineno + 1): + replacements[ln] = None + + if not kept: + continue # fully unused — all lines already removed + + # Rebuild a single-line import with only the kept aliases. + alias_strs = [f"{a.name} as {a.asname}" if a.asname else a.name for a in kept] + if isinstance(node, ast.ImportFrom): + level_dots = "." * (node.level or 0) + module = node.module or "" + new_line = f"from {level_dots}{module} import {', '.join(alias_strs)}\n" + else: + new_line = f"import {', '.join(alias_strs)}\n" + replacements[node.lineno] = new_line + + if not replacements: + return source + + result: List[str] = [] + for i, line in enumerate(lines, 1): + if i not in replacements: + result.append(line) + elif replacements[i] is not None: + result.append(replacements[i]) + # else: line is removed — skip it + return "".join(result) + + +def _strip_top_level_import_lines(src: str) -> str: + """Return *src* with all top-level import statements removed. + + Also removes module-level ``if TYPE_CHECKING:`` blocks, since their + imports are now redistributed to each sub-file via the import-info + system and emitting the block verbatim would produce the wrong relative + import path and/or an unused import in the wrong sub-file. + + Uses AST to locate the exact line range of each import node, correctly + handling multi-line imports. Returns *src* unchanged when it cannot be + parsed as Python. + """ + try: + tree = ast.parse(src) + except SyntaxError: + return src + remove: Set[int] = set() + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + for ln in range(node.lineno, node.end_lineno + 1): + remove.add(ln) + elif ( + isinstance(node, ast.If) + and isinstance(node.test, ast.Name) + and node.test.id == "TYPE_CHECKING" + ): + for ln in range(node.lineno, node.end_lineno + 1): + remove.add(ln) + if not remove: + return src + lines = src.splitlines(keepends=True) + return "".join(line for i, line in enumerate(lines, 1) if i not in remove) diff --git a/crispen/refactors/duplicate_extractor.py b/crispen/refactors/duplicate_extractor.py deleted file mode 100644 index e94f1b3..0000000 --- a/crispen/refactors/duplicate_extractor.py +++ /dev/null @@ -1,3205 +0,0 @@ -"""Refactor: extract duplicate code blocks into helper functions using an LLM.""" - -from __future__ import annotations - -import ast -import re -import sys -import textwrap -import threading -from dataclasses import dataclass -from typing import Dict, List, Optional, Sequence, Tuple - -import libcst as cst -from libcst.metadata import MetadataWrapper, PositionProvider - -from .. import llm_client as _llm_client -from ..import_sort import _sort_imports_pep8 -from .base import Refactor - -_MODEL = "claude-sonnet-4-6" -_MIN_WEIGHT = 3 -_MAX_SEQ_LEN = 8 - - -# --------------------------------------------------------------------------- -# Docstring stripping -# --------------------------------------------------------------------------- - - -def _strip_helper_docstring(helper_source: str) -> str: - """Remove the docstring from helper_source if the first function has one.""" - try: - tree = cst.parse_module(textwrap.dedent(helper_source)) - except cst.ParserSyntaxError: - return helper_source - - if not tree.body or not isinstance(tree.body[0], cst.FunctionDef): - return helper_source - - func = tree.body[0] - body = func.body - if not isinstance(body, cst.IndentedBlock) or not body.body: # pragma: no cover - return helper_source - - first = body.body[0] - if not ( - isinstance(first, cst.SimpleStatementLine) - and len(first.body) == 1 - and isinstance(first.body[0], cst.Expr) - and isinstance(first.body[0].value, (cst.SimpleString, cst.ConcatenatedString)) - ): - return helper_source - - rest = list(body.body[1:]) - if not rest: - return helper_source - - new_func = func.with_changes(body=body.with_changes(body=rest)) - return tree.with_changes(body=[new_func] + list(tree.body[1:])).code - - -# --------------------------------------------------------------------------- -# Hard-timeout helper -# --------------------------------------------------------------------------- - - -class _ApiTimeout(Exception): - """Raised when an LLM API call exceeds the hard per-call timeout.""" - - -def _run_with_timeout(func, timeout, *args, **kwargs): - """Run *func* in a daemon thread; raise _ApiTimeout if it doesn't finish. - - This enforces a hard wall-clock limit that is not affected by OS-level - blocking (e.g. DNS resolution) which application-layer timeouts cannot - interrupt. - """ - result: list = [None] - exc: list = [None] - - def target(): - try: - result[0] = func(*args, **kwargs) - except BaseException as e: - exc[0] = e - - t = threading.Thread(target=target, daemon=True) - t.start() - t.join(timeout=timeout) - if t.is_alive(): - raise _ApiTimeout(f"API call exceeded {timeout}s hard limit") - if exc[0] is not None: - raise exc[0] - return result[0] - - -# --------------------------------------------------------------------------- -# Recursive statement weight -# --------------------------------------------------------------------------- - - -def _node_weight(node: cst.CSTNode) -> int: - """Recursive statement weight: count all semantic statement units.""" - if isinstance(node, cst.SimpleStatementLine): - return len(node.body) - if isinstance(node, cst.IndentedBlock): - return sum(_node_weight(s) for s in node.body) - if isinstance(node, cst.Else): - return _node_weight(node.body) - if isinstance(node, cst.Finally): - return _node_weight(node.body) - if isinstance(node, (cst.FunctionDef, cst.ClassDef)): - return 1 - if not isinstance(node, (cst.If, cst.For, cst.While, cst.Try, cst.With)): - return 0 - weight = 1 + _node_weight(node.body) - orelse = getattr(node, "orelse", None) - if orelse is not None: - weight += _node_weight(orelse) - finalbody = getattr(node, "finalbody", None) - if finalbody is not None: - weight += _node_weight(finalbody) - if isinstance(node, cst.Try): - for handler in node.handlers: - weight += _node_weight(handler.body) - return weight - - -def _sequence_weight(stmts: List[cst.BaseStatement]) -> int: - return sum(_node_weight(s) for s in stmts) - - -def _has_def(stmts: List[cst.BaseStatement]) -> bool: - """Return True if any top-level statement is a function or class definition.""" - return any(isinstance(s, (cst.FunctionDef, cst.ClassDef)) for s in stmts) - - -# --------------------------------------------------------------------------- -# Normalization -# --------------------------------------------------------------------------- - - -class _ASTNormalizer(ast.NodeTransformer): - """Replace assignment-target Names with positional placeholders.""" - - def __init__(self) -> None: - self._map: Dict[str, str] = {} - self._counter = 0 - - def _placeholder(self, name: str) -> str: - if name not in self._map: - self._map[name] = f"_v{self._counter}" - self._counter += 1 - return self._map[name] - - def visit_Name(self, node: ast.Name) -> ast.Name: - if isinstance(node.ctx, (ast.Store, ast.Load)): - return ast.Name(id=self._placeholder(node.id), ctx=node.ctx) - return node - - -def _normalize_source(source: str) -> str: - """Return a normalized fingerprint of source code.""" - try: - tree = ast.parse(textwrap.dedent(source)) - except SyntaxError: - return source - normalizer = _ASTNormalizer() - normalized = normalizer.visit(tree) - ast.fix_missing_locations(normalized) - return ast.unparse(normalized) - - -# --------------------------------------------------------------------------- -# Sequence info -# --------------------------------------------------------------------------- - - -@dataclass -class _SeqInfo: - stmts: List[cst.BaseStatement] - start_line: int - end_line: int - scope: str - source: str - fingerprint: str - class_scope: Optional[str] = None # enclosing class name, or None if module-level - - -@dataclass -class _FunctionInfo: - name: str - source: str # raw source of complete function definition - scope: str # "" or enclosing class name - body_source: str # raw source of the function body (indented) - body_stmt_count: int # number of top-level statements in the body - params: List[str] # positional parameter names (empty → no-arg function) - - -# --------------------------------------------------------------------------- -# Sequence collector -# --------------------------------------------------------------------------- - - -class _SequenceCollector(cst.CSTVisitor): - METADATA_DEPENDENCIES = (PositionProvider,) - - def __init__( - self, - source_lines: List[str], - max_seq_len: int = _MAX_SEQ_LEN, - min_weight: int = _MIN_WEIGHT, - ) -> None: - self.sequences: List[_SeqInfo] = [] - self._scope_stack: List[str] = [""] - self._class_stack: List[str] = [] - self._source_lines = source_lines - self._max_seq_len = max_seq_len - self._min_weight = min_weight - - def visit_FunctionDef(self, node: cst.FunctionDef) -> Optional[bool]: - self._scope_stack.append(node.name.value) - return None - - def leave_FunctionDef(self, node: cst.FunctionDef) -> None: - self._scope_stack.pop() - - def visit_ClassDef(self, node: cst.ClassDef) -> Optional[bool]: - self._scope_stack.append(node.name.value) - self._class_stack.append(node.name.value) - return None - - def leave_ClassDef(self, node: cst.ClassDef) -> None: - self._scope_stack.pop() - self._class_stack.pop() - - def _process_body(self, body: Sequence) -> None: - stmt_info: List[Tuple[cst.BaseStatement, int, int]] = [] - for stmt in body: - try: - pos = self.get_metadata(PositionProvider, stmt) - stmt_info.append((stmt, pos.start.line, pos.end.line)) - except KeyError: # pragma: no cover - continue - - n = len(stmt_info) - scope = self._scope_stack[-1] - class_scope = self._class_stack[-1] if self._class_stack else None - for start_i in range(n): - for end_i in range( - start_i + 1, min(start_i + self._max_seq_len + 1, n + 1) - ): - window: List[cst.BaseStatement] = [ - s[0] for s in stmt_info[start_i:end_i] - ] - if _has_def(window): - continue - if _sequence_weight(window) < self._min_weight: - continue - start_line = stmt_info[start_i][1] - end_line = stmt_info[end_i - 1][2] - seq_source = "".join(self._source_lines[start_line - 1 : end_line]) - if _seq_source_contains_yield(seq_source): - continue - self.sequences.append( - _SeqInfo( - stmts=window, - start_line=start_line, - end_line=end_line, - scope=scope, - source=seq_source, - fingerprint=_normalize_source(seq_source), - class_scope=class_scope, - ) - ) - - def visit_Module(self, node: cst.Module) -> Optional[bool]: - self._process_body(node.body) - return None - - def visit_IndentedBlock(self, node: cst.IndentedBlock) -> Optional[bool]: - self._process_body(node.body) - return None - - -# --------------------------------------------------------------------------- -# Function collector -# --------------------------------------------------------------------------- - - -class _FunctionCollector(cst.CSTVisitor): - METADATA_DEPENDENCIES = (PositionProvider,) - - def __init__(self, source_lines: List[str]) -> None: - self.functions: List[_FunctionInfo] = [] - self._scope_stack: List[str] = [""] - self._scope_kind_stack: List[str] = ["module"] - self._source_lines = source_lines - - def visit_FunctionDef(self, node: cst.FunctionDef) -> Optional[bool]: - parent_kind = self._scope_kind_stack[-1] - if parent_kind in ("module", "class"): - try: - pos = self.get_metadata(PositionProvider, node) - func_source = "".join( - self._source_lines[pos.start.line - 1 : pos.end.line] - ) - body_pos = self.get_metadata(PositionProvider, node.body) - body_source = "".join( - self._source_lines[body_pos.start.line - 1 : body_pos.end.line] - ) - except KeyError: # pragma: no cover - func_source = "" - body_source = "" - body_stmt_count = len(node.body.body) - params = [p.name.value for p in node.params.params] - self.functions.append( - _FunctionInfo( - name=node.name.value, - source=func_source, - scope=self._scope_stack[-1], - body_source=body_source, - body_stmt_count=body_stmt_count, - params=params, - ) - ) - self._scope_stack.append(node.name.value) - self._scope_kind_stack.append("function") - return None - - def leave_FunctionDef(self, node: cst.FunctionDef) -> None: - self._scope_stack.pop() - self._scope_kind_stack.pop() - - def visit_ClassDef(self, node: cst.ClassDef) -> Optional[bool]: - self._scope_stack.append(node.name.value) - self._scope_kind_stack.append("class") - return None - - def leave_ClassDef(self, node: cst.ClassDef) -> None: - self._scope_stack.pop() - self._scope_kind_stack.pop() - - -# --------------------------------------------------------------------------- -# Function body fingerprint helpers -# --------------------------------------------------------------------------- - - -def _collect_called_names(source: str) -> set: - """Return a set of all names called (as functions) in *source*. - - Uses ast.parse + ast.walk to find all ast.Call nodes. Returns the - called name: func.id for ast.Name callees, func.attr for ast.Attribute - callees. On SyntaxError, returns an empty set. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - names: set = set() - for node in ast.walk(tree): - if isinstance(node, ast.Call): - if isinstance(node.func, ast.Name): - names.add(node.func.id) - elif isinstance(node.func, ast.Attribute): - names.add(node.func.attr) - return names - - -def _build_function_body_fps( - all_functions: List[_FunctionInfo], - called_names: set, -) -> Dict[str, _FunctionInfo]: - """Map normalized body fingerprint → _FunctionInfo for called functions. - - Only functions whose name appears in *called_names* are indexed, since - only those could be the target of a "replace with existing function" edit. - """ - fps: Dict[str, _FunctionInfo] = {} - for func in all_functions: - if func.name in called_names: - fp = _normalize_source(func.body_source) - fps[fp] = func - return fps - - -# --------------------------------------------------------------------------- -# Duplicate group finding -# --------------------------------------------------------------------------- - - -def _overlaps_diff(seq: _SeqInfo, changed_ranges: List[Tuple[int, int]]) -> bool: - return any( - seq.start_line <= r_end and seq.end_line >= r_start - for r_start, r_end in changed_ranges - ) - - -def _filter_maximal_groups(groups: List[List[_SeqInfo]]) -> List[List[_SeqInfo]]: - """Return only maximal groups, discarding those overlapping a larger group. - - Groups are sorted by their longest sequence (descending) and greedily selected: - a group is kept only if none of its sequences overlap an already-claimed line range. - This prevents multiple helpers being extracted for overlapping spans, where the - smaller extractions would end up unused after the larger one is applied. - """ - sorted_groups = sorted( - groups, - key=lambda g: max(s.end_line - s.start_line for s in g), - reverse=True, - ) - claimed: List[Tuple[int, int]] = [] - result = [] - for group in sorted_groups: - overlaps = any( - seq.start_line <= c_end and seq.end_line >= c_start - for seq in group - for c_start, c_end in claimed - ) - if not overlaps: - result.append(group) - for seq in group: - claimed.append((seq.start_line, seq.end_line)) - return result - - -def _has_internal_overlap(seqs: List[_SeqInfo]) -> bool: - """Return True if any two sequences in the group overlap each other. - - Overlapping sequences within a group indicate sequential repetition - (e.g. [A,B] and [B,C] both matching) rather than true duplication at - distinct call sites. Extracting a helper from such a group would leave - part of the original pattern unreplaced. - """ - sorted_seqs = sorted(seqs, key=lambda s: s.start_line) - for i in range(len(sorted_seqs) - 1): - if sorted_seqs[i].end_line >= sorted_seqs[i + 1].start_line: - return True - return False - - -def _find_duplicate_groups( - sequences: List[_SeqInfo], - changed_ranges: List[Tuple[int, int]], - max_groups: int = 5, -) -> List[List[_SeqInfo]]: - by_fp: Dict[str, List[_SeqInfo]] = {} - for seq in sequences: - by_fp.setdefault(seq.fingerprint, []).append(seq) - groups = [] - for seqs in by_fp.values(): - if len(seqs) < 2: - continue - if not any(_overlaps_diff(s, changed_ranges) for s in seqs): - continue - if _has_internal_overlap(seqs): - continue - groups.append(seqs) - groups = _filter_maximal_groups(groups) - return groups[:max_groups] - - -# --------------------------------------------------------------------------- -# LLM integration -# --------------------------------------------------------------------------- - -_VETO_TOOL: dict = { - "name": "evaluate_duplicate", - "description": ( - "Evaluate whether code blocks are semantic duplicates worth extracting" - ), - "input_schema": { - "type": "object", - "properties": { - "is_valid_duplicate": { - "type": "boolean", - "description": ( - "True if extracting a shared helper would improve clarity" - ), - }, - "reason": {"type": "string"}, - "extraction_notes": { - "type": "string", - "description": ( - "If accepting, note any potential pitfalls the extraction " - "step should watch out for — e.g., tricky variable scoping, " - "mutable arguments, subtle differences in variable names, or " - "return-value handling. Leave empty if none." - ), - }, - }, - "required": ["is_valid_duplicate", "reason"], - }, -} - -_VERIFY_TOOL: dict = { - "name": "verify_extraction", - "description": "Verify that an extracted helper function is semantically correct", - "input_schema": { - "type": "object", - "properties": { - "is_correct": { - "type": "boolean", - "description": ( - "True if the extraction is semantically equivalent to the originals" - ), - }, - "issues": { - "type": "array", - "items": {"type": "string"}, - "description": ( - "Specific issues found. Empty if correct. Each issue should " - "describe what is wrong and how the extraction should be fixed." - ), - }, - }, - "required": ["is_correct", "issues"], - }, -} - -_EXTRACT_TOOL: dict = { - "name": "extract_helper", - "description": "Extract duplicate code blocks into a helper function", - "input_schema": { - "type": "object", - "properties": { - "function_name": {"type": "string"}, - "placement": { - "type": "string", - "description": ( - "Where to place the helper: 'module_level' or " - "'staticmethod:ClassName'" - ), - }, - "helper_source": { - "type": "string", - "description": "Complete source of the helper function", - }, - "call_site_replacements": { - "type": "array", - "items": {"type": "string"}, - "description": ( - "Replacement source for each duplicate block, " - "in the same order as the input blocks. " - "Each replacement must preserve the original block's " - "leading indentation and end with a trailing newline. " - "Cover only the exact lines of the specified block — " - "do not include any code from before or after the block." - ), - }, - }, - "required": [ - "function_name", - "placement", - "helper_source", - "call_site_replacements", - ], - }, -} - -_CALL_GEN_TOOL: dict = { - "name": "generate_call", - "description": "Generate a call to an existing function that replaces a code block", - "input_schema": { - "type": "object", - "properties": { - "replacement": { - "type": "string", - "description": ( - "Complete replacement source " - "(including indentation and trailing newline)" - ), - } - }, - "required": ["replacement"], - }, -} - - -def _llm_veto( - client, - group: List[_SeqInfo], - model: str = _MODEL, - provider: str = "anthropic", - tool_choice_override: Optional[str] = None, - _timing_out=None, - rate_limit_retries: int = 6, - rate_limit_backoff: float = 20.0, -) -> Tuple[bool, str, str]: - blocks_text = "\n\n".join( - f"Block {i + 1} (scope: {s.scope}, lines {s.start_line}-{s.end_line}):\n" - f"```python\n{s.source.rstrip()}\n```" - for i, s in enumerate(group) - ) - prompt = ( - f"Here are {len(group)} structurally similar code blocks from the same " - f"Python file:\n\n{blocks_text}\n\n" - "Do these blocks represent the same semantic operation such that extracting " - "a shared helper function would improve clarity? Or are they coincidentally " - "similar but conceptually distinct?\n\n" - "If you accept (is_valid_duplicate=True), also fill in extraction_notes " - "with any potential pitfalls the extraction step should watch out for — " - "e.g., tricky variable scoping, mutable arguments, subtle differences in " - "variable names between blocks, or return-value handling edge cases." - ) - result = _llm_client.call_with_tool( - client, - provider, - model, - 384, - _VETO_TOOL, - "evaluate_duplicate", - [{"role": "user", "content": prompt}], - caller="DuplicateExtractor", - tool_choice_override=tool_choice_override, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff=rate_limit_backoff, - ) - if _timing_out is not None: - _timing_out.append(result) - if result.tool_input is not None: - return ( - result.tool_input["is_valid_duplicate"], - result.tool_input.get("reason", ""), - result.tool_input.get("extraction_notes", ""), - ) - return False, "no tool response", "" # pragma: no cover - - -def _llm_extract( - client, - group: List[_SeqInfo], - full_source: str, - escaping_vars: frozenset = frozenset(), - used_names: frozenset = frozenset(), - model: str = _MODEL, - helper_docstrings: bool = True, - provider: str = "anthropic", - veto_notes: str = "", - prev_failures: List[str] = [], - prev_output: Optional[dict] = None, - tool_choice_override: Optional[str] = None, - _timing_out=None, - rate_limit_retries: int = 6, - rate_limit_backoff: float = 20.0, -) -> Optional[dict]: - src_lines = full_source.splitlines(keepends=True) - block_entries = [] - for i, s in enumerate(group): - entry = ( - f"Block {i + 1} (scope: {s.scope}, lines {s.start_line}-{s.end_line}):\n" - f"```python\n{s.source.rstrip()}\n```" - ) - next_idx = s.end_line # 0-based index of the first line after the block - if next_idx < len(src_lines): - next_line = src_lines[next_idx].rstrip() - if next_line.strip(): - entry += ( - f"\nLine immediately after this block" - f" (must NOT appear in the replacement): `{next_line}`" - ) - block_entries.append(entry) - blocks_text = "\n\n".join(block_entries) - snippet = full_source[:4000] if len(full_source) > 4000 else full_source - escaping_note = "" - if escaping_vars: - vars_str = ", ".join(sorted(escaping_vars)) - escaping_note = ( - f"\n\nThe following variables are assigned within the duplicate block " - f"and referenced by code that immediately follows the block at one or " - f"more call sites: {vars_str}. The helper function must return these " - f"variables. At call sites where the return value is needed, capture it; " - f"at call sites where it is not needed, discard the return value." - ) - used_names_note = "" - if used_names: - names_str = ", ".join(sorted(used_names)) - used_names_note = ( - f"\n\nThe following function names are already defined in this file " - f"or reserved by a previous extraction: {names_str}. " - f"Do not use any of these names for the helper function." - ) - docstring_note = ( - "" - if helper_docstrings - else "\n\nDo not include a docstring in the helper function." - ) - veto_notes_note = "" - if veto_notes: - veto_notes_note = ( - f"\n\nNotes from code review (watch out for these pitfalls): " - f"{veto_notes[:500]}" - ) - failures_note = "" - if prev_failures: - failures_str = "\n".join(f"- {f}" for f in prev_failures) - prior_helper = prev_output.get("helper_source", "") - prior_repls = prev_output.get("call_site_replacements", []) - repls_text = "\n".join(f" [{i + 1}] {r!r}" for i, r in enumerate(prior_repls)) - failures_note = ( - f"\n\nThe previous extraction attempt produced:\n\n" - f"helper_source:\n```python\n{prior_helper}```\n\n" - f"call_site_replacements:\n{repls_text}\n\n" - f"But failed these checks:\n{failures_str}\n\n" - f"Please correct these issues in your new attempt." - ) - class_scopes = {s.class_scope for s in group} - all_same_class = len(class_scopes) == 1 and None not in class_scopes - if all_same_class: - same_class_name = next(iter(class_scopes)) - staticmethod_instruction = ( - f"All call sites are inside class '{same_class_name}'. " - f"You MUST use placement 'staticmethod:{same_class_name}'. " - ) - else: - staticmethod_instruction = ( - "Use module_level placement — call sites span different classes or scopes. " - ) - prompt = ( - "Extract the following duplicate code blocks from this Python file into a " - f"helper function.\n\nFile source:\n```python\n{snippet}\n```\n\n" - f"Duplicate blocks:\n{blocks_text}\n\n" - "Place the helper immediately before the enclosing function of its first use. " - f"{staticmethod_instruction}" - "Return complete, valid Python for the helper and each call site replacement. " - "Each call site replacement must start with the same leading indentation as " - "the block it replaces, end with a trailing newline, and cover only the exact " - "lines of the duplicate block — stopping before the 'Line immediately after " - "this block' marker shown above. Do not include any code from before or after " - "the block. " - "Double-check that only required parameters are passed to the helper — do not " - "include an unused parameter, or one that is overwritten before being read. " - "Be mindful of the code being removed from the call site: if variable " - "assignments are moved into the helper, those variables may no longer be " - "defined in the calling scope at that point. " - "If the helper uses a sentinel return value to signal an error path (such as " - "returning an empty collection), check for it at the call site with `==`, not " - "`is` — `is` only gives correct results for singletons like `None`, `True`, " - "and `False`, not for constructed objects like `set()`." - f"{escaping_note}" - f"{used_names_note}" - f"{docstring_note}" - f"{veto_notes_note}" - f"{failures_note}" - ) - result = _llm_client.call_with_tool( - client, - provider, - model, - 1024, - _EXTRACT_TOOL, - "extract_helper", - [{"role": "user", "content": prompt}], - caller="DuplicateExtractor", - tool_choice_override=tool_choice_override, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff=rate_limit_backoff, - ) - if _timing_out is not None: - _timing_out.append(result) - return result.tool_input - - -def _llm_veto_func_match( - client, - seq: _SeqInfo, - func: _FunctionInfo, - full_source: str, - model: str = _MODEL, - provider: str = "anthropic", - tool_choice_override: Optional[str] = None, - _timing_out=None, - rate_limit_retries: int = 6, - rate_limit_backoff: float = 20.0, -) -> Tuple[bool, str, str]: - """Ask the LLM whether *seq* performs the same operation as *func*'s body.""" - snippet = full_source[:4000] if len(full_source) > 4000 else full_source - prompt = ( - "A code block in a Python file may be replaceable by a call to an existing " - "function.\n\n" - f"Code block (scope: {seq.scope}, lines {seq.start_line}-{seq.end_line}):\n" - f"```python\n{seq.source.rstrip()}\n```\n\n" - f"Existing function '{func.name}':\n" - f"```python\n{func.source.rstrip()}\n```\n\n" - f"File source:\n```python\n{snippet}\n```\n\n" - "Does this code block perform the same semantic operation as the function " - "body, such that it could be replaced by a call to the function? " - "Use the evaluate_duplicate tool to answer." - ) - result = _llm_client.call_with_tool( - client, - provider, - model, - 256, - _VETO_TOOL, - "evaluate_duplicate", - [{"role": "user", "content": prompt}], - caller="DuplicateExtractor", - tool_choice_override=tool_choice_override, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff=rate_limit_backoff, - ) - if _timing_out is not None: - _timing_out.append(result) - if result.tool_input is not None: - return ( - result.tool_input["is_valid_duplicate"], - result.tool_input.get("reason", ""), - result.tool_input.get("extraction_notes", ""), - ) - return False, "no tool response", "" # pragma: no cover - - -def _generate_no_arg_call(seq: _SeqInfo, func: _FunctionInfo) -> str: - """Algorithmically generate a no-argument call to *func*, preserving indentation.""" - first_line = seq.source.splitlines()[0] - indent = first_line[: len(first_line) - len(first_line.lstrip())] - return indent + func.name + "()\n" - - -def _llm_generate_call( - client, - seq: _SeqInfo, - func: _FunctionInfo, - full_source: str, - model: str = _MODEL, - provider: str = "anthropic", - tool_choice_override: Optional[str] = None, - _timing_out=None, - rate_limit_retries: int = 6, - rate_limit_backoff: float = 20.0, -) -> Optional[str]: - """Ask the LLM to generate a call expression replacing *seq* with *func*.""" - snippet = full_source[:4000] if len(full_source) > 4000 else full_source - prompt = ( - f"Replace this code block with a call to the existing function" - f" '{func.name}'.\n\n" - f"Code block (scope: {seq.scope}, lines {seq.start_line}-{seq.end_line}):\n" - f"```python\n{seq.source.rstrip()}\n```\n\n" - f"Function '{func.name}':\n" - f"```python\n{func.source.rstrip()}\n```\n\n" - f"File source:\n```python\n{snippet}\n```\n\n" - "Generate a replacement that preserves the original indentation and ends " - "with a newline. Pass the replacement to the generate_call tool." - ) - result = _llm_client.call_with_tool( - client, - provider, - model, - 256, - _CALL_GEN_TOOL, - "generate_call", - [{"role": "user", "content": prompt}], - caller="DuplicateExtractor", - tool_choice_override=tool_choice_override, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff=rate_limit_backoff, - ) - if _timing_out is not None: - _timing_out.append(result) - if result.tool_input is not None: - return result.tool_input["replacement"] - return None # pragma: no cover - - -def _llm_verify_extraction( - client, - group: List[_SeqInfo], - helper_source: str, - call_replacements: List[str], - full_source: str, - model: str = _MODEL, - provider: str = "anthropic", - tool_choice_override: Optional[str] = None, - _timing_out=None, - rate_limit_retries: int = 6, - rate_limit_backoff: float = 20.0, -) -> Tuple[bool, List[str]]: - """Ask the LLM to verify the extraction is semantically correct. - - Returns ``(is_correct, issues)`` where *issues* is a list of specific - problems found. Returns ``(True, [])`` if the call times out or the LLM - cannot respond, so a verification failure never silently blocks commits. - """ - blocks_text = "\n\n".join( - f"Original block {i + 1} (scope: {s.scope}, " - f"lines {s.start_line}-{s.end_line}):\n" - f"```python\n{s.source.rstrip()}\n```" - for i, s in enumerate(group) - ) - replacements_text = "\n\n".join( - f"Replacement for block {i + 1}:\n```python\n{r.rstrip()}\n```" - for i, r in enumerate(call_replacements) - ) - src_lines = full_source.splitlines(keepends=True) - min_start = min(s.start_line for s in group) - max_end = max(s.end_line for s in group) - window_start = max(0, min_start - 30) - window_end = min(len(src_lines), max_end + 100) - snippet = "".join(src_lines[window_start:window_end]) - prompt = ( - "Verify that the following helper function extraction is semantically " - "correct by tracing through the code carefully.\n\n" - f"Original duplicate blocks:\n{blocks_text}\n\n" - f"Extracted helper:\n```python\n{helper_source.rstrip()}\n```\n\n" - f"Call site replacements:\n{replacements_text}\n\n" - f"Source context around duplicate blocks " - f"(lines {window_start + 1}–{window_end}):\n```python\n{snippet}\n```\n\n" - "Check each of the following:\n" - "1. Every variable read (but not locally assigned) in the original block " - "is passed as a parameter to the helper\n" - "2. Every variable assigned in the original block and used afterward is " - "returned by the helper and captured at the call site\n" - "3. No parameter is assigned before it is first read in the helper body\n" - "4. If the original block ends with a non-None return, the call site " - "replacement also propagates that return value\n" - "5. The call site replacements match the original indentation and cover " - "exactly the lines of the original block\n" - "6. If the helper is called more than once with different arguments, verify " - "each call site against the exact local variables that appeared in the " - "original code at that location — not merely variables of the same type. " - "Same-type variables (e.g. two dicts, two strings) that are both in scope " - "are a swap risk: confirm neither was substituted for the other across call " - "sites.\n" - "7. No line from the helper body is duplicated verbatim in the call site " - "replacement. If setup lines were extracted into the helper, they must not " - "also appear before or after the call — otherwise the extraction is wrong.\n" - "8. Does the function name clearly and accurately describe what the body " - "does? Flag the name if it is misleading, too generic, or omits a crucial " - "detail — for example, an important side-effect that the name gives no hint " - "of (e.g. a function named 'compute_total' that also writes to a database).\n" - "If correct, set is_correct=True and issues=[]. " - "Otherwise set is_correct=False and list each specific issue." - ) - result = _llm_client.call_with_tool( - client, - provider, - model, - 512, - _VERIFY_TOOL, - "verify_extraction", - [{"role": "user", "content": prompt}], - caller="DuplicateExtractor", - tool_choice_override=tool_choice_override, - rate_limit_retries=rate_limit_retries, - rate_limit_backoff=rate_limit_backoff, - ) - if _timing_out is not None: - _timing_out.append(result) - if result.tool_input is None: - return True, [] # pragma: no cover - return result.tool_input["is_correct"], result.tool_input.get("issues", []) - - -# --------------------------------------------------------------------------- -# Verification -# --------------------------------------------------------------------------- - - -def _normalize_replacement_indentation(seq: _SeqInfo, replacement: str) -> str: - """Re-indent *replacement* to match the original block's leading whitespace. - - The LLM sometimes returns replacements at column 0. This function - re-indents them to match the indentation of the corresponding original - block, so the assembled edit remains valid Python. - """ - orig_lines = [ln for ln in seq.source.splitlines() if ln.strip()] - if not orig_lines: - return replacement - first = orig_lines[0] - expected_indent = first[: len(first) - len(first.lstrip())] - dedented = textwrap.dedent(replacement) - if not expected_indent: - return dedented - return textwrap.indent(dedented, expected_indent) - - -def _collect_ast_store_names(node: ast.AST, names: List[str]) -> None: - """Recursively collect Name ids from an assignment target (Store context).""" - if isinstance(node, ast.Name): - names.append(node.id) - elif isinstance(node, (ast.Tuple, ast.List)): - for elt in node.elts: - _collect_ast_store_names(elt, names) - - -def _replace_unused_in_target( - target: ast.AST, following_src: str -) -> Tuple[ast.AST, bool, bool]: - """Replace unused Name nodes in *target* with ``_``. - - Returns ``(new_target, all_replaced, any_replaced)`` where: - - *all_replaced*: every name in the target was replaced (all unused). - - *any_replaced*: at least one name was replaced. - - Non-Name, non-Tuple/List targets (Attribute, Subscript, …) are treated as - *used* so we never accidentally strip an assignment we cannot analyse. - """ - if isinstance(target, ast.Name): - if re.search(r"\b" + re.escape(target.id) + r"\b", following_src): - return target, False, False # used → keep - return ast.Name(id="_", ctx=ast.Store()), True, True # unused → _ - if isinstance(target, (ast.Tuple, ast.List)): - new_elts: List[ast.AST] = [] - all_replaced = True - any_replaced = False - for elt in target.elts: - new_elt, elt_all, elt_any = _replace_unused_in_target(elt, following_src) - new_elts.append(new_elt) - if not elt_all: - all_replaced = False - if elt_any: - any_replaced = True - new_target = type(target)(elts=new_elts, ctx=ast.Store()) - return new_target, all_replaced, any_replaced - # Attribute, Subscript, Starred, etc. — treat as used. - return target, False, False - - -def _scope_end_line(source_lines: List[str], scope: str, after_line: int) -> int: - """Return the exclusive slice index into *source_lines* for the end of *scope*. - - ``after_line`` is the 1-based line number of the last line of the replaced - block. The returned index is suitable for ``source_lines[after_line:idx]`` - to get only the lines inside the enclosing scope that follow the block. - - For ``""`` scope the whole rest of the file is in scope, so - ``len(source_lines)`` is returned. For named function/class scopes the - innermost definition whose name matches *scope* and that contains - *after_line* is located via the AST; its end line is returned as the - exclusive slice bound (1-based end_lineno used directly as a 0-based - exclusive index is correct because line N is at index N-1, so slicing up - to index N includes line N). Falls back to ``len(source_lines)`` on any - parse error or if no matching scope is found. - """ - if scope == "": - return len(source_lines) - - source = "".join(source_lines) - try: - tree = ast.parse(source) - except SyntaxError: - return len(source_lines) - - # ast.walk is BFS, so outer scopes are visited before inner ones. Always - # overwriting best_end means the last match wins — which is the innermost - # (smallest) scope that still contains after_line. - best_end: int = len(source_lines) - for node in ast.walk(tree): - if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - continue - if node.name != scope: - continue - if not (node.lineno <= after_line <= node.end_lineno): - continue - best_end = node.end_lineno - - return best_end - - -def _strip_unused_call_assignments(replacement: str, following_lines: List[str]) -> str: - """Clean up unused assignment targets in a call-site replacement. - - For each ``Assign`` node whose right-hand side is a ``Call``: - - * **Single target** — unused ``Name`` elements in the target are replaced - with ``_``. If every element is unused the whole assignment is dropped - and only the call expression is emitted. Example:: - - result = _helper(x) → _helper(x) - a, b = _helper(x) (b used) → a, _ = _helper(x) - - * **Chained assignment** (``a = b = call()``) — stripped to just the call - only when every name across every target is unused; otherwise left alone. - - Augmented (``+=``) and annotated assignments are never touched. Assignment - targets that are not plain names or tuples/lists (e.g. ``self.x``) are - treated as *used* so we never accidentally remove live assignments. - - This prevents flake8 F841 "local variable assigned but never used" errors - introduced by the extraction. - """ - following_src = "".join(following_lines) - try: - dedented = textwrap.dedent(replacement) - tree = ast.parse(dedented) - except SyntaxError: - return replacement - - # Build a list of (start_ln, end_ln, new_src) edits. new_src is the - # replacement text for that statement (without leading indentation). - edits: List[Tuple[int, int, str]] = [] - for node in ast.walk(tree): - if not isinstance(node, ast.Assign): - continue - value_node = node.value - if isinstance(value_node, ast.Await) and isinstance(value_node.value, ast.Call): - pass # treat `result = await helper(...)` like `result = helper(...)` - elif not isinstance(value_node, ast.Call): - continue - - call_src = ast.unparse(value_node) - - if len(node.targets) == 1: - new_target, all_replaced, any_replaced = _replace_unused_in_target( - node.targets[0], following_src - ) - if all_replaced: - edits.append((node.lineno, node.end_lineno, call_src)) - elif any_replaced: - edits.append( - ( - node.lineno, - node.end_lineno, - ast.unparse(new_target) + " = " + call_src, - ) - ) - else: - # Chained assignment: strip only when every name is unused. - all_names: List[str] = [] - for t in node.targets: - _collect_ast_store_names(t, all_names) - if not all_names: - continue - if not any( - re.search(r"\b" + re.escape(n) + r"\b", following_src) - for n in all_names - ): - edits.append((node.lineno, node.end_lineno, call_src)) - - if not edits: - return replacement - - # Determine the leading indentation from the first non-empty line. - first_content = next((ln for ln in replacement.splitlines() if ln.strip()), "") - indent = first_content[: len(first_content) - len(first_content.lstrip())] - - # Apply edits in reverse line order so earlier indices stay valid. - dedented_lines = dedented.splitlines(keepends=True) - for start_ln, end_ln, new_src in sorted(edits, key=lambda x: x[0], reverse=True): - dedented_lines[start_ln - 1 : end_ln] = [new_src + "\n"] - - return textwrap.indent("".join(dedented_lines), indent) - - -_MUTABLE_CONSTRUCTORS = frozenset({"set", "list", "dict", "frozenset", "bytearray"}) - - -def _has_mutable_literal_is_check(source: str) -> bool: - """Return True if *source* contains identity checks against mutable literals. - - Patterns like ``x is set()``, ``x is []``, or ``x is {}`` are always - False in Python because each literal creates a new object at runtime. - Such patterns are a common LLM mistake when using a ``set()`` sentinel. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return False - for node in ast.walk(tree): - if not isinstance(node, ast.Compare): - continue - for op, comp in zip(node.ops, node.comparators): - if not isinstance(op, (ast.Is, ast.IsNot)): - continue - if isinstance(comp, (ast.List, ast.Set, ast.Dict, ast.Tuple)): - return True - if ( - isinstance(comp, ast.Call) - and isinstance(comp.func, ast.Name) - and comp.func.id in _MUTABLE_CONSTRUCTORS - ): - return True - return False - - -def _collect_attribute_names(source: str) -> set: - """Return all attribute names (dot-access names) anywhere in *source*.""" - try: - tree = ast.parse(source) - except SyntaxError: - return set() - return {node.attr for node in ast.walk(tree) if isinstance(node, ast.Attribute)} - - -def _collect_called_attr_names(source: str) -> set: - """Return attribute names used as method calls in *source*. - - Unlike :func:`_collect_attribute_names`, this only returns names that - appear as the attribute of a call expression (i.e. ``obj.method(...)``). - Plain attribute reads and type annotations like ``ast.AST`` are ignored, - so the new-method-call check does not produce false positives for - standard-library type references. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return set() - return { - node.func.attr - for node in ast.walk(tree) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) - } - - -def _has_funcdef(func_name: str, source: str) -> bool: - """Return True if func_name is defined anywhere in source.""" - try: - tree = ast.parse(source) - except SyntaxError: - return False - for node in ast.walk(tree): - if ( - isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - and node.name == func_name - ): - return True - return False - - -def _has_call_to(func_name: str, source: str) -> bool: - """Return True if func_name is called anywhere in source. - - Checks both direct calls (``func_name(...)``) and attribute calls - (``obj.func_name(...)``), covering both module-level helpers and - staticmethod calls. Returns False if source cannot be parsed. - """ - try: - tree = ast.parse(source) - except SyntaxError: - return False - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - if isinstance(node.func, ast.Name) and node.func.id == func_name: - return True - if isinstance(node.func, ast.Attribute) and node.func.attr == func_name: - return True - return False - - -def _verify_extraction( - helper_source: Optional[str], call_replacements: List[str] -) -> bool: - """Verify the extraction produces syntactically valid Python. - - Replacements are dedented and then wrapped in a dummy function before - compilation so that ``return`` / ``yield`` statements — which are legal - inside a function body — do not cause false SyntaxError rejections. - Pass helper_source=None to skip the helper compilation check (used when - replacing with an existing function rather than a newly extracted one). - """ - if helper_source is not None: - dedented_helper = textwrap.dedent(helper_source) - try: - compile(dedented_helper, "", "exec") - except SyntaxError: - return False - if _has_param_overwritten_before_read(helper_source): - return False - # Dedent before checking: helper may be indented (e.g. staticmethod). - # compile() already confirmed it's valid Python, so ast.parse will succeed. - if _has_mutable_literal_is_check(dedented_helper): - return False - for replacement in call_replacements: - dedented = textwrap.dedent(replacement) - # Wrap in a dummy function that contains a for loop so that - # ``return`` / ``yield`` (valid inside a function body) AND - # ``continue`` / ``break`` (valid inside a loop body) do not cause - # false SyntaxError rejections. Replacements are always placed back - # inside the caller's original context, which may include a loop. - wrapped = "def _check():\n for _ in []:\n" + textwrap.indent( - dedented, " " - ) - try: - compile(wrapped, "", "exec") - except SyntaxError: - # Retry with async wrapper for replacements that contain `await` - async_wrapped = "async def _check():\n for _ in []:\n" + textwrap.indent( - dedented, " " - ) - try: - compile(async_wrapped, "", "exec") - except SyntaxError: - return False - wrapped = async_wrapped - # Check the wrapped form so that indented/return-containing replacements - # parse successfully and give a definitive True/False answer. - if _has_mutable_literal_is_check(wrapped): - return False - return True - - -def _has_param_overwritten_before_read(helper_source: str) -> bool: - """Return True if any parameter is assigned before it is first read. - - This detects a common LLM mistake where a parameter is included in the - function signature but then immediately overwritten on the first line, - making the parameter useless and causing UnboundLocalError at call sites - that try to pass a value that was not yet assigned. - """ - try: - tree = ast.parse(textwrap.dedent(helper_source)) - except SyntaxError: # pragma: no cover - return False # pragma: no cover - for node in ast.walk(tree): - if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - continue - params = {arg.arg for arg in node.args.args} - params |= {arg.arg for arg in node.args.posonlyargs} - params |= {arg.arg for arg in node.args.kwonlyargs} - if node.args.vararg: - params.add(node.args.vararg.arg) - if node.args.kwarg: - params.add(node.args.kwarg.arg) - for stmt in node.body: - for n in ast.walk(stmt): - if isinstance(n, ast.Name) and n.id in params: - if isinstance(n.ctx, ast.Store): - return True - params.discard(n.id) # first use is a read — param is legitimate - return False - - -def _pyflakes_new_undefined_names(original: str, candidate: str) -> set: - """Return undefined names (F821) introduced by the edit. - - Compares pyflakes output before and after the edit and returns only names - that are newly undefined in the candidate — not ones already present in the - original source. This avoids false positives from pre-existing bare function - calls or module-level references that are valid in context but not resolvable - from a standalone snippet. - """ - import pyflakes.api - import pyflakes.messages - - class _Collector: - def __init__(self): - self.names: set = set() - - def unexpectedError(self, filename, msg): # pragma: no cover - pass - - def syntaxError(self, filename, msg, lineno, offset, text): # pragma: no cover - pass - - def flake(self, msg): - if isinstance(msg, pyflakes.messages.UndefinedName): - self.names.add(msg.message_args[0]) - - before = _Collector() - pyflakes.api.check(original, "", reporter=before) - after = _Collector() - pyflakes.api.check(candidate, "", reporter=after) - return after.names - before.names - - -def _is_pure_literal(node: ast.expr) -> bool: - """Return True if *node* is a side-effect-free literal expression. - - Covers ``ast.Constant`` (numbers, strings, bytes, True/False/None) and - recursively-pure container literals (list, tuple, set, dict). Anything - involving a function call or attribute access returns False. - """ - if isinstance(node, ast.Constant): - return True - if isinstance(node, (ast.List, ast.Tuple, ast.Set)): - return all(_is_pure_literal(e) for e in node.elts) - if isinstance(node, ast.Dict): - return all( - (k is None or _is_pure_literal(k)) and _is_pure_literal(v) - for k, v in zip(node.keys, node.values) - ) - return False - - -def _names_in_edit_texts(extraction_groups) -> set: - """Return all bare ``Name`` ids found in every edit text of *extraction_groups*. - - ``extraction_groups`` is the list of ``(func_name, group_edits, msg)`` - tuples accepted at the end of ``DuplicateExtractor._transform``. Each - ``group_edits`` entry is a ``(start, end, text)`` triple; *text* may be - the helper function source or a call-site replacement. Collecting names - from all of them gives the set of variables that the extraction actually - touched. - """ - names: set = set() - for _, g_edits, _ in extraction_groups: - for _start, _end, text in g_edits: - try: - tree = ast.parse(text) - except SyntaxError: - continue - for node in ast.walk(tree): - if isinstance(node, ast.Name): - names.add(node.id) - return names - - -def _pyflakes_strip_unused_simple_assigns(source: str, allowed_names: set) -> str: - """Remove simple literal initializations that became unused after extraction. - - Only considers assignments whose target name is in *allowed_names* — the - set of variable names that the extraction actually touched. This prevents - the cleaner from making unrelated changes to variables that were already - unused before the extraction ran. - - Runs pyflakes ``UnusedVariable`` (F841) detection on *source* and strips - any ``Assign`` statement whose right-hand side is a pure literal (no - function calls, no attribute accesses), so we never discard side effects. - - A ``compile()`` check guards against the rare case where the removed line - was the only statement in its block — if the result is invalid Python the - original source is returned unchanged. - """ - import pyflakes.api - import pyflakes.messages - - class _Collector: - def __init__(self): - self.linenos: set = set() - - def unexpectedError(self, filename, msg): # pragma: no cover - pass - - def syntaxError(self, filename, msg, lineno, offset, text): # pragma: no cover - pass - - def flake(self, msg): - if isinstance(msg, pyflakes.messages.UnusedVariable): - self.linenos.add(msg.lineno) - - reporter = _Collector() - pyflakes.api.check(source, "", reporter=reporter) - if not reporter.linenos: - return source - - try: - tree = ast.parse(source) - except SyntaxError: # pragma: no cover - return source # pragma: no cover - - lines_to_remove: set = set() - for node in ast.walk(tree): - if not isinstance(node, ast.Assign): - continue - if node.lineno not in reporter.linenos: - continue - # Restrict to names the extraction actually touched. - assigned: List[str] = [] - _collect_ast_store_names(node.targets[0], assigned) - if not assigned or not set(assigned).issubset(allowed_names): - continue - if _is_pure_literal(node.value): - lines_to_remove.update(range(node.lineno, node.end_lineno + 1)) - - if not lines_to_remove: - return source - - lines = source.splitlines(keepends=True) - cleaned = "".join( - line for i, line in enumerate(lines, 1) if i not in lines_to_remove - ) - try: - compile(cleaned, "", "exec") - except SyntaxError: - return source - return cleaned - - -def _missing_free_vars( - block_src: str, call_srcs: List[str], helper_src: str, source: str -) -> set: - """Return locally-scoped free variable names from block_src absent from the - replacement. - - Free variables are names that are *read* (appear in a ``Load`` context) but - not locally *assigned* (``Store``/``Del``) within the original block. To - avoid false positives from module-level names (imported symbols, globally- - defined functions) that the extracted helper can reference directly, the - check is restricted to names that appear as assignment targets or function - parameters somewhere in *source* — these are variables that live in a local - scope and cannot be reached by the helper without being threaded through as - arguments. - - After this filtering, every remaining name must appear as a bare ``Name`` - node somewhere in the call-site replacements or the helper body. A name - that vanishes from both indicates the LLM silently changed the data flow — - for example by turning a local variable reference into an attribute access - on one of the parameters (``new_source`` → ``transformer.new_source``). - - Returns the set of names that are absent from both. An empty set means the - check passes. Returns an empty set on any ``SyntaxError`` so a parse - failure does not block the extraction — the later ``compile()`` guard will - catch real syntax problems. - """ - try: - block_tree = ast.parse(textwrap.dedent(block_src)) - except SyntaxError: - return set() - - reads: set = set() - stores: set = set() - for node in ast.walk(block_tree): - if isinstance(node, ast.Name): - if isinstance(node.ctx, ast.Load): - reads.add(node.id) - else: - stores.add(node.id) - - free_vars = reads - stores - if not free_vars: - return set() - - # Restrict to names that are locally assigned or are function/lambda - # parameters somewhere in the full source. Module-level names that are - # only ever read (e.g. imported functions, global constants) are in scope - # from the helper definition too and do not need to be passed as args. - try: - source_tree = ast.parse(source) - except SyntaxError: - return set() - source_locals: set = set() - for node in ast.walk(source_tree): - if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): - source_locals.add(node.id) - elif isinstance(node, ast.arg): - source_locals.add(node.arg) - - free_vars = free_vars & source_locals - if not free_vars: - return set() - - replacement_names: set = set() - for src in list(call_srcs) + [helper_src]: - try: - repl_tree = ast.parse(textwrap.dedent(src)) - except SyntaxError: - return set() - for node in ast.walk(repl_tree): - if isinstance(node, ast.Name): - replacement_names.add(node.id) - - return free_vars - replacement_names - - -def _seq_ends_with_return(seq: _SeqInfo) -> bool: - """Return True if the last top-level statement is a non-None return. - - Detects the case where the LLM includes a ``return`` statement inside the - duplicate block but the generated replacement omits it, producing a - function that silently returns ``None`` instead of the original value. - - Bare ``return`` and ``return None`` are excluded: both are semantically - equivalent to falling off the end of a function, so dropping them in a - replacement causes no behavioral change. - """ - try: - tree = ast.parse(textwrap.dedent(seq.source)) - except SyntaxError: - return False - if not tree.body: - return False - last = tree.body[-1] - if not isinstance(last, ast.Return): - return False - # Bare `return` and `return None` are equivalent to implicit None return. - if last.value is None: - return False - if isinstance(last.value, ast.Constant) and last.value.value is None: - return False - return True - - -def _seq_source_contains_yield(source: str) -> bool: - """Return True if *source* contains ``yield`` or ``yield from`` outside - any nested function definition. - - Sequences with a yield cannot be safely extracted into a plain helper - function: extraction would make the helper a generator, forcing call sites - to iterate via ``for``/``async for`` instead of calling it directly. This - is a semantic transformation (e.g. ``async with X as c: yield c`` → - ``async for c in helper(): yield c``) that the extractor must not attempt. - """ - wrapped = "def _f():\n" + textwrap.indent(textwrap.dedent(source), " ") - try: - tree = ast.parse(wrapped) - except SyntaxError: - return False - if not tree.body or not isinstance( - tree.body[0], ast.FunctionDef - ): # pragma: no cover - return False - - def _walk(nodes): - for node in nodes: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - continue # don't cross into nested scope - if isinstance(node, (ast.Yield, ast.YieldFrom)): - return True - if _walk(ast.iter_child_nodes(node)): - return True - return False - - return _walk(tree.body[0].body) - - -def _replacement_contains_return(replacement: str) -> bool: - """Return True if *replacement* contains any return statement. - - Wraps the replacement in a dummy function before parsing so that - ``return`` statements — which are legal inside a function body — do not - cause false SyntaxError rejections. - """ - try: - wrapped = "def _check():\n" + textwrap.indent( - textwrap.dedent(replacement), " " - ) - tree = ast.parse(wrapped) - except SyntaxError: - return False - for node in ast.walk(tree): - if isinstance(node, ast.Return): - return True - return False - - -def _replacement_steals_post_block_line( - group: List[_SeqInfo], call_replacements: List[str], source_lines: List[str] -) -> bool: - """Return True if any replacement's last line duplicates the line after its block. - - The LLM occasionally appends the first statement *after* the replaced block - to the end of the replacement text. When applied, that statement then appears - twice in the assembled output: once inside the replacement and once as the - original untouched line. - """ - for seq, replacement in zip(group, call_replacements): - next_idx = seq.end_line # 0-based index of the first line after the block - # Scan forward past blank lines to find the first real post-block line. - while next_idx < len(source_lines) and not source_lines[next_idx].strip(): - next_idx += 1 - if next_idx >= len(source_lines): - continue - post_block = source_lines[next_idx].strip() - repl_lines = [ln.strip() for ln in replacement.splitlines() if ln.strip()] - if repl_lines and repl_lines[-1] == post_block: - return True - return False - - -def _helper_imports_local_name(helper_source: str, original_source: str) -> bool: - """Return True if helper_source imports a name that is only a local in original. - - Detects the LLM mistake of writing ``import X`` in the helper when ``X`` - was a function parameter or other local name in the original file, not an - importable module. Such imports fail at runtime with ModuleNotFoundError. - """ - try: - helper_tree = ast.parse(textwrap.dedent(helper_source)) - except SyntaxError: - return False - - helper_imports: set = set() - for node in ast.walk(helper_tree): - if isinstance(node, ast.Import): - for alias in node.names: - name = alias.asname if alias.asname else alias.name.split(".")[0] - helper_imports.add(name) - elif isinstance(node, ast.ImportFrom): - for alias in node.names: - name = alias.asname if alias.asname else alias.name - helper_imports.add(name) - - if not helper_imports: - return False - - try: - orig_tree = ast.parse(original_source) - except SyntaxError: - return False - - # Names already imported at the top level of the original file. - orig_top_imports: set = set() - for node in orig_tree.body: - if isinstance(node, ast.Import): - for alias in node.names: - name = alias.asname if alias.asname else alias.name.split(".")[0] - orig_top_imports.add(name) - elif isinstance(node, ast.ImportFrom): - for alias in node.names: - name = alias.asname if alias.asname else alias.name - orig_top_imports.add(name) - - new_helper_imports = helper_imports - orig_top_imports - if not new_helper_imports: - return False - - # Parameter names in the original file (potential mock-injected locals). - orig_params: set = set() - for node in ast.walk(orig_tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - for arg in node.args.args + node.args.posonlyargs + node.args.kwonlyargs: - orig_params.add(arg.arg) - if node.args.vararg: - orig_params.add(node.args.vararg.arg) - if node.args.kwarg: - orig_params.add(node.args.kwarg.arg) - - return bool(new_helper_imports & orig_params) - - -def _lift_and_dedup_imports(source: str) -> str: - """Lift misplaced module-level imports to the import block and deduplicate. - - When a helper is inserted before a function that is not the first in the - file, its leading ``from X import Y`` lines land after the first - ``def``/``class``, violating PEP 8. When a helper re-imports names - already present at the top, flake8 reports F811. This function fixes both: - - 1. Collect every simple, unindented ``from X import …`` / ``import X`` - line from anywhere in the file. - 2. Merge names for the same module (deduplicate). - 3. Emit the merged set within the top-of-file import block (before the - first ``def``/``class``), removing all later occurrences. - - Only single-line imports without parentheses, backslash continuations, or - inline comments are handled. Indented imports (``if TYPE_CHECKING:``, - function-local lazy imports, etc.) and wildcard imports are left untouched. - """ - lines = source.splitlines(keepends=True) - n = len(lines) - - # ── pass 1: find the import block boundary ────────────────────────────── - # The import block ends at the first unindented def/class line. - first_funcdef_idx = n - for i, line in enumerate(lines): - if line[:1] in (" ", "\t"): - continue - if re.match(r"^(?:async\s+def|def|class)\s", line.strip()): - first_funcdef_idx = i - break - - # ── pass 2: collect simple unindented import lines ────────────────────── - _FROM_RE = re.compile(r"^from\s+(\S+)\s+import\s+([^(\\#]+)$") - _PLAIN_RE = re.compile(r"^import\s+(\S+)$") - - all_imports: List[Tuple[int, str]] = [] # (line_idx, stripped_text) - import_indices: set = set() - last_block_import_idx = -1 - - for i, line in enumerate(lines): - if line[:1] in (" ", "\t"): - continue - stripped = line.strip() - mf = _FROM_RE.match(stripped) - if mf: - names_str = mf.group(2).strip() - if not names_str or names_str == "*": - continue - names = [nm.strip() for nm in names_str.split(",") if nm.strip()] - if not names: - continue - all_imports.append((i, stripped)) - import_indices.add(i) - if i < first_funcdef_idx: - last_block_import_idx = i - continue - mp = _PLAIN_RE.match(stripped) - if mp: - all_imports.append((i, stripped)) - import_indices.add(i) - if i < first_funcdef_idx: - last_block_import_idx = i - - if not all_imports: - return source - - # ── pass 3: build merged import map (ordered by first appearance) ─────── - from_map: Dict[str, List[str]] = {} # module -> merged name list - from_order: List[str] = [] - plain_order: List[str] = [] - plain_seen: set = set() - - for _, text in all_imports: - mf = _FROM_RE.match(text) - if mf: - module = mf.group(1) - names = [nm.strip() for nm in mf.group(2).split(",") if nm.strip()] - if module not in from_map: - from_map[module] = list(names) - from_order.append(module) - else: - existing_set = set(from_map[module]) - for name in names: - if name not in existing_set: - from_map[module].append(name) - existing_set.add(name) - else: - # Must be a plain import — guaranteed by pass 2 filter. - module = _PLAIN_RE.match(text).group(1) # type: ignore[union-attr] - if module not in plain_seen: - plain_order.append(module) - plain_seen.add(module) - - # ── early exit if nothing to do ───────────────────────────────────────── - has_misplaced = any(i >= first_funcdef_idx for i, _ in all_imports) - from_counts: Dict[str, int] = {} - plain_counts: Dict[str, int] = {} - for _, text in all_imports: - mf = _FROM_RE.match(text) - if mf: - mod = mf.group(1) - from_counts[mod] = from_counts.get(mod, 0) + 1 - else: - mod = _PLAIN_RE.match(text).group(1) # type: ignore[union-attr] - plain_counts[mod] = plain_counts.get(mod, 0) + 1 - if not ( - has_misplaced - or any(v > 1 for v in from_counts.values()) - or any(v > 1 for v in plain_counts.values()) - ): - return source - - # ── pass 4: build the complete sorted import block ────────────────────── - # Combine every merged import (existing block + newly lifted) and sort the - # whole list so stdlib never ends up after third-party just because it was - # a newly lifted import appended at the end. - all_final_imports = [ - f"from {mod} import {', '.join(from_map[mod])}" for mod in from_order - ] + [f"import {mod}" for mod in plain_order] - sorted_imports = _sort_imports_pep8(all_final_imports) - - first_block_import_idx = min( - (i for i, _ in all_imports if i < first_funcdef_idx), default=-1 - ) - - # ── pass 5: rebuild source ─────────────────────────────────────────────── - # Emit the sorted block at the first block import position (or just before - # the first def/class if there are no block imports). Skip all original - # import lines and blank lines within the original block region — the - # sorted block replaces them entirely. - result: List[str] = [] - import_block_emitted = False - - for i, line in enumerate(lines): - # Edge case: no block imports — insert before the first def/class. - if i == first_funcdef_idx and not import_block_emitted: - for imp in sorted_imports: - result.append(imp + "\n") - import_block_emitted = True - - # Emit the sorted block at the position of the first block import. - if i == first_block_import_idx: - for imp in sorted_imports: - result.append(imp + "\n") - import_block_emitted = True - continue # the original import line is replaced by the block above - - # Drop all other import lines (block duplicates and misplaced). - if i in import_indices: - continue - - # Drop blank lines that fell between import lines in the original block - # — they were section separators that the sorted block supersedes. - if ( - first_block_import_idx >= 0 - and first_block_import_idx < i <= last_block_import_idx - and not line.strip() - ): - continue - - result.append(line) - - result_str = "".join(result) - # When a misplaced import is removed, the blank line that visually separated - # it from the following def/class is left behind. Combined with the two - # trailing blank lines already written after the previous helper, this - # produces three consecutive blank lines — a PEP 8 / E303 violation. - # Collapse any run of 3+ blank lines down to exactly 2 (the PEP 8 maximum - # between top-level definitions). Four or more '\n' in a row means three - # or more blank lines; replace with exactly three '\n' (= two blank lines). - result_str = re.sub(r"\n{4,}", "\n\n\n", result_str) - return result_str - - -def _names_assigned_in(block_source: str) -> set: - """Return names assigned at the top level of block_source. - - Covers bare ``x = ...`` (ast.Assign) and augmented ``x += ...`` - (ast.AugAssign) statements only; other assignment forms are ignored. - """ - try: - tree = ast.parse(textwrap.dedent(block_source)) - except SyntaxError: - return set() - names: set = set() - for node in tree.body: - if isinstance(node, ast.Assign): - for target in node.targets: - for n in ast.walk(target): - if isinstance(n, ast.Name): - names.add(n.id) - elif isinstance(node, ast.AugAssign): - for n in ast.walk(node.target): - if isinstance(n, ast.Name): - names.add(n.id) - return names - - -def _find_escaping_vars(group: List[_SeqInfo], source_lines: List[str]) -> set: - """Return names assigned in any group sequence that are referenced after it. - - A variable "escapes" when the block assigns it and subsequent code in the - same scope (at the same or deeper indentation level) references it. - The helper must return these variables so callers that need them can - capture the return value. - """ - escaping: set = set() - for seq in group: - block_src = "".join(source_lines[seq.start_line - 1 : seq.end_line]) - assigned = _names_assigned_in(block_src) - if not assigned: - continue - - # Infer the block's indentation level from its first non-empty line. - first_line = next( - ( - ln - for ln in source_lines[seq.start_line - 1 : seq.end_line] - if ln.strip() - ), - "", - ) - block_indent = len(first_line) - len(first_line.lstrip()) - - # Collect lines that follow the block within the same scope. - # For indented blocks: stop when indentation falls below block_indent. - # For module-level (indent 0): stop at the next def/class statement. - after_lines: List[str] = [] - for line in source_lines[seq.end_line :]: - if not line.strip(): - after_lines.append(line) - continue - line_indent = len(line) - len(line.lstrip()) - if block_indent == 0: - if re.match(r"def |class ", line): - break - elif line_indent < block_indent: - break - after_lines.append(line) - - if not after_lines: - continue - - after_src = "".join(after_lines) - try: - after_tree = ast.parse(textwrap.dedent(after_src)) - except SyntaxError: - continue - - used_after = {n.id for n in ast.walk(after_tree) if isinstance(n, ast.Name)} - escaping |= assigned & used_after - - return escaping - - -def _extract_defined_names(source: str) -> set: - """Return all function and class names defined anywhere in *source*.""" - try: - tree = ast.parse(source) - except SyntaxError: - return set() - return { - node.name - for node in ast.walk(tree) - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) - } - - -def _would_create_proxy_wrappers( - group: List[_SeqInfo], all_functions: List[_FunctionInfo] -) -> bool: - """Return True if extracting this group would leave *some but not all* members - as trivial proxy wrappers. - - A function becomes a trivial proxy wrapper when its entire body is the - extracted block — after extraction it would contain only a single call to - the new helper, with no meaningful logic of its own. - - When *every* member of the group would become a proxy, extraction is still - worthwhile: all functions delegate to the same helper, which eliminates the - duplication. The problematic case is a mixed group where some members lose - all their logic while others keep meaningful bodies. - """ - proxy_count = 0 - non_module_count = 0 - for seq in group: - if seq.scope == "": - continue - non_module_count += 1 - func_outer_scope = ( - seq.class_scope if seq.class_scope is not None else "" - ) - for func in all_functions: - if func.name == seq.scope and func.scope == func_outer_scope: - if len(seq.stmts) == func.body_stmt_count: - proxy_count += 1 - break - return 0 < proxy_count < non_module_count - - -# --------------------------------------------------------------------------- -# Text editing -# --------------------------------------------------------------------------- - - -def _build_helper_insertion( - source_lines: List[str], - insert_pos: int, - helper_source: str, - placement: str, -) -> Tuple[int, int, str]: - """Build an edit tuple that inserts helper_source with correct surrounding blanks. - - Always returns a pure insertion (start == end) so that two groups inserting - before the same scope are never in conflict: pure insertions are not subject - to the overlap-skip logic in _apply_edits. - - The insertion point is placed after all blank lines that already exist - around insert_pos (right before the def/decorator line). Leading blank - lines are prepended only to make up the difference so the result always - has exactly ``blank_lines`` blank lines before the helper. - """ - blank_lines = 1 if placement.startswith("staticmethod:") else 2 - - # Count consecutive blank lines immediately before insert_pos. - before_blanks = 0 - i = insert_pos - 1 - while i >= 0 and not source_lines[i].strip(): - before_blanks += 1 - i -= 1 - - # Count consecutive blank lines at and immediately after insert_pos. - after_blanks = 0 - i = insert_pos - while i < len(source_lines) and not source_lines[i].strip(): - after_blanks += 1 - i += 1 - - # Insert right before the def/decorator (after all surrounding blanks). - insert_at = insert_pos + after_blanks - # Prepend only as many blank lines as are still missing. - leading = max(0, blank_lines - (before_blanks + after_blanks)) - clean = helper_source.strip("\n") + "\n" - text = "\n" * leading + clean + "\n" * blank_lines - return (insert_at, insert_at, text) - - -def _apply_edits(source: str, edits: List[Tuple[int, int, str]]) -> str: - """Apply (start_0, end_0, text) edits bottom-to-top. - - Indices are 0-based; lines[start_0:end_0] is replaced with text. - An insertion before line N uses start_0 == end_0 == N. - Overlapping replacement ranges are skipped. - """ - lines = source.splitlines(keepends=True) - if lines and not lines[-1].endswith("\n"): - lines[-1] += "\n" - - applied: List[Tuple[int, int]] = [] - for start, end, text in sorted(edits, key=lambda e: (e[0], e[1]), reverse=True): - is_insertion = start == end - if not is_insertion: - if any(a_start < end and a_end > start for a_start, a_end in applied): - continue - applied.append((start, end)) - new_lines = text.splitlines(keepends=True) - if new_lines and not new_lines[-1].endswith("\n"): - new_lines[-1] += "\n" - lines[start:end] = new_lines - - return "".join(lines) - - -def _skip_class_docstring(source_lines: List[str], after_class_line: int) -> int: - """Return the 0-based line index after the class docstring, if any. - - Given the line immediately after ``class Foo:`` (or its colon line), - advance past any leading blank lines and then past a string-literal - docstring (single- or triple-quoted). If no docstring is present, - returns ``after_class_line`` unchanged. - """ - i = after_class_line - n = len(source_lines) - # Skip blank lines inside the class body. - while i < n and not source_lines[i].strip(): - i += 1 - if i >= n: - return after_class_line - stripped = source_lines[i].lstrip() - # Check for a triple-quoted docstring. - for q in ('"""', "'''"): - if stripped.startswith(q): - # Check whether the closing quote is on the same line (after the - # opening). - rest = stripped[len(q) :] - if q in rest: - # Single-line triple-quoted docstring. - return i + 1 - # Multi-line: scan forward for the closing triple-quote. - i += 1 - while i < n: - if q in source_lines[i]: - return i + 1 - i += 1 - return i # malformed, best-effort - # Single-quoted docstring (rare but valid). - for q in ('"', "'"): - if stripped.startswith(q) and not stripped.startswith(q * 2): - return i + 1 - return after_class_line - - -def _find_insertion_point(source: str, scope: str) -> int: - """Return 0-based line index to insert before. - - For module scope, inserts after the last import. - For a named scope, inserts before the def/class line. - - If the named scope resolves to an indented ``def`` (i.e. a class method), - inserting a module-level helper immediately before it would end the class - definition prematurely — the remaining class methods would be silently - re-parsed as nested functions of the helper, producing valid-syntax but - broken code that ``compile()`` does not catch. In that case we walk - backwards to the enclosing class definition and insert before it instead. - """ - source_lines = source.splitlines() - if scope == "": - last_import = -1 - for i, line in enumerate(source_lines): - stripped = line.strip() - if stripped.startswith("import ") or stripped.startswith("from "): - last_import = i - return last_import + 1 - - pattern = re.compile(rf"^\s*(?:async\s+def|def|class)\s+{re.escape(scope)}\s*[\(:]") - for i, line in enumerate(source_lines): - if pattern.match(line): - method_indent = len(line) - len(line.lstrip()) - if method_indent > 0: - # The def is inside a class body. Walk backwards to find the - # enclosing class definition and insert before that instead. - # If the first lower-indent non-blank line is NOT a class - # definition (i.e. the def is a nested function inside a - # regular function), stop immediately so we don't mis-identify - # an unrelated class above the outer function as the enclosing - # class. - for j in range(i - 1, -1, -1): - prev = source_lines[j] - if not prev.strip(): - continue - prev_indent = len(prev) - len(prev.lstrip()) - if prev_indent < method_indent: - if re.match(r"\s*class\s+\w+", prev): - return j - break # nested function — fall through to decorator walk - # Walk backwards over any preceding decorator lines (including - # multi-line decorator arguments) so the helper is inserted - # before the decorator block, not between decorators and the def. - j = i - 1 - paren_depth = 0 - while j >= 0: - stripped = source_lines[j].strip() - if not stripped: - break - for ch in stripped: - if ch == ")": - paren_depth += 1 - elif ch == "(": - paren_depth -= 1 - if paren_depth == 0 and not stripped.startswith("@"): - break - j -= 1 - return j + 1 - return 0 - - -# --------------------------------------------------------------------------- -# Main refactor -# --------------------------------------------------------------------------- - - -class DuplicateExtractor(Refactor): - """Detect and extract duplicate code blocks into helper functions via LLM.""" - - def __init__( - self, - changed_ranges: List[Tuple[int, int]], - source: str = "", - verbose: bool = True, - min_weight: int = _MIN_WEIGHT, - max_seq_len: int = _MAX_SEQ_LEN, - model: str = _MODEL, - helper_docstrings: bool = False, - provider: str = "anthropic", - extraction_retries: int = 1, - llm_verify_retries: int = 1, - base_url: Optional[str] = None, - tool_choice: Optional[str] = None, - api_timeout: float = 60.0, - match_functions: bool = True, - timing: str = "detailed", - current_file: str = "", - rate_limit_retries: int = 6, - rate_limit_backoff: float = 20.0, - ) -> None: - super().__init__(changed_ranges, source=source, verbose=verbose) - self.current_file = current_file - self.timing = timing - self._min_weight = min_weight - self._base_max_seq_len = max_seq_len - self._model = model - self._helper_docstrings = helper_docstrings - self._provider = provider - self._extraction_retries = extraction_retries - self._llm_verify_retries = llm_verify_retries - self._base_url = base_url - self._tool_choice = tool_choice - self._api_timeout = api_timeout - self._hard_timeout = api_timeout + 30 - self._match_functions = match_functions - self._rate_limit_retries = rate_limit_retries - self._rate_limit_backoff = rate_limit_backoff - self._new_source: Optional[str] = None - if source: - self._analyze(source) - - def _analyze(self, source: str) -> None: - # 1. Parse tree; early-return on syntax error. - try: - tree = cst.parse_module(source) - except cst.ParserSyntaxError: - return - - # 2. Source lines. - source_lines = source.splitlines(keepends=True) - - # 3. Collect functions. - func_collector = _FunctionCollector(source_lines) - MetadataWrapper(tree).visit(func_collector) - all_functions = func_collector.functions - - # 4-5. Build function body fingerprint map (only for called functions). - called_names = _collect_called_names(source) - func_body_fps = _build_function_body_fps(all_functions, called_names) - - # 6. Compute max sequence length to capture full function bodies. - max_seq_len = max( - max(f.body_stmt_count for f in all_functions) if all_functions else 0, - self._base_max_seq_len, - ) - - # 7. Collect sequences. - collector = _SequenceCollector( - source_lines, max_seq_len=max_seq_len, min_weight=self._min_weight - ) - MetadataWrapper(tree).visit(collector) - - # 8. Preliminary duplicate groups. - groups = _find_duplicate_groups(collector.sequences, self.changed_ranges) - - # 9. Check whether any sequence can be replaced with an existing function. - has_func_matches = ( - self._match_functions - and func_body_fps - and any( - _overlaps_diff(seq, self.changed_ranges) - and seq.fingerprint in func_body_fps - and func_body_fps[seq.fingerprint].name != seq.scope - for seq in collector.sequences - ) - ) - - # 10. Early exit — nothing to do. - if not has_func_matches and not groups: - return - - # 12. Create API client. - api_key = _llm_client.get_api_key(self._provider, caller="DuplicateExtractor") - client = _llm_client.make_client( - self._provider, api_key, timeout=self._api_timeout, base_url=self._base_url - ) - edits: List[Tuple[int, int, str]] = [] - pending_changes: List[str] = [] - # Extraction groups tracked separately so the final combined check can - # drop any whose call-site edits were silently overridden by overlapping - # edits from another group or the func-match pass. - extraction_groups: List[Tuple[str, List[Tuple[int, int, str]], str]] = [] - matched_line_ranges: set = set() - - # 14. Function body match pass. - if self._match_functions and func_body_fps: - for seq in collector.sequences: - if not _overlaps_diff(seq, self.changed_ranges): - continue - if seq.fingerprint not in func_body_fps: - continue - func = func_body_fps[seq.fingerprint] - if func.name == seq.scope: - continue - if self.verbose: - print( - f"crispen: DuplicateExtractor: func-match check — " - f"scope '{seq.scope}': lines {seq.start_line}-{seq.end_line}" - f" → '{func.name}'", - file=sys.stderr, - flush=True, - ) - self.stats.llm_veto_calls += 1 - timing: list = [] - try: - is_valid, reason, _veto_notes = _run_with_timeout( - _llm_veto_func_match, - self._hard_timeout, - client, - seq, - func, - source, - self._model, - self._provider, - tool_choice_override=self._tool_choice, - _timing_out=timing, - rate_limit_retries=self._rate_limit_retries, - rate_limit_backoff=self._rate_limit_backoff, - ) - if timing: - lr = timing[0] - self.stats.record_llm_call( - lr.elapsed, - lr.input_tokens, - lr.output_tokens, - "veto", - "duplicate_extractor", - self.current_file, - ) - except _ApiTimeout: - print( - "crispen: DuplicateExtractor: → func-match veto timed out", - file=sys.stderr, - flush=True, - ) - continue - if self.verbose: - status = "ACCEPTED" if is_valid else "VETOED" - timing_suffix = "" - if self.timing == "detailed" and timing: - lr = timing[0] - timing_suffix = ( - f" [{lr.elapsed:.2f}s," - f" {lr.input_tokens:,} in / {lr.output_tokens:,} out]" - ) - print( - f"crispen: DuplicateExtractor: → {status}: {reason}" - f"{timing_suffix}", - file=sys.stderr, - flush=True, - ) - if not is_valid: - self.stats.llm_rejected += 1 - continue - timing2: list = [] - if func.scope == "" and not func.params: - replacement = _generate_no_arg_call(seq, func) - else: - self.stats.llm_edit_calls += 1 - try: - replacement = _run_with_timeout( - _llm_generate_call, - self._hard_timeout, - client, - seq, - func, - source, - self._model, - self._provider, - tool_choice_override=self._tool_choice, - _timing_out=timing2, - rate_limit_retries=self._rate_limit_retries, - rate_limit_backoff=self._rate_limit_backoff, - ) - if timing2: - lr = timing2[0] - self.stats.record_llm_call( - lr.elapsed, - lr.input_tokens, - lr.output_tokens, - "edit", - "duplicate_extractor", - self.current_file, - ) - except _ApiTimeout: - print( - "crispen: DuplicateExtractor:" - " → call generation timed out", - file=sys.stderr, - flush=True, - ) - continue - if replacement is None: - continue # pragma: no cover - if not _verify_extraction(None, [replacement]): - continue - if self.verbose: - timing_suffix = "" - if self.timing == "detailed" and timing2: - lr = timing2[0] - timing_suffix = ( - f" [{lr.elapsed:.2f}s," - f" {lr.input_tokens:,} in / {lr.output_tokens:,} out]" - ) - print( - f"crispen: DuplicateExtractor: → replacing '{seq.scope}'" - f" with '{func.name}()'{timing_suffix}", - file=sys.stderr, - flush=True, - ) - edits.append((seq.start_line - 1, seq.end_line, replacement)) - matched_line_ranges.add((seq.start_line, seq.end_line)) - pending_changes.append( - f"DuplicateExtractor: replaced '{seq.scope}' body" - f" with call to '{func.name}'" - ) - - # 15. Recompute duplicate groups excluding matched sequences. - if matched_line_ranges: - remaining = [ - s - for s in collector.sequences - if not any( - s.start_line <= r_end and s.end_line >= r_start - for r_start, r_end in matched_line_ranges - ) - ] - groups = _find_duplicate_groups(remaining, self.changed_ranges) - - # 16. Log group count. - if groups and self.verbose: - print( - f"crispen: DuplicateExtractor: found {len(groups)} duplicate group(s)", - file=sys.stderr, - flush=True, - ) - - # 17. Duplicate group extraction pass. - used_names = _extract_defined_names(source) - for group in groups: - # Compute escaping vars algorithmically before any LLM call so the - # extraction prompt can instruct the LLM to return them. - escaping_vars = frozenset(_find_escaping_vars(group, source_lines)) - - # Skip groups that would leave a function as a trivial proxy wrapper - # (i.e. the extracted block is the function's entire body). - if _would_create_proxy_wrappers(group, all_functions): - if self.verbose: - print( - "crispen: DuplicateExtractor: skipping group — " - "extraction would leave a trivial proxy wrapper", - file=sys.stderr, - flush=True, - ) - continue - - if self.verbose: - ranges_str = ", ".join( - f"lines {s.start_line}-{s.end_line}" for s in group - ) - print( - f"crispen: DuplicateExtractor: veto check — " - f"scope '{group[0].scope}': {ranges_str}", - file=sys.stderr, - flush=True, - ) - self.stats.llm_veto_calls += 1 - timing3: list = [] - try: - is_valid, reason, veto_notes = _run_with_timeout( - _llm_veto, - self._hard_timeout, - client, - group, - self._model, - self._provider, - tool_choice_override=self._tool_choice, - _timing_out=timing3, - rate_limit_retries=self._rate_limit_retries, - rate_limit_backoff=self._rate_limit_backoff, - ) - if timing3: - lr = timing3[0] - self.stats.record_llm_call( - lr.elapsed, - lr.input_tokens, - lr.output_tokens, - "veto", - "duplicate_extractor", - self.current_file, - ) - except _ApiTimeout: - print( - "crispen: DuplicateExtractor: API call timed out, skipping group", - file=sys.stderr, - flush=True, - ) - continue - if self.verbose: - status = "ACCEPTED" if is_valid else "VETOED" - timing_suffix = "" - if self.timing == "detailed" and timing3: - lr = timing3[0] - timing_suffix = ( - f" [{lr.elapsed:.2f}s," - f" {lr.input_tokens:,} in / {lr.output_tokens:,} out]" - ) - print( - f"crispen: DuplicateExtractor: → {status}: {reason}" - f"{timing_suffix}", - file=sys.stderr, - flush=True, - ) - if not is_valid: - self.stats.llm_rejected += 1 - continue - - # Extraction retry loop: attempt extraction up to - # 1 + _extraction_retries times on algorithmic failure, and up to - # 1 + _llm_verify_retries additional times on LLM verify failure. - alg_retries_left = self._extraction_retries - llm_verify_retries_left = self._llm_verify_retries - prev_failures: List[str] = [] - prev_output: Optional[dict] = None - - while True: - self.stats.llm_edit_calls += 1 - timing4: list = [] - try: - extraction = _run_with_timeout( - _llm_extract, - self._hard_timeout, - client, - group, - source, - escaping_vars, - used_names=frozenset(used_names), - model=self._model, - helper_docstrings=self._helper_docstrings, - provider=self._provider, - veto_notes=veto_notes, - prev_failures=prev_failures, - prev_output=prev_output, - tool_choice_override=self._tool_choice, - _timing_out=timing4, - rate_limit_retries=self._rate_limit_retries, - rate_limit_backoff=self._rate_limit_backoff, - ) - if timing4: - lr = timing4[0] - self.stats.record_llm_call( - lr.elapsed, - lr.input_tokens, - lr.output_tokens, - "edit", - "duplicate_extractor", - self.current_file, - ) - except _ApiTimeout: - print( - "crispen: DuplicateExtractor: API call timed out," - " skipping group", - file=sys.stderr, - flush=True, - ) - break - if extraction is None: - break # pragma: no cover - - if self.verbose and self.timing == "detailed" and timing4: - lr = timing4[0] - print( - f"crispen: DuplicateExtractor: → extraction" - f" [{lr.elapsed:.2f}s," - f" {lr.input_tokens:,} in / {lr.output_tokens:,} out]", - file=sys.stderr, - flush=True, - ) - - helper_source = extraction["helper_source"] - if not self._helper_docstrings: - helper_source = _strip_helper_docstring(helper_source) - call_replacements = extraction["call_site_replacements"] - placement = extraction.get("placement", "module_level") - # Auto-indent 0-indent helpers for staticmethod: placement. - # The LLM sometimes writes a module-level def even when it - # selects staticmethod:ClassName. Inserting 0-indent code - # inside the class body ends the class silently and makes all - # subsequent methods nested inside the helper — valid syntax - # but semantically broken, so compile() does not catch it. - if placement.startswith("staticmethod:") and helper_source: - first_code = next( - (ln for ln in helper_source.splitlines() if ln.strip()), "" - ) - if first_code and not first_code[0].isspace(): - helper_source = textwrap.indent(helper_source, " ") - func_name = extraction["function_name"] - - # Helpers are always file-internal; enforce a leading underscore. - if not func_name.startswith("_"): - _old_name = func_name - func_name = "_" + func_name - _rename_pat = re.compile(r"\b" + re.escape(_old_name) + r"\b") - helper_source = _rename_pat.sub(func_name, helper_source) - call_replacements = [ - _rename_pat.sub(func_name, r) for r in call_replacements - ] - - _check_failed = False - _failures: List[str] = [] - - # Check 1: name collision - # Pre-check: placement consistency with call-site class scopes. - if placement.startswith("staticmethod:"): - group_class_scopes = {s.class_scope for s in group} - if len(group_class_scopes) != 1 or None in group_class_scopes: - _failures.append( - "staticmethod placement is invalid when call sites span " - "multiple classes or scopes; use module_level instead" - ) - if self.verbose: - print( - "crispen: DuplicateExtractor: extraction FAILED — " - "staticmethod placement invalid for cross-class group", - file=sys.stderr, - flush=True, - ) - _check_failed = True - elif placement.split(":", 1)[1] != next(iter(group_class_scopes)): - named_class = placement.split(":", 1)[1] - actual_class = next(iter(group_class_scopes)) - _failures.append( - f"staticmethod names class '{named_class}' but all call " - f"sites are in '{actual_class}'; use " - f"'staticmethod:{actual_class}' instead" - ) - if self.verbose: - print( - f"crispen: DuplicateExtractor: extraction FAILED — " - f"staticmethod names wrong class '{named_class}' " - f"(actual: '{actual_class}')", - file=sys.stderr, - flush=True, - ) - _check_failed = True - if placement == "module_level": - # Reject if any call site invokes the helper as an instance - # method (self.(...)) — that is inconsistent with - # module-level placement and will fail at runtime. - _self_call_pat = re.compile(rf"\bself\.{re.escape(func_name)}\s*\(") - if any(_self_call_pat.search(r) for r in call_replacements): - group_class_scopes = {s.class_scope for s in group} - if ( - len(group_class_scopes) == 1 - and None not in group_class_scopes - ): - only_class = next(iter(group_class_scopes)) - placement_hint = f"use 'staticmethod:{only_class}' instead" - else: - placement_hint = ( - f"change call sites to call " - f"'{func_name}(...)' directly" - ) - _failures.append( - f"module_level placement is inconsistent with call " - f"sites that invoke the helper as " - f"'self.{func_name}(...)'; {placement_hint}" - ) - if self.verbose: - print( - f"crispen: DuplicateExtractor: extraction FAILED" - f" — module_level placement conflicts with " - f"self.{func_name}() call sites", - file=sys.stderr, - flush=True, - ) - _check_failed = True - if func_name in used_names: - _failures.append( - f"name collision: '{func_name}' is already defined," - " choose a different name" - ) - if self.verbose: - print( - f"crispen: DuplicateExtractor: extraction FAILED — " - f"name collision: '{func_name}' is already defined", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - # Check 2: call site count - if not _check_failed and len(call_replacements) != len(group): - _failures.append( - f"wrong call_site_replacements count" - f" (expected {len(group)}, got {len(call_replacements)})" - ) - if self.verbose: - print( - f"crispen: DuplicateExtractor: extraction FAILED — " - f"wrong call_site_replacements count " - f"(expected {len(group)}, got {len(call_replacements)})", - file=sys.stderr, - flush=True, - ) - print( - f"crispen: DuplicateExtractor: helper_source: " - f"{helper_source!r}", - file=sys.stderr, - flush=True, - ) - print( - f"crispen: DuplicateExtractor: call_site_replacements: " - f"{call_replacements!r}", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - if not _check_failed: - # Normalize each replacement's indentation to match its - # original block. The LLM sometimes returns replacements at - # column 0; this re-indents them so the assembled edit is - # valid Python. - call_replacements = [ - _normalize_replacement_indentation(seq, r) - for seq, r in zip(group, call_replacements) - ] - - # Strip unused variable assignments from call-site - # replacements. The LLM may assign return values that are - # never used after the block (e.g. when the helper returns a - # value only needed at some call sites), which would produce - # flake8 F841 warnings. - call_replacements = [ - _strip_unused_call_assignments( - r, - source_lines[ - seq.end_line : _scope_end_line( - source_lines, seq.scope, seq.end_line - ) - ], - ) - for seq, r in zip(group, call_replacements) - ] - - # Check 3: post-block line theft - if _replacement_steals_post_block_line( - group, call_replacements, source_lines - ): - _failures.append( - "replacement duplicates the line after the block" - ) - if self.verbose: - print( - "crispen: DuplicateExtractor: extraction FAILED — " - "replacement duplicates the line after the block", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - # Check 4: syntax validation - if not _check_failed and not _verify_extraction( - helper_source, call_replacements - ): - _failures.append("invalid helper or replacement syntax") - if self.verbose: - print( - "crispen: DuplicateExtractor: extraction FAILED — " - "invalid helper or replacement syntax", - file=sys.stderr, - flush=True, - ) - print( - f"crispen: DuplicateExtractor: helper_source: " - f"{helper_source!r}", - file=sys.stderr, - flush=True, - ) - print( - f"crispen: DuplicateExtractor: call_site_replacements: " - f"{call_replacements!r}", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - # Check 5: return statement consistency - if not _check_failed and any( - _seq_ends_with_return(seq) - and not _replacement_contains_return(repl) - for seq, repl in zip(group, call_replacements) - ): - _failures.append("block ends with return but replacement omits it") - if self.verbose: - print( - "crispen: DuplicateExtractor: extraction FAILED — " - "block ends with return but replacement omits it", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - # Check 6: helper must not import local names - if not _check_failed and _helper_imports_local_name( - helper_source, source - ): - _failures.append( - "helper imports a name that is a parameter/local" - " in the original file" - ) - if self.verbose: - print( - "crispen: DuplicateExtractor: extraction FAILED — " - "helper imports a name that is a parameter/local " - "in the original file", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - # Check 7: new attribute access - if not _check_failed: - new_attrs = _collect_called_attr_names( - textwrap.dedent(helper_source) - ) - _collect_called_attr_names(source) - if new_attrs: - _failures.append( - f"helper introduces new attribute access(es) not in" - f" original: {', '.join(sorted(new_attrs))}" - ) - if self.verbose: - print( - f"crispen: DuplicateExtractor: extraction FAILED — " - f"helper introduces new attribute access(es) not in" - f" original: {', '.join(sorted(new_attrs))}", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - # Check 8: free variable preservation - if not _check_failed: - seq0 = group[0] - block_src = "".join( - source_lines[seq0.start_line - 1 : seq0.end_line] - ) - missing = _missing_free_vars( - block_src, call_replacements, helper_source, source - ) - if missing: - _failures.append( - f"free variable(s) from original block missing in" - f" replacement: {', '.join(sorted(missing))}" - ) - if self.verbose: - print( - f"crispen: DuplicateExtractor: extraction FAILED — " - f"free variable(s) from original block missing in " - f"replacement: {', '.join(sorted(missing))}", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - # Build this group's edits (only if pre-edit checks passed). - group_edits: List[Tuple[int, int, str]] = [] - candidate = "" - if not _check_failed: - for seq, replacement in zip(group, call_replacements): - group_edits.append( - (seq.start_line - 1, seq.end_line, replacement) - ) - first_seq = min(group, key=lambda s: s.start_line) - if placement.startswith("staticmethod:"): - # Insert inside the class body, after "class Foo:" and - # any class docstring (which must remain first). - scope = placement.split(":", 1)[1] - class_line = _find_insertion_point(source, scope) - insert_pos = _skip_class_docstring(source_lines, class_line + 1) - else: - scope = first_seq.scope - insert_pos = _find_insertion_point(source, scope) - group_edits.append( - _build_helper_insertion( - source_lines, insert_pos, helper_source, placement - ) - ) - # Compile the per-group candidate independently so one bad - # extraction doesn't discard valid ones for the same file. - candidate = _apply_edits(source, group_edits) - - # Re-strip unused variable assignments using the assembled - # candidate's following lines. The initial pass (above) - # used the original source, which can incorrectly retain an - # assignment when another call site's original block - # referenced the same name. Re-running with candidate - # following lines also handles partial-tuple targets - # (``a, _ = helper()``) the same way the initial pass does. - cand_lines = candidate.splitlines(keepends=True) - restripped = [] - for seq, repl in zip(group, call_replacements): - cs0 = seq.start_line - 1 - offset = sum( - len(et.splitlines(keepends=True)) - (ee - es) - for (es, ee, et) in group_edits - if es < cs0 - ) - new_end = cs0 + offset + len(repl.splitlines(keepends=True)) - scope_end = _scope_end_line(cand_lines, seq.scope, new_end) - restripped.append( - _strip_unused_call_assignments( - repl, cand_lines[new_end:scope_end] - ) - ) - if restripped != call_replacements: - call_replacements = restripped - group_edits = [ - (seq.start_line - 1, seq.end_line, r) - for seq, r in zip(group, call_replacements) - ] - group_edits.append( - _build_helper_insertion( - source_lines, insert_pos, helper_source, placement - ) - ) - candidate = _apply_edits(source, group_edits) - - # Check 9: assembled output is valid Python - try: - compile(candidate, "", "exec") - except SyntaxError as exc: - _failures.append(f"assembled edit not valid Python: {exc}") - if self.verbose: - print( - f"crispen: DuplicateExtractor: extraction FAILED — " - f"assembled edit not valid Python: {exc}", - file=sys.stderr, - flush=True, - ) - print( - f"crispen: DuplicateExtractor: helper_source: " - f"{helper_source!r}", - file=sys.stderr, - flush=True, - ) - print( - f"crispen: DuplicateExtractor:" - f" call_site_replacements: " - f"{call_replacements!r}", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - # Check 10: extracted function is actually called - if not _check_failed and not _has_call_to(func_name, candidate): - _failures.append( - f"'{func_name}' not called in candidate output" - ) - if self.verbose: - print( - f"crispen: DuplicateExtractor: extraction FAILED — " - f"'{func_name}' not called in candidate output", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - # Check 11: no new undefined names - if not _check_failed: - undef = _pyflakes_new_undefined_names(source, candidate) - if undef: - _failures.append( - f"undefined name(s) introduced by edit: " - f"{', '.join(sorted(undef))}" - ) - if self.verbose: - print( - f"crispen: DuplicateExtractor:" - f" extraction FAILED — " - f"undefined name(s) introduced by edit: " - f"{', '.join(sorted(undef))}", - file=sys.stderr, - flush=True, - ) - _check_failed = True - - # Retry decision for algorithmic failures - if _check_failed: - if alg_retries_left > 0: - alg_retries_left -= 1 - prev_failures = _failures - prev_output = extraction - if self.verbose: - print( - f"crispen: DuplicateExtractor: → retrying" - f" extraction ({alg_retries_left} retries" - f" remaining after algorithmic failure)", - file=sys.stderr, - flush=True, - ) - continue - self.stats.algorithmic_rejected += 1 - break # exhausted algorithmic retries — skip group - - # ---- LLM verification step ---- - self.stats.llm_verify_calls += 1 - timing5: list = [] - try: - verify_ok, verify_issues = _run_with_timeout( - _llm_verify_extraction, - self._hard_timeout, - client, - group, - helper_source, - call_replacements, - source, - self._model, - self._provider, - tool_choice_override=self._tool_choice, - _timing_out=timing5, - rate_limit_retries=self._rate_limit_retries, - rate_limit_backoff=self._rate_limit_backoff, - ) - if timing5: - lr = timing5[0] - self.stats.record_llm_call( - lr.elapsed, - lr.input_tokens, - lr.output_tokens, - "verify", - "duplicate_extractor", - self.current_file, - ) - except _ApiTimeout: - if self.verbose: - print( - "crispen: DuplicateExtractor: → verify timed out," - " accepting extraction", - file=sys.stderr, - flush=True, - ) - verify_ok, verify_issues = True, [] - - if self.verbose: - v_status = "ACCEPTED" if verify_ok else "REJECTED" - timing_suffix = "" - if self.timing == "detailed" and timing5: - lr = timing5[0] - timing_suffix = ( - f" [{lr.elapsed:.2f}s," - f" {lr.input_tokens:,} in / {lr.output_tokens:,} out]" - ) - print( - f"crispen: DuplicateExtractor: → verify {v_status}" - f"{timing_suffix}", - file=sys.stderr, - flush=True, - ) - if not verify_ok: - for issue in verify_issues: - print( - f"crispen: DuplicateExtractor:" f" issue: {issue}", - file=sys.stderr, - flush=True, - ) - - if not verify_ok: - if llm_verify_retries_left > 0: - llm_verify_retries_left -= 1 - prev_failures = [ - f"LLM verification issue: {i}" for i in verify_issues - ] - prev_output = extraction - if self.verbose: - print( - f"crispen: DuplicateExtractor: → retrying" - f" extraction after verify rejection" - f" ({llm_verify_retries_left} retries remaining)", - file=sys.stderr, - flush=True, - ) - continue - self.stats.llm_rejected += 1 - break # exhausted LLM verify retries — skip group - - # ---- All checks passed: accept this extraction ---- - used_names.add(func_name) - if self.verbose: - print( - f"crispen: DuplicateExtractor: extracting '{func_name}'", - file=sys.stderr, - flush=True, - ) - extraction_groups.append( - ( - func_name, - group_edits, - f"DuplicateExtractor: extracted '{func_name}' " - f"from {len(group)} duplicate blocks", - ) - ) - break # done with this group - - # 18. Combine all accepted edits, verify all extracted functions are - # actually called in the combined output, then write. - all_edits = list(edits) - for _, g_edits, _ in extraction_groups: - all_edits.extend(g_edits) - - if all_edits: - combined = _apply_edits(source, all_edits) - - # Drop any extraction group whose extracted function is not called - # in the combined output. This happens when call-site edits are - # silently skipped by the overlap detector because they conflict - # with edits from another group or from the func-match pass. - uncalled = { - name - for name, _, _ in extraction_groups - if not _has_call_to(name, combined) - } - if uncalled: - for name in sorted(uncalled): - if self.verbose: - print( - f"crispen: DuplicateExtractor: extraction DROPPED — " - f"'{name}' not called in combined output " - f"(call-site edits overridden by overlapping edits)", - file=sys.stderr, - flush=True, - ) - extraction_groups = [ - (n, g, m) for n, g, m in extraction_groups if n not in uncalled - ] - all_edits = list(edits) - for _, g_edits, _ in extraction_groups: - all_edits.extend(g_edits) - combined = _apply_edits(source, all_edits) - - # Drop any extraction group whose helper function is not defined in - # the combined output. This happens when two groups insert helpers - # before the same scope: _build_helper_insertion absorbs surrounding - # blank lines into a replacement edit, so the second group's helper - # insertion is silently skipped by the overlap detector — leaving a - # call to the helper but no definition. - undefined_helpers = { - name - for name, _, _ in extraction_groups - if not _has_funcdef(name, combined) - } - if undefined_helpers: - for name in sorted(undefined_helpers): - if self.verbose: - print( - f"crispen: DuplicateExtractor: extraction DROPPED — " - f"'{name}' not defined in combined output " - f"(helper insertion blocked by overlapping edit)", - file=sys.stderr, - flush=True, - ) - extraction_groups = [ - (n, g, m) - for n, g, m in extraction_groups - if n not in undefined_helpers - ] - all_edits = list(edits) - for _, g_edits, _ in extraction_groups: - all_edits.extend(g_edits) - combined = _apply_edits(source, all_edits) - - all_pending = list(pending_changes) - for _, _, msg in extraction_groups: - all_pending.append(msg) - - if all_edits: - _extracted_names = _names_in_edit_texts(extraction_groups) - combined = _pyflakes_strip_unused_simple_assigns( - combined, _extracted_names - ) - self._new_source = _lift_and_dedup_imports(combined) - self.changes_made.extend(all_pending) - - def get_rewritten_source(self) -> Optional[str]: - return self._new_source diff --git a/crispen/refactors/duplicate_extractor/__init__.py b/crispen/refactors/duplicate_extractor/__init__.py new file mode 100644 index 0000000..5d1c8ec --- /dev/null +++ b/crispen/refactors/duplicate_extractor/__init__.py @@ -0,0 +1,58 @@ +"""Refactor: extract duplicate code blocks into helper functions using an LLM.""" + +from __future__ import annotations +from .collectors import _FunctionCollector # fmt: skip # noqa: F401, E501 +from .collectors import _FunctionInfo # fmt: skip # noqa: F401, E501 +from .collectors import _SeqInfo # fmt: skip # noqa: F401, E501 +from .collectors import _SequenceCollector # fmt: skip # noqa: F401, E501 +from .collectors import _build_function_body_fps # fmt: skip # noqa: F401, E501 +from .collectors import _collect_called_names # fmt: skip # noqa: F401, E501 +from .collectors import _filter_maximal_groups # fmt: skip # noqa: F401, E501 +from .collectors import _find_duplicate_groups # fmt: skip # noqa: F401, E501 +from .collectors import _has_internal_overlap # fmt: skip # noqa: F401, E501 +from .collectors import _overlaps_diff # fmt: skip # noqa: F401, E501 +from .collectors import _seq_source_contains_yield # fmt: skip # noqa: F401, E501 +from .llm_integration import _generate_no_arg_call # fmt: skip # noqa: F401, E501 +from .llm_integration import _llm_extract # fmt: skip # noqa: F401, E501 +from .llm_integration import _llm_generate_call # fmt: skip # noqa: F401, E501 +from .llm_integration import _llm_verify_extraction # fmt: skip # noqa: F401, E501 +from .llm_integration import _llm_veto # fmt: skip # noqa: F401, E501 +from .llm_integration import _llm_veto_func_match # fmt: skip # noqa: F401, E501 +from .text_editing import _apply_edits # fmt: skip # noqa: F401, E501 +from .text_editing import _build_helper_insertion # fmt: skip # noqa: F401, E501 +from .text_editing import _find_insertion_point # fmt: skip # noqa: F401, E501 +from .text_editing import _skip_class_docstring # fmt: skip # noqa: F401, E501 +from .utils import _ApiTimeout # fmt: skip # noqa: F401, E501 +from .utils import _has_def # fmt: skip # noqa: F401, E501 +from .utils import _node_weight # fmt: skip # noqa: F401, E501 +from .utils import _normalize_source # fmt: skip # noqa: F401, E501 +from .utils import _run_with_timeout # fmt: skip # noqa: F401, E501 +from .utils import _sequence_weight # fmt: skip # noqa: F401, E501 +from .utils import _strip_helper_docstring # fmt: skip # noqa: F401, E501 +from .verification import _collect_ast_store_names # fmt: skip # noqa: F401, E501 +from .verification import _collect_attribute_names # fmt: skip # noqa: F401, E501 +from .verification import _collect_called_attr_names # fmt: skip # noqa: F401, E501 +from .verification import _extract_defined_names # fmt: skip # noqa: F401, E501 +from .verification import _find_escaping_vars # fmt: skip # noqa: F401, E501 +from .verification import _has_call_to # fmt: skip # noqa: F401, E501 +from .verification import _has_funcdef # fmt: skip # noqa: F401, E501 +from .verification import _has_mutable_literal_is_check # fmt: skip # noqa: F401, E501 +from .verification import _has_param_overwritten_before_read # fmt: skip # noqa: F401, E501 +from .verification import _helper_imports_local_name # fmt: skip # noqa: F401, E501 +from .verification import _is_pure_literal # fmt: skip # noqa: F401, E501 +from .verification import _lift_and_dedup_imports # fmt: skip # noqa: F401, E501 +from .verification import _missing_free_vars # fmt: skip # noqa: F401, E501 +from .verification import _names_assigned_in # fmt: skip # noqa: F401, E501 +from .verification import _names_in_edit_texts # fmt: skip # noqa: F401, E501 +from .verification import _normalize_replacement_indentation # fmt: skip # noqa: F401, E501 +from .verification import _pyflakes_new_undefined_names # fmt: skip # noqa: F401, E501 +from .verification import _pyflakes_strip_unused_simple_assigns # fmt: skip # noqa: F401, E501 +from .verification import _replace_unused_in_target # fmt: skip # noqa: F401, E501 +from .verification import _replacement_contains_return # fmt: skip # noqa: F401, E501 +from .verification import _replacement_steals_post_block_line # fmt: skip # noqa: F401, E501 +from .verification import _scope_end_line # fmt: skip # noqa: F401, E501 +from .verification import _seq_ends_with_return # fmt: skip # noqa: F401, E501 +from .verification import _strip_unused_call_assignments # fmt: skip # noqa: F401, E501 +from .verification import _verify_extraction # fmt: skip # noqa: F401, E501 +from .verification import _would_create_proxy_wrappers # fmt: skip # noqa: F401, E501 +from .extractor import DuplicateExtractor # fmt: skip # noqa: F401, E501 diff --git a/crispen/refactors/duplicate_extractor/collectors.py b/crispen/refactors/duplicate_extractor/collectors.py new file mode 100644 index 0000000..23bc595 --- /dev/null +++ b/crispen/refactors/duplicate_extractor/collectors.py @@ -0,0 +1,312 @@ +from __future__ import annotations +from dataclasses import dataclass +from typing import Dict, List, Optional, Sequence, Tuple +import ast +import textwrap +from libcst.metadata import PositionProvider +import libcst as cst +from .utils import ( + _MAX_SEQ_LEN, + _MIN_WEIGHT, + _has_def, + _normalize_source, + _sequence_weight, +) + + +@dataclass +class _SeqInfo: + stmts: List[cst.BaseStatement] + start_line: int + end_line: int + scope: str + source: str + fingerprint: str + class_scope: Optional[str] = None # enclosing class name, or None if module-level + + +@dataclass +class _FunctionInfo: + name: str + source: str # raw source of complete function definition + scope: str # "" or enclosing class name + body_source: str # raw source of the function body (indented) + body_stmt_count: int # number of top-level statements in the body + params: List[str] # positional parameter names (empty → no-arg function) + + +def _seq_source_contains_yield(source: str) -> bool: + """Return True if *source* contains ``yield`` or ``yield from`` outside + any nested function definition. + + Sequences with a yield cannot be safely extracted into a plain helper + function: extraction would make the helper a generator, forcing call sites + to iterate via ``for``/``async for`` instead of calling it directly. This + is a semantic transformation (e.g. ``async with X as c: yield c`` → + ``async for c in helper(): yield c``) that the extractor must not attempt. + """ + wrapped = "def _f():\n" + textwrap.indent(textwrap.dedent(source), " ") + try: + tree = ast.parse(wrapped) + except SyntaxError: + return False + if not tree.body or not isinstance( + tree.body[0], ast.FunctionDef + ): # pragma: no cover + return False + + def _walk(nodes): + for node in nodes: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue # don't cross into nested scope + if isinstance(node, (ast.Yield, ast.YieldFrom)): + return True + if _walk(ast.iter_child_nodes(node)): + return True + return False + + return _walk(tree.body[0].body) + + +class _SequenceCollector(cst.CSTVisitor): + METADATA_DEPENDENCIES = (PositionProvider,) + + def __init__( + self, + source_lines: List[str], + max_seq_len: int = _MAX_SEQ_LEN, + min_weight: int = _MIN_WEIGHT, + ) -> None: + self.sequences: List[_SeqInfo] = [] + self._scope_stack: List[str] = [""] + self._class_stack: List[str] = [] + self._source_lines = source_lines + self._max_seq_len = max_seq_len + self._min_weight = min_weight + + def visit_FunctionDef(self, node: cst.FunctionDef) -> Optional[bool]: + self._scope_stack.append(node.name.value) + return None + + def leave_FunctionDef(self, node: cst.FunctionDef) -> None: + self._scope_stack.pop() + + def visit_ClassDef(self, node: cst.ClassDef) -> Optional[bool]: + self._scope_stack.append(node.name.value) + self._class_stack.append(node.name.value) + return None + + def leave_ClassDef(self, node: cst.ClassDef) -> None: + self._scope_stack.pop() + self._class_stack.pop() + + def _process_body(self, body: Sequence) -> None: + stmt_info: List[Tuple[cst.BaseStatement, int, int]] = [] + for stmt in body: + try: + pos = self.get_metadata(PositionProvider, stmt) + stmt_info.append((stmt, pos.start.line, pos.end.line)) + except KeyError: # pragma: no cover + continue + + n = len(stmt_info) + scope = self._scope_stack[-1] + class_scope = self._class_stack[-1] if self._class_stack else None + for start_i in range(n): + for end_i in range( + start_i + 1, min(start_i + self._max_seq_len + 1, n + 1) + ): + window: List[cst.BaseStatement] = [ + s[0] for s in stmt_info[start_i:end_i] + ] + if _has_def(window): + continue + if _sequence_weight(window) < self._min_weight: + continue + start_line = stmt_info[start_i][1] + end_line = stmt_info[end_i - 1][2] + seq_source = "".join(self._source_lines[start_line - 1 : end_line]) + if _seq_source_contains_yield(seq_source): + continue + self.sequences.append( + _SeqInfo( + stmts=window, + start_line=start_line, + end_line=end_line, + scope=scope, + source=seq_source, + fingerprint=_normalize_source(seq_source), + class_scope=class_scope, + ) + ) + + def visit_Module(self, node: cst.Module) -> Optional[bool]: + self._process_body(node.body) + return None + + def visit_IndentedBlock(self, node: cst.IndentedBlock) -> Optional[bool]: + self._process_body(node.body) + return None + + +class _FunctionCollector(cst.CSTVisitor): + METADATA_DEPENDENCIES = (PositionProvider,) + + def __init__(self, source_lines: List[str]) -> None: + self.functions: List[_FunctionInfo] = [] + self._scope_stack: List[str] = [""] + self._scope_kind_stack: List[str] = ["module"] + self._source_lines = source_lines + + def visit_FunctionDef(self, node: cst.FunctionDef) -> Optional[bool]: + parent_kind = self._scope_kind_stack[-1] + if parent_kind in ("module", "class"): + try: + pos = self.get_metadata(PositionProvider, node) + func_source = "".join( + self._source_lines[pos.start.line - 1 : pos.end.line] + ) + body_pos = self.get_metadata(PositionProvider, node.body) + body_source = "".join( + self._source_lines[body_pos.start.line - 1 : body_pos.end.line] + ) + except KeyError: # pragma: no cover + func_source = "" + body_source = "" + body_stmt_count = len(node.body.body) + params = [p.name.value for p in node.params.params] + self.functions.append( + _FunctionInfo( + name=node.name.value, + source=func_source, + scope=self._scope_stack[-1], + body_source=body_source, + body_stmt_count=body_stmt_count, + params=params, + ) + ) + self._scope_stack.append(node.name.value) + self._scope_kind_stack.append("function") + return None + + def leave_FunctionDef(self, node: cst.FunctionDef) -> None: + self._scope_stack.pop() + self._scope_kind_stack.pop() + + def visit_ClassDef(self, node: cst.ClassDef) -> Optional[bool]: + self._scope_stack.append(node.name.value) + self._scope_kind_stack.append("class") + return None + + def leave_ClassDef(self, node: cst.ClassDef) -> None: + self._scope_stack.pop() + self._scope_kind_stack.pop() + + +def _collect_called_names(source: str) -> set: + """Return a set of all names called (as functions) in *source*. + + Uses ast.parse + ast.walk to find all ast.Call nodes. Returns the + called name: func.id for ast.Name callees, func.attr for ast.Attribute + callees. On SyntaxError, returns an empty set. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + names: set = set() + for node in ast.walk(tree): + if isinstance(node, ast.Call): + if isinstance(node.func, ast.Name): + names.add(node.func.id) + elif isinstance(node.func, ast.Attribute): + names.add(node.func.attr) + return names + + +def _build_function_body_fps( + all_functions: List[_FunctionInfo], + called_names: set, +) -> Dict[str, _FunctionInfo]: + """Map normalized body fingerprint → _FunctionInfo for called functions. + + Only functions whose name appears in *called_names* are indexed, since + only those could be the target of a "replace with existing function" edit. + """ + fps: Dict[str, _FunctionInfo] = {} + for func in all_functions: + if func.name in called_names: + fp = _normalize_source(func.body_source) + fps[fp] = func + return fps + + +def _overlaps_diff(seq: _SeqInfo, changed_ranges: List[Tuple[int, int]]) -> bool: + return any( + seq.start_line <= r_end and seq.end_line >= r_start + for r_start, r_end in changed_ranges + ) + + +def _filter_maximal_groups(groups: List[List[_SeqInfo]]) -> List[List[_SeqInfo]]: + """Return only maximal groups, discarding those overlapping a larger group. + + Groups are sorted by their longest sequence (descending) and greedily selected: + a group is kept only if none of its sequences overlap an already-claimed line range. + This prevents multiple helpers being extracted for overlapping spans, where the + smaller extractions would end up unused after the larger one is applied. + """ + sorted_groups = sorted( + groups, + key=lambda g: max(s.end_line - s.start_line for s in g), + reverse=True, + ) + claimed: List[Tuple[int, int]] = [] + result = [] + for group in sorted_groups: + overlaps = any( + seq.start_line <= c_end and seq.end_line >= c_start + for seq in group + for c_start, c_end in claimed + ) + if not overlaps: + result.append(group) + for seq in group: + claimed.append((seq.start_line, seq.end_line)) + return result + + +def _has_internal_overlap(seqs: List[_SeqInfo]) -> bool: + """Return True if any two sequences in the group overlap each other. + + Overlapping sequences within a group indicate sequential repetition + (e.g. [A,B] and [B,C] both matching) rather than true duplication at + distinct call sites. Extracting a helper from such a group would leave + part of the original pattern unreplaced. + """ + sorted_seqs = sorted(seqs, key=lambda s: s.start_line) + for i in range(len(sorted_seqs) - 1): + if sorted_seqs[i].end_line >= sorted_seqs[i + 1].start_line: + return True + return False + + +def _find_duplicate_groups( + sequences: List[_SeqInfo], + changed_ranges: List[Tuple[int, int]], + max_groups: int = 5, +) -> List[List[_SeqInfo]]: + by_fp: Dict[str, List[_SeqInfo]] = {} + for seq in sequences: + by_fp.setdefault(seq.fingerprint, []).append(seq) + groups = [] + for seqs in by_fp.values(): + if len(seqs) < 2: + continue + if not any(_overlaps_diff(s, changed_ranges) for s in seqs): + continue + if _has_internal_overlap(seqs): + continue + groups.append(seqs) + groups = _filter_maximal_groups(groups) + return groups[:max_groups] diff --git a/crispen/refactors/duplicate_extractor/extractor.py b/crispen/refactors/duplicate_extractor/extractor.py new file mode 100644 index 0000000..53af025 --- /dev/null +++ b/crispen/refactors/duplicate_extractor/extractor.py @@ -0,0 +1,1073 @@ +from __future__ import annotations +from typing import List, Optional, Tuple +import re +import sys +import textwrap +from libcst.metadata import MetadataWrapper +import libcst as cst +from ... import llm_client as _llm_client +from ..base import Refactor +from .collectors import ( + _FunctionCollector, + _SequenceCollector, + _build_function_body_fps, + _collect_called_names, + _find_duplicate_groups, + _overlaps_diff, +) +from .llm_integration import ( + _generate_no_arg_call, + _llm_extract, + _llm_generate_call, + _llm_verify_extraction, + _llm_veto, + _llm_veto_func_match, +) +from .text_editing import ( + _apply_edits, + _build_helper_insertion, + _find_insertion_point, + _skip_class_docstring, +) +from .utils import ( + _ApiTimeout, + _MAX_SEQ_LEN, + _MIN_WEIGHT, + _MODEL, + _run_with_timeout, + _strip_helper_docstring, +) +from .verification import ( + _collect_called_attr_names, + _extract_defined_names, + _find_escaping_vars, + _has_call_to, + _has_funcdef, + _helper_imports_local_name, + _lift_and_dedup_imports, + _missing_free_vars, + _names_in_edit_texts, + _normalize_replacement_indentation, + _pyflakes_new_undefined_names, + _pyflakes_strip_unused_simple_assigns, + _replacement_contains_return, + _replacement_steals_post_block_line, + _scope_end_line, + _seq_ends_with_return, + _strip_unused_call_assignments, + _verify_extraction, + _would_create_proxy_wrappers, +) + + +class DuplicateExtractor(Refactor): + """Detect and extract duplicate code blocks into helper functions via LLM.""" + + def __init__( + self, + changed_ranges: List[Tuple[int, int]], + source: str = "", + verbose: bool = True, + min_weight: int = _MIN_WEIGHT, + max_seq_len: int = _MAX_SEQ_LEN, + model: str = _MODEL, + helper_docstrings: bool = False, + provider: str = "anthropic", + extraction_retries: int = 1, + llm_verify_retries: int = 1, + base_url: Optional[str] = None, + tool_choice: Optional[str] = None, + api_timeout: float = 60.0, + match_functions: bool = True, + timing: str = "detailed", + current_file: str = "", + rate_limit_retries: int = 6, + rate_limit_backoff: float = 20.0, + ) -> None: + super().__init__(changed_ranges, source=source, verbose=verbose) + self.current_file = current_file + self.timing = timing + self._min_weight = min_weight + self._base_max_seq_len = max_seq_len + self._model = model + self._helper_docstrings = helper_docstrings + self._provider = provider + self._extraction_retries = extraction_retries + self._llm_verify_retries = llm_verify_retries + self._base_url = base_url + self._tool_choice = tool_choice + self._api_timeout = api_timeout + self._hard_timeout = api_timeout + 30 + self._match_functions = match_functions + self._rate_limit_retries = rate_limit_retries + self._rate_limit_backoff = rate_limit_backoff + self._new_source: Optional[str] = None + if source: + self._analyze(source) + + def _analyze(self, source: str) -> None: + # 1. Parse tree; early-return on syntax error. + try: + tree = cst.parse_module(source) + except cst.ParserSyntaxError: + return + + # 2. Source lines. + source_lines = source.splitlines(keepends=True) + + # 3. Collect functions. + func_collector = _FunctionCollector(source_lines) + MetadataWrapper(tree).visit(func_collector) + all_functions = func_collector.functions + + # 4-5. Build function body fingerprint map (only for called functions). + called_names = _collect_called_names(source) + func_body_fps = _build_function_body_fps(all_functions, called_names) + + # 6. Compute max sequence length to capture full function bodies. + max_seq_len = max( + max(f.body_stmt_count for f in all_functions) if all_functions else 0, + self._base_max_seq_len, + ) + + # 7. Collect sequences. + collector = _SequenceCollector( + source_lines, max_seq_len=max_seq_len, min_weight=self._min_weight + ) + MetadataWrapper(tree).visit(collector) + + # 8. Preliminary duplicate groups. + groups = _find_duplicate_groups(collector.sequences, self.changed_ranges) + + # 9. Check whether any sequence can be replaced with an existing function. + has_func_matches = ( + self._match_functions + and func_body_fps + and any( + _overlaps_diff(seq, self.changed_ranges) + and seq.fingerprint in func_body_fps + and func_body_fps[seq.fingerprint].name != seq.scope + for seq in collector.sequences + ) + ) + + # 10. Early exit — nothing to do. + if not has_func_matches and not groups: + return + + # 12. Create API client. + api_key = _llm_client.get_api_key(self._provider, caller="DuplicateExtractor") + client = _llm_client.make_client( + self._provider, api_key, timeout=self._api_timeout, base_url=self._base_url + ) + edits: List[Tuple[int, int, str]] = [] + pending_changes: List[str] = [] + # Extraction groups tracked separately so the final combined check can + # drop any whose call-site edits were silently overridden by overlapping + # edits from another group or the func-match pass. + extraction_groups: List[Tuple[str, List[Tuple[int, int, str]], str]] = [] + matched_line_ranges: set = set() + + # 14. Function body match pass. + if self._match_functions and func_body_fps: + for seq in collector.sequences: + if not _overlaps_diff(seq, self.changed_ranges): + continue + if seq.fingerprint not in func_body_fps: + continue + func = func_body_fps[seq.fingerprint] + if func.name == seq.scope: + continue + if self.verbose: + print( + f"crispen: DuplicateExtractor: func-match check — " + f"scope '{seq.scope}': lines {seq.start_line}-{seq.end_line}" + f" → '{func.name}'", + file=sys.stderr, + flush=True, + ) + self.stats.llm_veto_calls += 1 + timing: list = [] + try: + is_valid, reason, _veto_notes = _run_with_timeout( + _llm_veto_func_match, + self._hard_timeout, + client, + seq, + func, + source, + self._model, + self._provider, + tool_choice_override=self._tool_choice, + _timing_out=timing, + rate_limit_retries=self._rate_limit_retries, + rate_limit_backoff=self._rate_limit_backoff, + ) + if timing: + lr = timing[0] + self.stats.record_llm_call( + lr.elapsed, + lr.input_tokens, + lr.output_tokens, + "veto", + "duplicate_extractor", + self.current_file, + ) + except _ApiTimeout: + print( + "crispen: DuplicateExtractor: → func-match veto timed out", + file=sys.stderr, + flush=True, + ) + continue + if self.verbose: + status = "ACCEPTED" if is_valid else "VETOED" + timing_suffix = "" + if self.timing == "detailed" and timing: + lr = timing[0] + timing_suffix = ( + f" [{lr.elapsed:.2f}s," + f" {lr.input_tokens:,} in / {lr.output_tokens:,} out]" + ) + print( + f"crispen: DuplicateExtractor: → {status}: {reason}" + f"{timing_suffix}", + file=sys.stderr, + flush=True, + ) + if not is_valid: + self.stats.llm_rejected += 1 + continue + timing2: list = [] + if func.scope == "" and not func.params: + replacement = _generate_no_arg_call(seq, func) + else: + self.stats.llm_edit_calls += 1 + try: + replacement = _run_with_timeout( + _llm_generate_call, + self._hard_timeout, + client, + seq, + func, + source, + self._model, + self._provider, + tool_choice_override=self._tool_choice, + _timing_out=timing2, + rate_limit_retries=self._rate_limit_retries, + rate_limit_backoff=self._rate_limit_backoff, + ) + if timing2: + lr = timing2[0] + self.stats.record_llm_call( + lr.elapsed, + lr.input_tokens, + lr.output_tokens, + "edit", + "duplicate_extractor", + self.current_file, + ) + except _ApiTimeout: + print( + "crispen: DuplicateExtractor:" + " → call generation timed out", + file=sys.stderr, + flush=True, + ) + continue + if replacement is None: + continue # pragma: no cover + if not _verify_extraction(None, [replacement]): + continue + if self.verbose: + timing_suffix = "" + if self.timing == "detailed" and timing2: + lr = timing2[0] + timing_suffix = ( + f" [{lr.elapsed:.2f}s," + f" {lr.input_tokens:,} in / {lr.output_tokens:,} out]" + ) + print( + f"crispen: DuplicateExtractor: → replacing '{seq.scope}'" + f" with '{func.name}()'{timing_suffix}", + file=sys.stderr, + flush=True, + ) + edits.append((seq.start_line - 1, seq.end_line, replacement)) + matched_line_ranges.add((seq.start_line, seq.end_line)) + pending_changes.append( + f"DuplicateExtractor: replaced '{seq.scope}' body" + f" with call to '{func.name}'" + ) + + # 15. Recompute duplicate groups excluding matched sequences. + if matched_line_ranges: + remaining = [ + s + for s in collector.sequences + if not any( + s.start_line <= r_end and s.end_line >= r_start + for r_start, r_end in matched_line_ranges + ) + ] + groups = _find_duplicate_groups(remaining, self.changed_ranges) + + # 16. Log group count. + if groups and self.verbose: + print( + f"crispen: DuplicateExtractor: found {len(groups)} duplicate group(s)", + file=sys.stderr, + flush=True, + ) + + # 17. Duplicate group extraction pass. + used_names = _extract_defined_names(source) + for group in groups: + # Compute escaping vars algorithmically before any LLM call so the + # extraction prompt can instruct the LLM to return them. + escaping_vars = frozenset(_find_escaping_vars(group, source_lines)) + + # Skip groups that would leave a function as a trivial proxy wrapper + # (i.e. the extracted block is the function's entire body). + if _would_create_proxy_wrappers(group, all_functions): + if self.verbose: + print( + "crispen: DuplicateExtractor: skipping group — " + "extraction would leave a trivial proxy wrapper", + file=sys.stderr, + flush=True, + ) + continue + + if self.verbose: + ranges_str = ", ".join( + f"lines {s.start_line}-{s.end_line}" for s in group + ) + print( + f"crispen: DuplicateExtractor: veto check — " + f"scope '{group[0].scope}': {ranges_str}", + file=sys.stderr, + flush=True, + ) + self.stats.llm_veto_calls += 1 + timing3: list = [] + try: + is_valid, reason, veto_notes = _run_with_timeout( + _llm_veto, + self._hard_timeout, + client, + group, + self._model, + self._provider, + tool_choice_override=self._tool_choice, + _timing_out=timing3, + rate_limit_retries=self._rate_limit_retries, + rate_limit_backoff=self._rate_limit_backoff, + ) + if timing3: + lr = timing3[0] + self.stats.record_llm_call( + lr.elapsed, + lr.input_tokens, + lr.output_tokens, + "veto", + "duplicate_extractor", + self.current_file, + ) + except _ApiTimeout: + print( + "crispen: DuplicateExtractor: API call timed out, skipping group", + file=sys.stderr, + flush=True, + ) + continue + if self.verbose: + status = "ACCEPTED" if is_valid else "VETOED" + timing_suffix = "" + if self.timing == "detailed" and timing3: + lr = timing3[0] + timing_suffix = ( + f" [{lr.elapsed:.2f}s," + f" {lr.input_tokens:,} in / {lr.output_tokens:,} out]" + ) + print( + f"crispen: DuplicateExtractor: → {status}: {reason}" + f"{timing_suffix}", + file=sys.stderr, + flush=True, + ) + if not is_valid: + self.stats.llm_rejected += 1 + continue + + # Extraction retry loop: attempt extraction up to + # 1 + _extraction_retries times on algorithmic failure, and up to + # 1 + _llm_verify_retries additional times on LLM verify failure. + alg_retries_left = self._extraction_retries + llm_verify_retries_left = self._llm_verify_retries + prev_failures: List[str] = [] + prev_output: Optional[dict] = None + + while True: + self.stats.llm_edit_calls += 1 + timing4: list = [] + try: + extraction = _run_with_timeout( + _llm_extract, + self._hard_timeout, + client, + group, + source, + escaping_vars, + used_names=frozenset(used_names), + model=self._model, + helper_docstrings=self._helper_docstrings, + provider=self._provider, + veto_notes=veto_notes, + prev_failures=prev_failures, + prev_output=prev_output, + tool_choice_override=self._tool_choice, + _timing_out=timing4, + rate_limit_retries=self._rate_limit_retries, + rate_limit_backoff=self._rate_limit_backoff, + ) + if timing4: + lr = timing4[0] + self.stats.record_llm_call( + lr.elapsed, + lr.input_tokens, + lr.output_tokens, + "edit", + "duplicate_extractor", + self.current_file, + ) + except _ApiTimeout: + print( + "crispen: DuplicateExtractor: API call timed out," + " skipping group", + file=sys.stderr, + flush=True, + ) + break + if extraction is None: + break # pragma: no cover + + if self.verbose and self.timing == "detailed" and timing4: + lr = timing4[0] + print( + f"crispen: DuplicateExtractor: → extraction" + f" [{lr.elapsed:.2f}s," + f" {lr.input_tokens:,} in / {lr.output_tokens:,} out]", + file=sys.stderr, + flush=True, + ) + + helper_source = extraction["helper_source"] + if not self._helper_docstrings: + helper_source = _strip_helper_docstring(helper_source) + call_replacements = extraction["call_site_replacements"] + placement = extraction.get("placement", "module_level") + # Auto-indent 0-indent helpers for staticmethod: placement. + # The LLM sometimes writes a module-level def even when it + # selects staticmethod:ClassName. Inserting 0-indent code + # inside the class body ends the class silently and makes all + # subsequent methods nested inside the helper — valid syntax + # but semantically broken, so compile() does not catch it. + if placement.startswith("staticmethod:") and helper_source: + first_code = next( + (ln for ln in helper_source.splitlines() if ln.strip()), "" + ) + if first_code and not first_code[0].isspace(): + helper_source = textwrap.indent(helper_source, " ") + func_name = extraction["function_name"] + + # Helpers are always file-internal; enforce a leading underscore. + if not func_name.startswith("_"): + _old_name = func_name + func_name = "_" + func_name + _rename_pat = re.compile(r"\b" + re.escape(_old_name) + r"\b") + helper_source = _rename_pat.sub(func_name, helper_source) + call_replacements = [ + _rename_pat.sub(func_name, r) for r in call_replacements + ] + + _check_failed = False + _failures: List[str] = [] + + # Check 1: name collision + # Pre-check: placement consistency with call-site class scopes. + if placement.startswith("staticmethod:"): + group_class_scopes = {s.class_scope for s in group} + if len(group_class_scopes) != 1 or None in group_class_scopes: + _failures.append( + "staticmethod placement is invalid when call sites span " + "multiple classes or scopes; use module_level instead" + ) + if self.verbose: + print( + "crispen: DuplicateExtractor: extraction FAILED — " + "staticmethod placement invalid for cross-class group", + file=sys.stderr, + flush=True, + ) + _check_failed = True + elif placement.split(":", 1)[1] != next(iter(group_class_scopes)): + named_class = placement.split(":", 1)[1] + actual_class = next(iter(group_class_scopes)) + _failures.append( + f"staticmethod names class '{named_class}' but all call " + f"sites are in '{actual_class}'; use " + f"'staticmethod:{actual_class}' instead" + ) + if self.verbose: + print( + f"crispen: DuplicateExtractor: extraction FAILED — " + f"staticmethod names wrong class '{named_class}' " + f"(actual: '{actual_class}')", + file=sys.stderr, + flush=True, + ) + _check_failed = True + if placement == "module_level": + # Reject if any call site invokes the helper as an instance + # method (self.(...)) — that is inconsistent with + # module-level placement and will fail at runtime. + _self_call_pat = re.compile(rf"\bself\.{re.escape(func_name)}\s*\(") + if any(_self_call_pat.search(r) for r in call_replacements): + group_class_scopes = {s.class_scope for s in group} + if ( + len(group_class_scopes) == 1 + and None not in group_class_scopes + ): + only_class = next(iter(group_class_scopes)) + placement_hint = f"use 'staticmethod:{only_class}' instead" + else: + placement_hint = ( + f"change call sites to call " + f"'{func_name}(...)' directly" + ) + _failures.append( + f"module_level placement is inconsistent with call " + f"sites that invoke the helper as " + f"'self.{func_name}(...)'; {placement_hint}" + ) + if self.verbose: + print( + f"crispen: DuplicateExtractor: extraction FAILED" + f" — module_level placement conflicts with " + f"self.{func_name}() call sites", + file=sys.stderr, + flush=True, + ) + _check_failed = True + if func_name in used_names: + _failures.append( + f"name collision: '{func_name}' is already defined," + " choose a different name" + ) + if self.verbose: + print( + f"crispen: DuplicateExtractor: extraction FAILED — " + f"name collision: '{func_name}' is already defined", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + # Check 2: call site count + if not _check_failed and len(call_replacements) != len(group): + _failures.append( + f"wrong call_site_replacements count" + f" (expected {len(group)}, got {len(call_replacements)})" + ) + if self.verbose: + print( + f"crispen: DuplicateExtractor: extraction FAILED — " + f"wrong call_site_replacements count " + f"(expected {len(group)}, got {len(call_replacements)})", + file=sys.stderr, + flush=True, + ) + print( + f"crispen: DuplicateExtractor: helper_source: " + f"{helper_source!r}", + file=sys.stderr, + flush=True, + ) + print( + f"crispen: DuplicateExtractor: call_site_replacements: " + f"{call_replacements!r}", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + if not _check_failed: + # Normalize each replacement's indentation to match its + # original block. The LLM sometimes returns replacements at + # column 0; this re-indents them so the assembled edit is + # valid Python. + call_replacements = [ + _normalize_replacement_indentation(seq, r) + for seq, r in zip(group, call_replacements) + ] + + # Strip unused variable assignments from call-site + # replacements. The LLM may assign return values that are + # never used after the block (e.g. when the helper returns a + # value only needed at some call sites), which would produce + # flake8 F841 warnings. + call_replacements = [ + _strip_unused_call_assignments( + r, + source_lines[ + seq.end_line : _scope_end_line( + source_lines, seq.scope, seq.end_line + ) + ], + ) + for seq, r in zip(group, call_replacements) + ] + + # Check 3: post-block line theft + if _replacement_steals_post_block_line( + group, call_replacements, source_lines + ): + _failures.append( + "replacement duplicates the line after the block" + ) + if self.verbose: + print( + "crispen: DuplicateExtractor: extraction FAILED — " + "replacement duplicates the line after the block", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + # Check 4: syntax validation + if not _check_failed and not _verify_extraction( + helper_source, call_replacements + ): + _failures.append("invalid helper or replacement syntax") + if self.verbose: + print( + "crispen: DuplicateExtractor: extraction FAILED — " + "invalid helper or replacement syntax", + file=sys.stderr, + flush=True, + ) + print( + f"crispen: DuplicateExtractor: helper_source: " + f"{helper_source!r}", + file=sys.stderr, + flush=True, + ) + print( + f"crispen: DuplicateExtractor: call_site_replacements: " + f"{call_replacements!r}", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + # Check 5: return statement consistency + if not _check_failed and any( + _seq_ends_with_return(seq) + and not _replacement_contains_return(repl) + for seq, repl in zip(group, call_replacements) + ): + _failures.append("block ends with return but replacement omits it") + if self.verbose: + print( + "crispen: DuplicateExtractor: extraction FAILED — " + "block ends with return but replacement omits it", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + # Check 6: helper must not import local names + if not _check_failed and _helper_imports_local_name( + helper_source, source + ): + _failures.append( + "helper imports a name that is a parameter/local" + " in the original file" + ) + if self.verbose: + print( + "crispen: DuplicateExtractor: extraction FAILED — " + "helper imports a name that is a parameter/local " + "in the original file", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + # Check 7: new attribute access + if not _check_failed: + new_attrs = _collect_called_attr_names( + textwrap.dedent(helper_source) + ) - _collect_called_attr_names(source) + if new_attrs: + _failures.append( + f"helper introduces new attribute access(es) not in" + f" original: {', '.join(sorted(new_attrs))}" + ) + if self.verbose: + print( + f"crispen: DuplicateExtractor: extraction FAILED — " + f"helper introduces new attribute access(es) not in" + f" original: {', '.join(sorted(new_attrs))}", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + # Check 8: free variable preservation + if not _check_failed: + seq0 = group[0] + block_src = "".join( + source_lines[seq0.start_line - 1 : seq0.end_line] + ) + missing = _missing_free_vars( + block_src, call_replacements, helper_source, source + ) + if missing: + _failures.append( + f"free variable(s) from original block missing in" + f" replacement: {', '.join(sorted(missing))}" + ) + if self.verbose: + print( + f"crispen: DuplicateExtractor: extraction FAILED — " + f"free variable(s) from original block missing in " + f"replacement: {', '.join(sorted(missing))}", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + # Build this group's edits (only if pre-edit checks passed). + group_edits: List[Tuple[int, int, str]] = [] + candidate = "" + if not _check_failed: + for seq, replacement in zip(group, call_replacements): + group_edits.append( + (seq.start_line - 1, seq.end_line, replacement) + ) + first_seq = min(group, key=lambda s: s.start_line) + if placement.startswith("staticmethod:"): + # Insert inside the class body, after "class Foo:" and + # any class docstring (which must remain first). + scope = placement.split(":", 1)[1] + class_line = _find_insertion_point(source, scope) + insert_pos = _skip_class_docstring(source_lines, class_line + 1) + else: + scope = first_seq.scope + insert_pos = _find_insertion_point(source, scope) + group_edits.append( + _build_helper_insertion( + source_lines, insert_pos, helper_source, placement + ) + ) + # Compile the per-group candidate independently so one bad + # extraction doesn't discard valid ones for the same file. + candidate = _apply_edits(source, group_edits) + + # Re-strip unused variable assignments using the assembled + # candidate's following lines. The initial pass (above) + # used the original source, which can incorrectly retain an + # assignment when another call site's original block + # referenced the same name. Re-running with candidate + # following lines also handles partial-tuple targets + # (``a, _ = helper()``) the same way the initial pass does. + cand_lines = candidate.splitlines(keepends=True) + restripped = [] + for seq, repl in zip(group, call_replacements): + cs0 = seq.start_line - 1 + offset = sum( + len(et.splitlines(keepends=True)) - (ee - es) + for (es, ee, et) in group_edits + if es < cs0 + ) + new_end = cs0 + offset + len(repl.splitlines(keepends=True)) + scope_end = _scope_end_line(cand_lines, seq.scope, new_end) + restripped.append( + _strip_unused_call_assignments( + repl, cand_lines[new_end:scope_end] + ) + ) + if restripped != call_replacements: + call_replacements = restripped + group_edits = [ + (seq.start_line - 1, seq.end_line, r) + for seq, r in zip(group, call_replacements) + ] + group_edits.append( + _build_helper_insertion( + source_lines, insert_pos, helper_source, placement + ) + ) + candidate = _apply_edits(source, group_edits) + + # Check 9: assembled output is valid Python + try: + compile(candidate, "", "exec") + except SyntaxError as exc: + _failures.append(f"assembled edit not valid Python: {exc}") + if self.verbose: + print( + f"crispen: DuplicateExtractor: extraction FAILED — " + f"assembled edit not valid Python: {exc}", + file=sys.stderr, + flush=True, + ) + print( + f"crispen: DuplicateExtractor: helper_source: " + f"{helper_source!r}", + file=sys.stderr, + flush=True, + ) + print( + f"crispen: DuplicateExtractor:" + f" call_site_replacements: " + f"{call_replacements!r}", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + # Check 10: extracted function is actually called + if not _check_failed and not _has_call_to(func_name, candidate): + _failures.append( + f"'{func_name}' not called in candidate output" + ) + if self.verbose: + print( + f"crispen: DuplicateExtractor: extraction FAILED — " + f"'{func_name}' not called in candidate output", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + # Check 11: no new undefined names + if not _check_failed: + undef = _pyflakes_new_undefined_names(source, candidate) + if undef: + _failures.append( + f"undefined name(s) introduced by edit: " + f"{', '.join(sorted(undef))}" + ) + if self.verbose: + print( + f"crispen: DuplicateExtractor:" + f" extraction FAILED — " + f"undefined name(s) introduced by edit: " + f"{', '.join(sorted(undef))}", + file=sys.stderr, + flush=True, + ) + _check_failed = True + + # Retry decision for algorithmic failures + if _check_failed: + if alg_retries_left > 0: + alg_retries_left -= 1 + prev_failures = _failures + prev_output = extraction + if self.verbose: + print( + f"crispen: DuplicateExtractor: → retrying" + f" extraction ({alg_retries_left} retries" + f" remaining after algorithmic failure)", + file=sys.stderr, + flush=True, + ) + continue + self.stats.algorithmic_rejected += 1 + break # exhausted algorithmic retries — skip group + + # ---- LLM verification step ---- + self.stats.llm_verify_calls += 1 + timing5: list = [] + try: + verify_ok, verify_issues = _run_with_timeout( + _llm_verify_extraction, + self._hard_timeout, + client, + group, + helper_source, + call_replacements, + source, + self._model, + self._provider, + tool_choice_override=self._tool_choice, + _timing_out=timing5, + rate_limit_retries=self._rate_limit_retries, + rate_limit_backoff=self._rate_limit_backoff, + ) + if timing5: + lr = timing5[0] + self.stats.record_llm_call( + lr.elapsed, + lr.input_tokens, + lr.output_tokens, + "verify", + "duplicate_extractor", + self.current_file, + ) + except _ApiTimeout: + if self.verbose: + print( + "crispen: DuplicateExtractor: → verify timed out," + " accepting extraction", + file=sys.stderr, + flush=True, + ) + verify_ok, verify_issues = True, [] + + if self.verbose: + v_status = "ACCEPTED" if verify_ok else "REJECTED" + timing_suffix = "" + if self.timing == "detailed" and timing5: + lr = timing5[0] + timing_suffix = ( + f" [{lr.elapsed:.2f}s," + f" {lr.input_tokens:,} in / {lr.output_tokens:,} out]" + ) + print( + f"crispen: DuplicateExtractor: → verify {v_status}" + f"{timing_suffix}", + file=sys.stderr, + flush=True, + ) + if not verify_ok: + for issue in verify_issues: + print( + f"crispen: DuplicateExtractor:" f" issue: {issue}", + file=sys.stderr, + flush=True, + ) + + if not verify_ok: + if llm_verify_retries_left > 0: + llm_verify_retries_left -= 1 + prev_failures = [ + f"LLM verification issue: {i}" for i in verify_issues + ] + prev_output = extraction + if self.verbose: + print( + f"crispen: DuplicateExtractor: → retrying" + f" extraction after verify rejection" + f" ({llm_verify_retries_left} retries remaining)", + file=sys.stderr, + flush=True, + ) + continue + self.stats.llm_rejected += 1 + break # exhausted LLM verify retries — skip group + + # ---- All checks passed: accept this extraction ---- + used_names.add(func_name) + if self.verbose: + print( + f"crispen: DuplicateExtractor: extracting '{func_name}'", + file=sys.stderr, + flush=True, + ) + extraction_groups.append( + ( + func_name, + group_edits, + f"DuplicateExtractor: extracted '{func_name}' " + f"from {len(group)} duplicate blocks", + ) + ) + break # done with this group + + # 18. Combine all accepted edits, verify all extracted functions are + # actually called in the combined output, then write. + all_edits = list(edits) + for _, g_edits, _ in extraction_groups: + all_edits.extend(g_edits) + + if all_edits: + combined = _apply_edits(source, all_edits) + + # Drop any extraction group whose extracted function is not called + # in the combined output. This happens when call-site edits are + # silently skipped by the overlap detector because they conflict + # with edits from another group or from the func-match pass. + uncalled = { + name + for name, _, _ in extraction_groups + if not _has_call_to(name, combined) + } + if uncalled: + for name in sorted(uncalled): + if self.verbose: + print( + f"crispen: DuplicateExtractor: extraction DROPPED — " + f"'{name}' not called in combined output " + f"(call-site edits overridden by overlapping edits)", + file=sys.stderr, + flush=True, + ) + extraction_groups = [ + (n, g, m) for n, g, m in extraction_groups if n not in uncalled + ] + all_edits = list(edits) + for _, g_edits, _ in extraction_groups: + all_edits.extend(g_edits) + combined = _apply_edits(source, all_edits) + + # Drop any extraction group whose helper function is not defined in + # the combined output. This happens when two groups insert helpers + # before the same scope: _build_helper_insertion absorbs surrounding + # blank lines into a replacement edit, so the second group's helper + # insertion is silently skipped by the overlap detector — leaving a + # call to the helper but no definition. + undefined_helpers = { + name + for name, _, _ in extraction_groups + if not _has_funcdef(name, combined) + } + if undefined_helpers: + for name in sorted(undefined_helpers): + if self.verbose: + print( + f"crispen: DuplicateExtractor: extraction DROPPED — " + f"'{name}' not defined in combined output " + f"(helper insertion blocked by overlapping edit)", + file=sys.stderr, + flush=True, + ) + extraction_groups = [ + (n, g, m) + for n, g, m in extraction_groups + if n not in undefined_helpers + ] + all_edits = list(edits) + for _, g_edits, _ in extraction_groups: + all_edits.extend(g_edits) + combined = _apply_edits(source, all_edits) + + all_pending = list(pending_changes) + for _, _, msg in extraction_groups: + all_pending.append(msg) + + if all_edits: + _extracted_names = _names_in_edit_texts(extraction_groups) + combined = _pyflakes_strip_unused_simple_assigns( + combined, _extracted_names + ) + self._new_source = _lift_and_dedup_imports(combined) + self.changes_made.extend(all_pending) + + def get_rewritten_source(self) -> Optional[str]: + return self._new_source diff --git a/crispen/refactors/duplicate_extractor/llm_integration.py b/crispen/refactors/duplicate_extractor/llm_integration.py new file mode 100644 index 0000000..385d118 --- /dev/null +++ b/crispen/refactors/duplicate_extractor/llm_integration.py @@ -0,0 +1,493 @@ +from __future__ import annotations +from typing import List, Optional, Tuple +from ... import llm_client as _llm_client +from .collectors import _FunctionInfo, _SeqInfo +from .utils import _MODEL + + +_VETO_TOOL: dict = { + "name": "evaluate_duplicate", + "description": ( + "Evaluate whether code blocks are semantic duplicates worth extracting" + ), + "input_schema": { + "type": "object", + "properties": { + "is_valid_duplicate": { + "type": "boolean", + "description": ( + "True if extracting a shared helper would improve clarity" + ), + }, + "reason": {"type": "string"}, + "extraction_notes": { + "type": "string", + "description": ( + "If accepting, note any potential pitfalls the extraction " + "step should watch out for — e.g., tricky variable scoping, " + "mutable arguments, subtle differences in variable names, or " + "return-value handling. Leave empty if none." + ), + }, + }, + "required": ["is_valid_duplicate", "reason"], + }, +} + +_VERIFY_TOOL: dict = { + "name": "verify_extraction", + "description": "Verify that an extracted helper function is semantically correct", + "input_schema": { + "type": "object", + "properties": { + "is_correct": { + "type": "boolean", + "description": ( + "True if the extraction is semantically equivalent to the originals" + ), + }, + "issues": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Specific issues found. Empty if correct. Each issue should " + "describe what is wrong and how the extraction should be fixed." + ), + }, + }, + "required": ["is_correct", "issues"], + }, +} + +_EXTRACT_TOOL: dict = { + "name": "extract_helper", + "description": "Extract duplicate code blocks into a helper function", + "input_schema": { + "type": "object", + "properties": { + "function_name": {"type": "string"}, + "placement": { + "type": "string", + "description": ( + "Where to place the helper: 'module_level' or " + "'staticmethod:ClassName'" + ), + }, + "helper_source": { + "type": "string", + "description": "Complete source of the helper function", + }, + "call_site_replacements": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Replacement source for each duplicate block, " + "in the same order as the input blocks. " + "Each replacement must preserve the original block's " + "leading indentation and end with a trailing newline. " + "Cover only the exact lines of the specified block — " + "do not include any code from before or after the block." + ), + }, + }, + "required": [ + "function_name", + "placement", + "helper_source", + "call_site_replacements", + ], + }, +} + +_CALL_GEN_TOOL: dict = { + "name": "generate_call", + "description": "Generate a call to an existing function that replaces a code block", + "input_schema": { + "type": "object", + "properties": { + "replacement": { + "type": "string", + "description": ( + "Complete replacement source " + "(including indentation and trailing newline)" + ), + } + }, + "required": ["replacement"], + }, +} + + +def _llm_veto( + client, + group: List[_SeqInfo], + model: str = _MODEL, + provider: str = "anthropic", + tool_choice_override: Optional[str] = None, + _timing_out=None, + rate_limit_retries: int = 6, + rate_limit_backoff: float = 20.0, +) -> Tuple[bool, str, str]: + blocks_text = "\n\n".join( + f"Block {i + 1} (scope: {s.scope}, lines {s.start_line}-{s.end_line}):\n" + f"```python\n{s.source.rstrip()}\n```" + for i, s in enumerate(group) + ) + prompt = ( + f"Here are {len(group)} structurally similar code blocks from the same " + f"Python file:\n\n{blocks_text}\n\n" + "Do these blocks represent the same semantic operation such that extracting " + "a shared helper function would improve clarity? Or are they coincidentally " + "similar but conceptually distinct?\n\n" + "If you accept (is_valid_duplicate=True), also fill in extraction_notes " + "with any potential pitfalls the extraction step should watch out for — " + "e.g., tricky variable scoping, mutable arguments, subtle differences in " + "variable names between blocks, or return-value handling edge cases." + ) + result = _llm_client.call_with_tool( + client, + provider, + model, + 384, + _VETO_TOOL, + "evaluate_duplicate", + [{"role": "user", "content": prompt}], + caller="DuplicateExtractor", + tool_choice_override=tool_choice_override, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff=rate_limit_backoff, + ) + if _timing_out is not None: + _timing_out.append(result) + if result.tool_input is not None: + return ( + result.tool_input["is_valid_duplicate"], + result.tool_input.get("reason", ""), + result.tool_input.get("extraction_notes", ""), + ) + return False, "no tool response", "" # pragma: no cover + + +def _llm_extract( + client, + group: List[_SeqInfo], + full_source: str, + escaping_vars: frozenset = frozenset(), + used_names: frozenset = frozenset(), + model: str = _MODEL, + helper_docstrings: bool = True, + provider: str = "anthropic", + veto_notes: str = "", + prev_failures: List[str] = [], + prev_output: Optional[dict] = None, + tool_choice_override: Optional[str] = None, + _timing_out=None, + rate_limit_retries: int = 6, + rate_limit_backoff: float = 20.0, +) -> Optional[dict]: + src_lines = full_source.splitlines(keepends=True) + block_entries = [] + for i, s in enumerate(group): + entry = ( + f"Block {i + 1} (scope: {s.scope}, lines {s.start_line}-{s.end_line}):\n" + f"```python\n{s.source.rstrip()}\n```" + ) + next_idx = s.end_line # 0-based index of the first line after the block + if next_idx < len(src_lines): + next_line = src_lines[next_idx].rstrip() + if next_line.strip(): + entry += ( + f"\nLine immediately after this block" + f" (must NOT appear in the replacement): `{next_line}`" + ) + block_entries.append(entry) + blocks_text = "\n\n".join(block_entries) + snippet = full_source[:4000] if len(full_source) > 4000 else full_source + escaping_note = "" + if escaping_vars: + vars_str = ", ".join(sorted(escaping_vars)) + escaping_note = ( + f"\n\nThe following variables are assigned within the duplicate block " + f"and referenced by code that immediately follows the block at one or " + f"more call sites: {vars_str}. The helper function must return these " + f"variables. At call sites where the return value is needed, capture it; " + f"at call sites where it is not needed, discard the return value." + ) + used_names_note = "" + if used_names: + names_str = ", ".join(sorted(used_names)) + used_names_note = ( + f"\n\nThe following function names are already defined in this file " + f"or reserved by a previous extraction: {names_str}. " + f"Do not use any of these names for the helper function." + ) + docstring_note = ( + "" + if helper_docstrings + else "\n\nDo not include a docstring in the helper function." + ) + veto_notes_note = "" + if veto_notes: + veto_notes_note = ( + f"\n\nNotes from code review (watch out for these pitfalls): " + f"{veto_notes[:500]}" + ) + failures_note = "" + if prev_failures: + failures_str = "\n".join(f"- {f}" for f in prev_failures) + prior_helper = prev_output.get("helper_source", "") + prior_repls = prev_output.get("call_site_replacements", []) + repls_text = "\n".join(f" [{i + 1}] {r!r}" for i, r in enumerate(prior_repls)) + failures_note = ( + f"\n\nThe previous extraction attempt produced:\n\n" + f"helper_source:\n```python\n{prior_helper}```\n\n" + f"call_site_replacements:\n{repls_text}\n\n" + f"But failed these checks:\n{failures_str}\n\n" + f"Please correct these issues in your new attempt." + ) + class_scopes = {s.class_scope for s in group} + all_same_class = len(class_scopes) == 1 and None not in class_scopes + if all_same_class: + same_class_name = next(iter(class_scopes)) + staticmethod_instruction = ( + f"All call sites are inside class '{same_class_name}'. " + f"You MUST use placement 'staticmethod:{same_class_name}'. " + ) + else: + staticmethod_instruction = ( + "Use module_level placement — call sites span different classes or scopes. " + ) + prompt = ( + "Extract the following duplicate code blocks from this Python file into a " + f"helper function.\n\nFile source:\n```python\n{snippet}\n```\n\n" + f"Duplicate blocks:\n{blocks_text}\n\n" + "Place the helper immediately before the enclosing function of its first use. " + f"{staticmethod_instruction}" + "Return complete, valid Python for the helper and each call site replacement. " + "Each call site replacement must start with the same leading indentation as " + "the block it replaces, end with a trailing newline, and cover only the exact " + "lines of the duplicate block — stopping before the 'Line immediately after " + "this block' marker shown above. Do not include any code from before or after " + "the block. " + "Double-check that only required parameters are passed to the helper — do not " + "include an unused parameter, or one that is overwritten before being read. " + "Be mindful of the code being removed from the call site: if variable " + "assignments are moved into the helper, those variables may no longer be " + "defined in the calling scope at that point. " + "If the helper uses a sentinel return value to signal an error path (such as " + "returning an empty collection), check for it at the call site with `==`, not " + "`is` — `is` only gives correct results for singletons like `None`, `True`, " + "and `False`, not for constructed objects like `set()`." + f"{escaping_note}" + f"{used_names_note}" + f"{docstring_note}" + f"{veto_notes_note}" + f"{failures_note}" + ) + result = _llm_client.call_with_tool( + client, + provider, + model, + 1024, + _EXTRACT_TOOL, + "extract_helper", + [{"role": "user", "content": prompt}], + caller="DuplicateExtractor", + tool_choice_override=tool_choice_override, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff=rate_limit_backoff, + ) + if _timing_out is not None: + _timing_out.append(result) + return result.tool_input + + +def _llm_veto_func_match( + client, + seq: _SeqInfo, + func: _FunctionInfo, + full_source: str, + model: str = _MODEL, + provider: str = "anthropic", + tool_choice_override: Optional[str] = None, + _timing_out=None, + rate_limit_retries: int = 6, + rate_limit_backoff: float = 20.0, +) -> Tuple[bool, str, str]: + """Ask the LLM whether *seq* performs the same operation as *func*'s body.""" + snippet = full_source[:4000] if len(full_source) > 4000 else full_source + prompt = ( + "A code block in a Python file may be replaceable by a call to an existing " + "function.\n\n" + f"Code block (scope: {seq.scope}, lines {seq.start_line}-{seq.end_line}):\n" + f"```python\n{seq.source.rstrip()}\n```\n\n" + f"Existing function '{func.name}':\n" + f"```python\n{func.source.rstrip()}\n```\n\n" + f"File source:\n```python\n{snippet}\n```\n\n" + "Does this code block perform the same semantic operation as the function " + "body, such that it could be replaced by a call to the function? " + "Use the evaluate_duplicate tool to answer." + ) + result = _llm_client.call_with_tool( + client, + provider, + model, + 256, + _VETO_TOOL, + "evaluate_duplicate", + [{"role": "user", "content": prompt}], + caller="DuplicateExtractor", + tool_choice_override=tool_choice_override, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff=rate_limit_backoff, + ) + if _timing_out is not None: + _timing_out.append(result) + if result.tool_input is not None: + return ( + result.tool_input["is_valid_duplicate"], + result.tool_input.get("reason", ""), + result.tool_input.get("extraction_notes", ""), + ) + return False, "no tool response", "" # pragma: no cover + + +def _generate_no_arg_call(seq: _SeqInfo, func: _FunctionInfo) -> str: + """Algorithmically generate a no-argument call to *func*, preserving indentation.""" + first_line = seq.source.splitlines()[0] + indent = first_line[: len(first_line) - len(first_line.lstrip())] + return indent + func.name + "()\n" + + +def _llm_generate_call( + client, + seq: _SeqInfo, + func: _FunctionInfo, + full_source: str, + model: str = _MODEL, + provider: str = "anthropic", + tool_choice_override: Optional[str] = None, + _timing_out=None, + rate_limit_retries: int = 6, + rate_limit_backoff: float = 20.0, +) -> Optional[str]: + """Ask the LLM to generate a call expression replacing *seq* with *func*.""" + snippet = full_source[:4000] if len(full_source) > 4000 else full_source + prompt = ( + f"Replace this code block with a call to the existing function" + f" '{func.name}'.\n\n" + f"Code block (scope: {seq.scope}, lines {seq.start_line}-{seq.end_line}):\n" + f"```python\n{seq.source.rstrip()}\n```\n\n" + f"Function '{func.name}':\n" + f"```python\n{func.source.rstrip()}\n```\n\n" + f"File source:\n```python\n{snippet}\n```\n\n" + "Generate a replacement that preserves the original indentation and ends " + "with a newline. Pass the replacement to the generate_call tool." + ) + result = _llm_client.call_with_tool( + client, + provider, + model, + 256, + _CALL_GEN_TOOL, + "generate_call", + [{"role": "user", "content": prompt}], + caller="DuplicateExtractor", + tool_choice_override=tool_choice_override, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff=rate_limit_backoff, + ) + if _timing_out is not None: + _timing_out.append(result) + if result.tool_input is not None: + return result.tool_input["replacement"] + return None # pragma: no cover + + +def _llm_verify_extraction( + client, + group: List[_SeqInfo], + helper_source: str, + call_replacements: List[str], + full_source: str, + model: str = _MODEL, + provider: str = "anthropic", + tool_choice_override: Optional[str] = None, + _timing_out=None, + rate_limit_retries: int = 6, + rate_limit_backoff: float = 20.0, +) -> Tuple[bool, List[str]]: + """Ask the LLM to verify the extraction is semantically correct. + + Returns ``(is_correct, issues)`` where *issues* is a list of specific + problems found. Returns ``(True, [])`` if the call times out or the LLM + cannot respond, so a verification failure never silently blocks commits. + """ + blocks_text = "\n\n".join( + f"Original block {i + 1} (scope: {s.scope}, " + f"lines {s.start_line}-{s.end_line}):\n" + f"```python\n{s.source.rstrip()}\n```" + for i, s in enumerate(group) + ) + replacements_text = "\n\n".join( + f"Replacement for block {i + 1}:\n```python\n{r.rstrip()}\n```" + for i, r in enumerate(call_replacements) + ) + src_lines = full_source.splitlines(keepends=True) + min_start = min(s.start_line for s in group) + max_end = max(s.end_line for s in group) + window_start = max(0, min_start - 30) + window_end = min(len(src_lines), max_end + 100) + snippet = "".join(src_lines[window_start:window_end]) + prompt = ( + "Verify that the following helper function extraction is semantically " + "correct by tracing through the code carefully.\n\n" + f"Original duplicate blocks:\n{blocks_text}\n\n" + f"Extracted helper:\n```python\n{helper_source.rstrip()}\n```\n\n" + f"Call site replacements:\n{replacements_text}\n\n" + f"Source context around duplicate blocks " + f"(lines {window_start + 1}–{window_end}):\n```python\n{snippet}\n```\n\n" + "Check each of the following:\n" + "1. Every variable read (but not locally assigned) in the original block " + "is passed as a parameter to the helper\n" + "2. Every variable assigned in the original block and used afterward is " + "returned by the helper and captured at the call site\n" + "3. No parameter is assigned before it is first read in the helper body\n" + "4. If the original block ends with a non-None return, the call site " + "replacement also propagates that return value\n" + "5. The call site replacements match the original indentation and cover " + "exactly the lines of the original block\n" + "6. If the helper is called more than once with different arguments, verify " + "each call site against the exact local variables that appeared in the " + "original code at that location — not merely variables of the same type. " + "Same-type variables (e.g. two dicts, two strings) that are both in scope " + "are a swap risk: confirm neither was substituted for the other across call " + "sites.\n" + "7. No line from the helper body is duplicated verbatim in the call site " + "replacement. If setup lines were extracted into the helper, they must not " + "also appear before or after the call — otherwise the extraction is wrong.\n" + "8. Does the function name clearly and accurately describe what the body " + "does? Flag the name if it is misleading, too generic, or omits a crucial " + "detail — for example, an important side-effect that the name gives no hint " + "of (e.g. a function named 'compute_total' that also writes to a database).\n" + "If correct, set is_correct=True and issues=[]. " + "Otherwise set is_correct=False and list each specific issue." + ) + result = _llm_client.call_with_tool( + client, + provider, + model, + 512, + _VERIFY_TOOL, + "verify_extraction", + [{"role": "user", "content": prompt}], + caller="DuplicateExtractor", + tool_choice_override=tool_choice_override, + rate_limit_retries=rate_limit_retries, + rate_limit_backoff=rate_limit_backoff, + ) + if _timing_out is not None: + _timing_out.append(result) + if result.tool_input is None: + return True, [] # pragma: no cover + return result.tool_input["is_correct"], result.tool_input.get("issues", []) diff --git a/crispen/refactors/duplicate_extractor/text_editing.py b/crispen/refactors/duplicate_extractor/text_editing.py new file mode 100644 index 0000000..cf17d32 --- /dev/null +++ b/crispen/refactors/duplicate_extractor/text_editing.py @@ -0,0 +1,174 @@ +from __future__ import annotations +from typing import List, Tuple +import re + + +def _build_helper_insertion( + source_lines: List[str], + insert_pos: int, + helper_source: str, + placement: str, +) -> Tuple[int, int, str]: + """Build an edit tuple that inserts helper_source with correct surrounding blanks. + + Always returns a pure insertion (start == end) so that two groups inserting + before the same scope are never in conflict: pure insertions are not subject + to the overlap-skip logic in _apply_edits. + + The insertion point is placed after all blank lines that already exist + around insert_pos (right before the def/decorator line). Leading blank + lines are prepended only to make up the difference so the result always + has exactly ``blank_lines`` blank lines before the helper. + """ + blank_lines = 1 if placement.startswith("staticmethod:") else 2 + + # Count consecutive blank lines immediately before insert_pos. + before_blanks = 0 + i = insert_pos - 1 + while i >= 0 and not source_lines[i].strip(): + before_blanks += 1 + i -= 1 + + # Count consecutive blank lines at and immediately after insert_pos. + after_blanks = 0 + i = insert_pos + while i < len(source_lines) and not source_lines[i].strip(): + after_blanks += 1 + i += 1 + + # Insert right before the def/decorator (after all surrounding blanks). + insert_at = insert_pos + after_blanks + # Prepend only as many blank lines as are still missing. + leading = max(0, blank_lines - (before_blanks + after_blanks)) + clean = helper_source.strip("\n") + "\n" + text = "\n" * leading + clean + "\n" * blank_lines + return (insert_at, insert_at, text) + + +def _apply_edits(source: str, edits: List[Tuple[int, int, str]]) -> str: + """Apply (start_0, end_0, text) edits bottom-to-top. + + Indices are 0-based; lines[start_0:end_0] is replaced with text. + An insertion before line N uses start_0 == end_0 == N. + Overlapping replacement ranges are skipped. + """ + lines = source.splitlines(keepends=True) + if lines and not lines[-1].endswith("\n"): + lines[-1] += "\n" + + applied: List[Tuple[int, int]] = [] + for start, end, text in sorted(edits, key=lambda e: (e[0], e[1]), reverse=True): + is_insertion = start == end + if not is_insertion: + if any(a_start < end and a_end > start for a_start, a_end in applied): + continue + applied.append((start, end)) + new_lines = text.splitlines(keepends=True) + if new_lines and not new_lines[-1].endswith("\n"): + new_lines[-1] += "\n" + lines[start:end] = new_lines + + return "".join(lines) + + +def _skip_class_docstring(source_lines: List[str], after_class_line: int) -> int: + """Return the 0-based line index after the class docstring, if any. + + Given the line immediately after ``class Foo:`` (or its colon line), + advance past any leading blank lines and then past a string-literal + docstring (single- or triple-quoted). If no docstring is present, + returns ``after_class_line`` unchanged. + """ + i = after_class_line + n = len(source_lines) + # Skip blank lines inside the class body. + while i < n and not source_lines[i].strip(): + i += 1 + if i >= n: + return after_class_line + stripped = source_lines[i].lstrip() + # Check for a triple-quoted docstring. + for q in ('"""', "'''"): + if stripped.startswith(q): + # Check whether the closing quote is on the same line (after the + # opening). + rest = stripped[len(q) :] + if q in rest: + # Single-line triple-quoted docstring. + return i + 1 + # Multi-line: scan forward for the closing triple-quote. + i += 1 + while i < n: + if q in source_lines[i]: + return i + 1 + i += 1 + return i # malformed, best-effort + # Single-quoted docstring (rare but valid). + for q in ('"', "'"): + if stripped.startswith(q) and not stripped.startswith(q * 2): + return i + 1 + return after_class_line + + +def _find_insertion_point(source: str, scope: str) -> int: + """Return 0-based line index to insert before. + + For module scope, inserts after the last import. + For a named scope, inserts before the def/class line. + + If the named scope resolves to an indented ``def`` (i.e. a class method), + inserting a module-level helper immediately before it would end the class + definition prematurely — the remaining class methods would be silently + re-parsed as nested functions of the helper, producing valid-syntax but + broken code that ``compile()`` does not catch. In that case we walk + backwards to the enclosing class definition and insert before it instead. + """ + source_lines = source.splitlines() + if scope == "": + last_import = -1 + for i, line in enumerate(source_lines): + stripped = line.strip() + if stripped.startswith("import ") or stripped.startswith("from "): + last_import = i + return last_import + 1 + + pattern = re.compile(rf"^\s*(?:async\s+def|def|class)\s+{re.escape(scope)}\s*[\(:]") + for i, line in enumerate(source_lines): + if pattern.match(line): + method_indent = len(line) - len(line.lstrip()) + if method_indent > 0: + # The def is inside a class body. Walk backwards to find the + # enclosing class definition and insert before that instead. + # If the first lower-indent non-blank line is NOT a class + # definition (i.e. the def is a nested function inside a + # regular function), stop immediately so we don't mis-identify + # an unrelated class above the outer function as the enclosing + # class. + for j in range(i - 1, -1, -1): + prev = source_lines[j] + if not prev.strip(): + continue + prev_indent = len(prev) - len(prev.lstrip()) + if prev_indent < method_indent: + if re.match(r"\s*class\s+\w+", prev): + return j + break # nested function — fall through to decorator walk + # Walk backwards over any preceding decorator lines (including + # multi-line decorator arguments) so the helper is inserted + # before the decorator block, not between decorators and the def. + j = i - 1 + paren_depth = 0 + while j >= 0: + stripped = source_lines[j].strip() + if not stripped: + break + for ch in stripped: + if ch == ")": + paren_depth += 1 + elif ch == "(": + paren_depth -= 1 + if paren_depth == 0 and not stripped.startswith("@"): + break + j -= 1 + return j + 1 + return 0 diff --git a/crispen/refactors/duplicate_extractor/utils.py b/crispen/refactors/duplicate_extractor/utils.py new file mode 100644 index 0000000..c70248b --- /dev/null +++ b/crispen/refactors/duplicate_extractor/utils.py @@ -0,0 +1,140 @@ +from __future__ import annotations +from typing import Dict, List +import ast +import textwrap +import threading +import libcst as cst + + +_MODEL = "claude-sonnet-4-6" +_MIN_WEIGHT = 3 +_MAX_SEQ_LEN = 8 + + +def _strip_helper_docstring(helper_source: str) -> str: + """Remove the docstring from helper_source if the first function has one.""" + try: + tree = cst.parse_module(textwrap.dedent(helper_source)) + except cst.ParserSyntaxError: + return helper_source + + if not tree.body or not isinstance(tree.body[0], cst.FunctionDef): + return helper_source + + func = tree.body[0] + body = func.body + if not isinstance(body, cst.IndentedBlock) or not body.body: # pragma: no cover + return helper_source + + first = body.body[0] + if not ( + isinstance(first, cst.SimpleStatementLine) + and len(first.body) == 1 + and isinstance(first.body[0], cst.Expr) + and isinstance(first.body[0].value, (cst.SimpleString, cst.ConcatenatedString)) + ): + return helper_source + + rest = list(body.body[1:]) + if not rest: + return helper_source + + new_func = func.with_changes(body=body.with_changes(body=rest)) + return tree.with_changes(body=[new_func] + list(tree.body[1:])).code + + +class _ApiTimeout(Exception): + """Raised when an LLM API call exceeds the hard per-call timeout.""" + + +def _run_with_timeout(func, timeout, *args, **kwargs): + """Run *func* in a daemon thread; raise _ApiTimeout if it doesn't finish. + + This enforces a hard wall-clock limit that is not affected by OS-level + blocking (e.g. DNS resolution) which application-layer timeouts cannot + interrupt. + """ + result: list = [None] + exc: list = [None] + + def target(): + try: + result[0] = func(*args, **kwargs) + except BaseException as e: + exc[0] = e + + t = threading.Thread(target=target, daemon=True) + t.start() + t.join(timeout=timeout) + if t.is_alive(): + raise _ApiTimeout(f"API call exceeded {timeout}s hard limit") + if exc[0] is not None: + raise exc[0] + return result[0] + + +def _node_weight(node: cst.CSTNode) -> int: + """Recursive statement weight: count all semantic statement units.""" + if isinstance(node, cst.SimpleStatementLine): + return len(node.body) + if isinstance(node, cst.IndentedBlock): + return sum(_node_weight(s) for s in node.body) + if isinstance(node, cst.Else): + return _node_weight(node.body) + if isinstance(node, cst.Finally): + return _node_weight(node.body) + if isinstance(node, (cst.FunctionDef, cst.ClassDef)): + return 1 + if not isinstance(node, (cst.If, cst.For, cst.While, cst.Try, cst.With)): + return 0 + weight = 1 + _node_weight(node.body) + orelse = getattr(node, "orelse", None) + if orelse is not None: + weight += _node_weight(orelse) + finalbody = getattr(node, "finalbody", None) + if finalbody is not None: + weight += _node_weight(finalbody) + if isinstance(node, cst.Try): + for handler in node.handlers: + weight += _node_weight(handler.body) + return weight + + +def _sequence_weight(stmts: List[cst.BaseStatement]) -> int: + return sum(_node_weight(s) for s in stmts) + + +def _has_def(stmts: List[cst.BaseStatement]) -> bool: + """Return True if any top-level statement is a function or class definition.""" + return any(isinstance(s, (cst.FunctionDef, cst.ClassDef)) for s in stmts) + + +class _ASTNormalizer(ast.NodeTransformer): + """Replace assignment-target Names with positional placeholders.""" + + def __init__(self) -> None: + self._map: Dict[str, str] = {} + self._counter = 0 + + def _placeholder(self, name: str) -> str: + if name not in self._map: + self._map[name] = f"_v{self._counter}" + self._counter += 1 + return self._map[name] + + def visit_Name(self, node: ast.Name) -> ast.Name: + if isinstance(node.ctx, (ast.Store, ast.Load)): + return ast.Name(id=self._placeholder(node.id), ctx=node.ctx) + return node + + +def _normalize_source(source: str) -> str: + """Return a normalized fingerprint of source code.""" + try: + tree = ast.parse(textwrap.dedent(source)) + except SyntaxError: + return source + normalizer = _ASTNormalizer() + normalized = normalizer.visit(tree) + ast.fix_missing_locations(normalized) + return ast.unparse(normalized) diff --git a/crispen/refactors/duplicate_extractor/verification/__init__.py b/crispen/refactors/duplicate_extractor/verification/__init__.py new file mode 100644 index 0000000..6332c15 --- /dev/null +++ b/crispen/refactors/duplicate_extractor/verification/__init__.py @@ -0,0 +1,27 @@ +from __future__ import annotations +from .call_site import _lift_and_dedup_imports # fmt: skip # noqa: F401, E501 +from .call_site import _replace_unused_in_target # fmt: skip # noqa: F401, E501 +from .call_site import _replacement_contains_return # fmt: skip # noqa: F401, E501 +from .call_site import _replacement_steals_post_block_line # fmt: skip # noqa: F401, E501 +from .call_site import _strip_unused_call_assignments # fmt: skip # noqa: F401, E501 +from .scope_analysis import _find_escaping_vars # fmt: skip # noqa: F401, E501 +from .scope_analysis import _has_param_overwritten_before_read # fmt: skip # noqa: F401, E501 +from .scope_analysis import _helper_imports_local_name # fmt: skip # noqa: F401, E501 +from .scope_analysis import _missing_free_vars # fmt: skip # noqa: F401, E501 +from .scope_analysis import _names_assigned_in # fmt: skip # noqa: F401, E501 +from .scope_analysis import _scope_end_line # fmt: skip # noqa: F401, E501 +from .utils import _collect_ast_store_names # fmt: skip # noqa: F401, E501 +from .utils import _collect_attribute_names # fmt: skip # noqa: F401, E501 +from .utils import _collect_called_attr_names # fmt: skip # noqa: F401, E501 +from .utils import _extract_defined_names # fmt: skip # noqa: F401, E501 +from .utils import _has_call_to # fmt: skip # noqa: F401, E501 +from .utils import _has_funcdef # fmt: skip # noqa: F401, E501 +from .utils import _is_pure_literal # fmt: skip # noqa: F401, E501 +from .utils import _names_in_edit_texts # fmt: skip # noqa: F401, E501 +from .utils import _normalize_replacement_indentation # fmt: skip # noqa: F401, E501 +from .utils import _seq_ends_with_return # fmt: skip # noqa: F401, E501 +from .validation import _has_mutable_literal_is_check # fmt: skip # noqa: F401, E501 +from .validation import _pyflakes_new_undefined_names # fmt: skip # noqa: F401, E501 +from .validation import _pyflakes_strip_unused_simple_assigns # fmt: skip # noqa: F401, E501 +from .validation import _verify_extraction # fmt: skip # noqa: F401, E501 +from .validation import _would_create_proxy_wrappers # fmt: skip # noqa: F401, E501 diff --git a/crispen/refactors/duplicate_extractor/verification/call_site.py b/crispen/refactors/duplicate_extractor/verification/call_site.py new file mode 100644 index 0000000..5f9f656 --- /dev/null +++ b/crispen/refactors/duplicate_extractor/verification/call_site.py @@ -0,0 +1,344 @@ +from __future__ import annotations +from typing import Dict, List, Tuple +import ast +import re +import textwrap +from ....import_sort import _sort_imports_pep8 +from ..collectors import _SeqInfo +from .utils import _collect_ast_store_names + + +def _replace_unused_in_target( + target: ast.AST, following_src: str +) -> Tuple[ast.AST, bool, bool]: + """Replace unused Name nodes in *target* with ``_``. + + Returns ``(new_target, all_replaced, any_replaced)`` where: + - *all_replaced*: every name in the target was replaced (all unused). + - *any_replaced*: at least one name was replaced. + + Non-Name, non-Tuple/List targets (Attribute, Subscript, …) are treated as + *used* so we never accidentally strip an assignment we cannot analyse. + """ + if isinstance(target, ast.Name): + if re.search(r"\b" + re.escape(target.id) + r"\b", following_src): + return target, False, False # used → keep + return ast.Name(id="_", ctx=ast.Store()), True, True # unused → _ + if isinstance(target, (ast.Tuple, ast.List)): + new_elts: List[ast.AST] = [] + all_replaced = True + any_replaced = False + for elt in target.elts: + new_elt, elt_all, elt_any = _replace_unused_in_target(elt, following_src) + new_elts.append(new_elt) + if not elt_all: + all_replaced = False + if elt_any: + any_replaced = True + new_target = type(target)(elts=new_elts, ctx=ast.Store()) + return new_target, all_replaced, any_replaced + # Attribute, Subscript, Starred, etc. — treat as used. + return target, False, False + + +def _strip_unused_call_assignments(replacement: str, following_lines: List[str]) -> str: + """Clean up unused assignment targets in a call-site replacement. + + For each ``Assign`` node whose right-hand side is a ``Call``: + + * **Single target** — unused ``Name`` elements in the target are replaced + with ``_``. If every element is unused the whole assignment is dropped + and only the call expression is emitted. Example:: + + result = _helper(x) → _helper(x) + a, b = _helper(x) (b used) → a, _ = _helper(x) + + * **Chained assignment** (``a = b = call()``) — stripped to just the call + only when every name across every target is unused; otherwise left alone. + + Augmented (``+=``) and annotated assignments are never touched. Assignment + targets that are not plain names or tuples/lists (e.g. ``self.x``) are + treated as *used* so we never accidentally remove live assignments. + + This prevents flake8 F841 "local variable assigned but never used" errors + introduced by the extraction. + """ + following_src = "".join(following_lines) + try: + dedented = textwrap.dedent(replacement) + tree = ast.parse(dedented) + except SyntaxError: + return replacement + + # Build a list of (start_ln, end_ln, new_src) edits. new_src is the + # replacement text for that statement (without leading indentation). + edits: List[Tuple[int, int, str]] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + value_node = node.value + if isinstance(value_node, ast.Await) and isinstance(value_node.value, ast.Call): + pass # treat `result = await helper(...)` like `result = helper(...)` + elif not isinstance(value_node, ast.Call): + continue + + call_src = ast.unparse(value_node) + + if len(node.targets) == 1: + new_target, all_replaced, any_replaced = _replace_unused_in_target( + node.targets[0], following_src + ) + if all_replaced: + edits.append((node.lineno, node.end_lineno, call_src)) + elif any_replaced: + edits.append( + ( + node.lineno, + node.end_lineno, + ast.unparse(new_target) + " = " + call_src, + ) + ) + else: + # Chained assignment: strip only when every name is unused. + all_names: List[str] = [] + for t in node.targets: + _collect_ast_store_names(t, all_names) + if not all_names: + continue + if not any( + re.search(r"\b" + re.escape(n) + r"\b", following_src) + for n in all_names + ): + edits.append((node.lineno, node.end_lineno, call_src)) + + if not edits: + return replacement + + # Determine the leading indentation from the first non-empty line. + first_content = next((ln for ln in replacement.splitlines() if ln.strip()), "") + indent = first_content[: len(first_content) - len(first_content.lstrip())] + + # Apply edits in reverse line order so earlier indices stay valid. + dedented_lines = dedented.splitlines(keepends=True) + for start_ln, end_ln, new_src in sorted(edits, key=lambda x: x[0], reverse=True): + dedented_lines[start_ln - 1 : end_ln] = [new_src + "\n"] + + return textwrap.indent("".join(dedented_lines), indent) + + +def _replacement_contains_return(replacement: str) -> bool: + """Return True if *replacement* contains any return statement. + + Wraps the replacement in a dummy function before parsing so that + ``return`` statements — which are legal inside a function body — do not + cause false SyntaxError rejections. + """ + try: + wrapped = "def _check():\n" + textwrap.indent( + textwrap.dedent(replacement), " " + ) + tree = ast.parse(wrapped) + except SyntaxError: + return False + for node in ast.walk(tree): + if isinstance(node, ast.Return): + return True + return False + + +def _replacement_steals_post_block_line( + group: List[_SeqInfo], call_replacements: List[str], source_lines: List[str] +) -> bool: + """Return True if any replacement's last line duplicates the line after its block. + + The LLM occasionally appends the first statement *after* the replaced block + to the end of the replacement text. When applied, that statement then appears + twice in the assembled output: once inside the replacement and once as the + original untouched line. + """ + for seq, replacement in zip(group, call_replacements): + next_idx = seq.end_line # 0-based index of the first line after the block + # Scan forward past blank lines to find the first real post-block line. + while next_idx < len(source_lines) and not source_lines[next_idx].strip(): + next_idx += 1 + if next_idx >= len(source_lines): + continue + post_block = source_lines[next_idx].strip() + repl_lines = [ln.strip() for ln in replacement.splitlines() if ln.strip()] + if repl_lines and repl_lines[-1] == post_block: + return True + return False + + +def _lift_and_dedup_imports(source: str) -> str: + """Lift misplaced module-level imports to the import block and deduplicate. + + When a helper is inserted before a function that is not the first in the + file, its leading ``from X import Y`` lines land after the first + ``def``/``class``, violating PEP 8. When a helper re-imports names + already present at the top, flake8 reports F811. This function fixes both: + + 1. Collect every simple, unindented ``from X import …`` / ``import X`` + line from anywhere in the file. + 2. Merge names for the same module (deduplicate). + 3. Emit the merged set within the top-of-file import block (before the + first ``def``/``class``), removing all later occurrences. + + Only single-line imports without parentheses, backslash continuations, or + inline comments are handled. Indented imports (``if TYPE_CHECKING:``, + function-local lazy imports, etc.) and wildcard imports are left untouched. + """ + lines = source.splitlines(keepends=True) + n = len(lines) + + # ── pass 1: find the import block boundary ────────────────────────────── + # The import block ends at the first unindented def/class line. + first_funcdef_idx = n + for i, line in enumerate(lines): + if line[:1] in (" ", "\t"): + continue + if re.match(r"^(?:async\s+def|def|class)\s", line.strip()): + first_funcdef_idx = i + break + + # ── pass 2: collect simple unindented import lines ────────────────────── + _FROM_RE = re.compile(r"^from\s+(\S+)\s+import\s+([^(\\#]+)$") + _PLAIN_RE = re.compile(r"^import\s+(\S+)$") + + all_imports: List[Tuple[int, str]] = [] # (line_idx, stripped_text) + import_indices: set = set() + last_block_import_idx = -1 + + for i, line in enumerate(lines): + if line[:1] in (" ", "\t"): + continue + stripped = line.strip() + mf = _FROM_RE.match(stripped) + if mf: + names_str = mf.group(2).strip() + if not names_str or names_str == "*": + continue + names = [nm.strip() for nm in names_str.split(",") if nm.strip()] + if not names: + continue + all_imports.append((i, stripped)) + import_indices.add(i) + if i < first_funcdef_idx: + last_block_import_idx = i + continue + mp = _PLAIN_RE.match(stripped) + if mp: + all_imports.append((i, stripped)) + import_indices.add(i) + if i < first_funcdef_idx: + last_block_import_idx = i + + if not all_imports: + return source + + # ── pass 3: build merged import map (ordered by first appearance) ─────── + from_map: Dict[str, List[str]] = {} # module -> merged name list + from_order: List[str] = [] + plain_order: List[str] = [] + plain_seen: set = set() + + for _, text in all_imports: + mf = _FROM_RE.match(text) + if mf: + module = mf.group(1) + names = [nm.strip() for nm in mf.group(2).split(",") if nm.strip()] + if module not in from_map: + from_map[module] = list(names) + from_order.append(module) + else: + existing_set = set(from_map[module]) + for name in names: + if name not in existing_set: + from_map[module].append(name) + existing_set.add(name) + else: + # Must be a plain import — guaranteed by pass 2 filter. + module = _PLAIN_RE.match(text).group(1) # type: ignore[union-attr] + if module not in plain_seen: + plain_order.append(module) + plain_seen.add(module) + + # ── early exit if nothing to do ───────────────────────────────────────── + has_misplaced = any(i >= first_funcdef_idx for i, _ in all_imports) + from_counts: Dict[str, int] = {} + plain_counts: Dict[str, int] = {} + for _, text in all_imports: + mf = _FROM_RE.match(text) + if mf: + mod = mf.group(1) + from_counts[mod] = from_counts.get(mod, 0) + 1 + else: + mod = _PLAIN_RE.match(text).group(1) # type: ignore[union-attr] + plain_counts[mod] = plain_counts.get(mod, 0) + 1 + if not ( + has_misplaced + or any(v > 1 for v in from_counts.values()) + or any(v > 1 for v in plain_counts.values()) + ): + return source + + # ── pass 4: build the complete sorted import block ────────────────────── + # Combine every merged import (existing block + newly lifted) and sort the + # whole list so stdlib never ends up after third-party just because it was + # a newly lifted import appended at the end. + all_final_imports = [ + f"from {mod} import {', '.join(from_map[mod])}" for mod in from_order + ] + [f"import {mod}" for mod in plain_order] + sorted_imports = _sort_imports_pep8(all_final_imports) + + first_block_import_idx = min( + (i for i, _ in all_imports if i < first_funcdef_idx), default=-1 + ) + + # ── pass 5: rebuild source ─────────────────────────────────────────────── + # Emit the sorted block at the first block import position (or just before + # the first def/class if there are no block imports). Skip all original + # import lines and blank lines within the original block region — the + # sorted block replaces them entirely. + result: List[str] = [] + import_block_emitted = False + + for i, line in enumerate(lines): + # Edge case: no block imports — insert before the first def/class. + if i == first_funcdef_idx and not import_block_emitted: + for imp in sorted_imports: + result.append(imp + "\n") + import_block_emitted = True + + # Emit the sorted block at the position of the first block import. + if i == first_block_import_idx: + for imp in sorted_imports: + result.append(imp + "\n") + import_block_emitted = True + continue # the original import line is replaced by the block above + + # Drop all other import lines (block duplicates and misplaced). + if i in import_indices: + continue + + # Drop blank lines that fell between import lines in the original block + # — they were section separators that the sorted block supersedes. + if ( + first_block_import_idx >= 0 + and first_block_import_idx < i <= last_block_import_idx + and not line.strip() + ): + continue + + result.append(line) + + result_str = "".join(result) + # When a misplaced import is removed, the blank line that visually separated + # it from the following def/class is left behind. Combined with the two + # trailing blank lines already written after the previous helper, this + # produces three consecutive blank lines — a PEP 8 / E303 violation. + # Collapse any run of 3+ blank lines down to exactly 2 (the PEP 8 maximum + # between top-level definitions). Four or more '\n' in a row means three + # or more blank lines; replace with exactly three '\n' (= two blank lines). + result_str = re.sub(r"\n{4,}", "\n\n\n", result_str) + return result_str diff --git a/crispen/refactors/duplicate_extractor/verification/scope_analysis.py b/crispen/refactors/duplicate_extractor/verification/scope_analysis.py new file mode 100644 index 0000000..1e40c78 --- /dev/null +++ b/crispen/refactors/duplicate_extractor/verification/scope_analysis.py @@ -0,0 +1,296 @@ +from __future__ import annotations +from typing import List +import ast +import re +import textwrap +from ..collectors import _SeqInfo + + +def _scope_end_line(source_lines: List[str], scope: str, after_line: int) -> int: + """Return the exclusive slice index into *source_lines* for the end of *scope*. + + ``after_line`` is the 1-based line number of the last line of the replaced + block. The returned index is suitable for ``source_lines[after_line:idx]`` + to get only the lines inside the enclosing scope that follow the block. + + For ``""`` scope the whole rest of the file is in scope, so + ``len(source_lines)`` is returned. For named function/class scopes the + innermost definition whose name matches *scope* and that contains + *after_line* is located via the AST; its end line is returned as the + exclusive slice bound (1-based end_lineno used directly as a 0-based + exclusive index is correct because line N is at index N-1, so slicing up + to index N includes line N). Falls back to ``len(source_lines)`` on any + parse error or if no matching scope is found. + """ + if scope == "": + return len(source_lines) + + source = "".join(source_lines) + try: + tree = ast.parse(source) + except SyntaxError: + return len(source_lines) + + # ast.walk is BFS, so outer scopes are visited before inner ones. Always + # overwriting best_end means the last match wins — which is the innermost + # (smallest) scope that still contains after_line. + best_end: int = len(source_lines) + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + continue + if node.name != scope: + continue + if not (node.lineno <= after_line <= node.end_lineno): + continue + best_end = node.end_lineno + + return best_end + + +def _has_param_overwritten_before_read(helper_source: str) -> bool: + """Return True if any parameter is assigned before it is first read. + + This detects a common LLM mistake where a parameter is included in the + function signature but then immediately overwritten on the first line, + making the parameter useless and causing UnboundLocalError at call sites + that try to pass a value that was not yet assigned. + """ + try: + tree = ast.parse(textwrap.dedent(helper_source)) + except SyntaxError: # pragma: no cover + return False # pragma: no cover + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + params = {arg.arg for arg in node.args.args} + params |= {arg.arg for arg in node.args.posonlyargs} + params |= {arg.arg for arg in node.args.kwonlyargs} + if node.args.vararg: + params.add(node.args.vararg.arg) + if node.args.kwarg: + params.add(node.args.kwarg.arg) + for stmt in node.body: + for n in ast.walk(stmt): + if isinstance(n, ast.Name) and n.id in params: + if isinstance(n.ctx, ast.Store): + return True + params.discard(n.id) # first use is a read — param is legitimate + return False + + +def _missing_free_vars( + block_src: str, call_srcs: List[str], helper_src: str, source: str +) -> set: + """Return locally-scoped free variable names from block_src absent from the + replacement. + + Free variables are names that are *read* (appear in a ``Load`` context) but + not locally *assigned* (``Store``/``Del``) within the original block. To + avoid false positives from module-level names (imported symbols, globally- + defined functions) that the extracted helper can reference directly, the + check is restricted to names that appear as assignment targets or function + parameters somewhere in *source* — these are variables that live in a local + scope and cannot be reached by the helper without being threaded through as + arguments. + + After this filtering, every remaining name must appear as a bare ``Name`` + node somewhere in the call-site replacements or the helper body. A name + that vanishes from both indicates the LLM silently changed the data flow — + for example by turning a local variable reference into an attribute access + on one of the parameters (``new_source`` → ``transformer.new_source``). + + Returns the set of names that are absent from both. An empty set means the + check passes. Returns an empty set on any ``SyntaxError`` so a parse + failure does not block the extraction — the later ``compile()`` guard will + catch real syntax problems. + """ + try: + block_tree = ast.parse(textwrap.dedent(block_src)) + except SyntaxError: + return set() + + reads: set = set() + stores: set = set() + for node in ast.walk(block_tree): + if isinstance(node, ast.Name): + if isinstance(node.ctx, ast.Load): + reads.add(node.id) + else: + stores.add(node.id) + + free_vars = reads - stores + if not free_vars: + return set() + + # Restrict to names that are locally assigned or are function/lambda + # parameters somewhere in the full source. Module-level names that are + # only ever read (e.g. imported functions, global constants) are in scope + # from the helper definition too and do not need to be passed as args. + try: + source_tree = ast.parse(source) + except SyntaxError: + return set() + source_locals: set = set() + for node in ast.walk(source_tree): + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + source_locals.add(node.id) + elif isinstance(node, ast.arg): + source_locals.add(node.arg) + + free_vars = free_vars & source_locals + if not free_vars: + return set() + + replacement_names: set = set() + for src in list(call_srcs) + [helper_src]: + try: + repl_tree = ast.parse(textwrap.dedent(src)) + except SyntaxError: + return set() + for node in ast.walk(repl_tree): + if isinstance(node, ast.Name): + replacement_names.add(node.id) + + return free_vars - replacement_names + + +def _helper_imports_local_name(helper_source: str, original_source: str) -> bool: + """Return True if helper_source imports a name that is only a local in original. + + Detects the LLM mistake of writing ``import X`` in the helper when ``X`` + was a function parameter or other local name in the original file, not an + importable module. Such imports fail at runtime with ModuleNotFoundError. + """ + try: + helper_tree = ast.parse(textwrap.dedent(helper_source)) + except SyntaxError: + return False + + helper_imports: set = set() + for node in ast.walk(helper_tree): + if isinstance(node, ast.Import): + for alias in node.names: + name = alias.asname if alias.asname else alias.name.split(".")[0] + helper_imports.add(name) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + name = alias.asname if alias.asname else alias.name + helper_imports.add(name) + + if not helper_imports: + return False + + try: + orig_tree = ast.parse(original_source) + except SyntaxError: + return False + + # Names already imported at the top level of the original file. + orig_top_imports: set = set() + for node in orig_tree.body: + if isinstance(node, ast.Import): + for alias in node.names: + name = alias.asname if alias.asname else alias.name.split(".")[0] + orig_top_imports.add(name) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + name = alias.asname if alias.asname else alias.name + orig_top_imports.add(name) + + new_helper_imports = helper_imports - orig_top_imports + if not new_helper_imports: + return False + + # Parameter names in the original file (potential mock-injected locals). + orig_params: set = set() + for node in ast.walk(orig_tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for arg in node.args.args + node.args.posonlyargs + node.args.kwonlyargs: + orig_params.add(arg.arg) + if node.args.vararg: + orig_params.add(node.args.vararg.arg) + if node.args.kwarg: + orig_params.add(node.args.kwarg.arg) + + return bool(new_helper_imports & orig_params) + + +def _names_assigned_in(block_source: str) -> set: + """Return names assigned at the top level of block_source. + + Covers bare ``x = ...`` (ast.Assign) and augmented ``x += ...`` + (ast.AugAssign) statements only; other assignment forms are ignored. + """ + try: + tree = ast.parse(textwrap.dedent(block_source)) + except SyntaxError: + return set() + names: set = set() + for node in tree.body: + if isinstance(node, ast.Assign): + for target in node.targets: + for n in ast.walk(target): + if isinstance(n, ast.Name): + names.add(n.id) + elif isinstance(node, ast.AugAssign): + for n in ast.walk(node.target): + if isinstance(n, ast.Name): + names.add(n.id) + return names + + +def _find_escaping_vars(group: List[_SeqInfo], source_lines: List[str]) -> set: + """Return names assigned in any group sequence that are referenced after it. + + A variable "escapes" when the block assigns it and subsequent code in the + same scope (at the same or deeper indentation level) references it. + The helper must return these variables so callers that need them can + capture the return value. + """ + escaping: set = set() + for seq in group: + block_src = "".join(source_lines[seq.start_line - 1 : seq.end_line]) + assigned = _names_assigned_in(block_src) + if not assigned: + continue + + # Infer the block's indentation level from its first non-empty line. + first_line = next( + ( + ln + for ln in source_lines[seq.start_line - 1 : seq.end_line] + if ln.strip() + ), + "", + ) + block_indent = len(first_line) - len(first_line.lstrip()) + + # Collect lines that follow the block within the same scope. + # For indented blocks: stop when indentation falls below block_indent. + # For module-level (indent 0): stop at the next def/class statement. + after_lines: List[str] = [] + for line in source_lines[seq.end_line :]: + if not line.strip(): + after_lines.append(line) + continue + line_indent = len(line) - len(line.lstrip()) + if block_indent == 0: + if re.match(r"def |class ", line): + break + elif line_indent < block_indent: + break + after_lines.append(line) + + if not after_lines: + continue + + after_src = "".join(after_lines) + try: + after_tree = ast.parse(textwrap.dedent(after_src)) + except SyntaxError: + continue + + used_after = {n.id for n in ast.walk(after_tree) if isinstance(n, ast.Name)} + escaping |= assigned & used_after + + return escaping diff --git a/crispen/refactors/duplicate_extractor/verification/utils.py b/crispen/refactors/duplicate_extractor/verification/utils.py new file mode 100644 index 0000000..20163cc --- /dev/null +++ b/crispen/refactors/duplicate_extractor/verification/utils.py @@ -0,0 +1,183 @@ +from __future__ import annotations +from typing import List +import ast +import textwrap +from ..collectors import _SeqInfo + + +def _normalize_replacement_indentation(seq: _SeqInfo, replacement: str) -> str: + """Re-indent *replacement* to match the original block's leading whitespace. + + The LLM sometimes returns replacements at column 0. This function + re-indents them to match the indentation of the corresponding original + block, so the assembled edit remains valid Python. + """ + orig_lines = [ln for ln in seq.source.splitlines() if ln.strip()] + if not orig_lines: + return replacement + first = orig_lines[0] + expected_indent = first[: len(first) - len(first.lstrip())] + dedented = textwrap.dedent(replacement) + if not expected_indent: + return dedented + return textwrap.indent(dedented, expected_indent) + + +def _collect_ast_store_names(node: ast.AST, names: List[str]) -> None: + """Recursively collect Name ids from an assignment target (Store context).""" + if isinstance(node, ast.Name): + names.append(node.id) + elif isinstance(node, (ast.Tuple, ast.List)): + for elt in node.elts: + _collect_ast_store_names(elt, names) + + +_MUTABLE_CONSTRUCTORS = frozenset({"set", "list", "dict", "frozenset", "bytearray"}) + + +def _collect_attribute_names(source: str) -> set: + """Return all attribute names (dot-access names) anywhere in *source*.""" + try: + tree = ast.parse(source) + except SyntaxError: + return set() + return {node.attr for node in ast.walk(tree) if isinstance(node, ast.Attribute)} + + +def _collect_called_attr_names(source: str) -> set: + """Return attribute names used as method calls in *source*. + + Unlike :func:`_collect_attribute_names`, this only returns names that + appear as the attribute of a call expression (i.e. ``obj.method(...)``). + Plain attribute reads and type annotations like ``ast.AST`` are ignored, + so the new-method-call check does not produce false positives for + standard-library type references. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return set() + return { + node.func.attr + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + + +def _has_funcdef(func_name: str, source: str) -> bool: + """Return True if func_name is defined anywhere in source.""" + try: + tree = ast.parse(source) + except SyntaxError: + return False + for node in ast.walk(tree): + if ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == func_name + ): + return True + return False + + +def _has_call_to(func_name: str, source: str) -> bool: + """Return True if func_name is called anywhere in source. + + Checks both direct calls (``func_name(...)``) and attribute calls + (``obj.func_name(...)``), covering both module-level helpers and + staticmethod calls. Returns False if source cannot be parsed. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return False + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if isinstance(node.func, ast.Name) and node.func.id == func_name: + return True + if isinstance(node.func, ast.Attribute) and node.func.attr == func_name: + return True + return False + + +def _is_pure_literal(node: ast.expr) -> bool: + """Return True if *node* is a side-effect-free literal expression. + + Covers ``ast.Constant`` (numbers, strings, bytes, True/False/None) and + recursively-pure container literals (list, tuple, set, dict). Anything + involving a function call or attribute access returns False. + """ + if isinstance(node, ast.Constant): + return True + if isinstance(node, (ast.List, ast.Tuple, ast.Set)): + return all(_is_pure_literal(e) for e in node.elts) + if isinstance(node, ast.Dict): + return all( + (k is None or _is_pure_literal(k)) and _is_pure_literal(v) + for k, v in zip(node.keys, node.values) + ) + return False + + +def _names_in_edit_texts(extraction_groups) -> set: + """Return all bare ``Name`` ids found in every edit text of *extraction_groups*. + + ``extraction_groups`` is the list of ``(func_name, group_edits, msg)`` + tuples accepted at the end of ``DuplicateExtractor._transform``. Each + ``group_edits`` entry is a ``(start, end, text)`` triple; *text* may be + the helper function source or a call-site replacement. Collecting names + from all of them gives the set of variables that the extraction actually + touched. + """ + names: set = set() + for _, g_edits, _ in extraction_groups: + for _start, _end, text in g_edits: + try: + tree = ast.parse(text) + except SyntaxError: + continue + for node in ast.walk(tree): + if isinstance(node, ast.Name): + names.add(node.id) + return names + + +def _seq_ends_with_return(seq: _SeqInfo) -> bool: + """Return True if the last top-level statement is a non-None return. + + Detects the case where the LLM includes a ``return`` statement inside the + duplicate block but the generated replacement omits it, producing a + function that silently returns ``None`` instead of the original value. + + Bare ``return`` and ``return None`` are excluded: both are semantically + equivalent to falling off the end of a function, so dropping them in a + replacement causes no behavioral change. + """ + try: + tree = ast.parse(textwrap.dedent(seq.source)) + except SyntaxError: + return False + if not tree.body: + return False + last = tree.body[-1] + if not isinstance(last, ast.Return): + return False + # Bare `return` and `return None` are equivalent to implicit None return. + if last.value is None: + return False + if isinstance(last.value, ast.Constant) and last.value.value is None: + return False + return True + + +def _extract_defined_names(source: str) -> set: + """Return all function and class names defined anywhere in *source*.""" + try: + tree = ast.parse(source) + except SyntaxError: + return set() + return { + node.name + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + } diff --git a/crispen/refactors/duplicate_extractor/verification/validation.py b/crispen/refactors/duplicate_extractor/verification/validation.py new file mode 100644 index 0000000..ddcba12 --- /dev/null +++ b/crispen/refactors/duplicate_extractor/verification/validation.py @@ -0,0 +1,223 @@ +from __future__ import annotations +from typing import List, Optional +import ast +import textwrap +from ..collectors import _FunctionInfo, _SeqInfo +from .utils import _MUTABLE_CONSTRUCTORS, _collect_ast_store_names, _is_pure_literal +from .scope_analysis import _has_param_overwritten_before_read + + +def _has_mutable_literal_is_check(source: str) -> bool: + """Return True if *source* contains identity checks against mutable literals. + + Patterns like ``x is set()``, ``x is []``, or ``x is {}`` are always + False in Python because each literal creates a new object at runtime. + Such patterns are a common LLM mistake when using a ``set()`` sentinel. + """ + try: + tree = ast.parse(source) + except SyntaxError: + return False + for node in ast.walk(tree): + if not isinstance(node, ast.Compare): + continue + for op, comp in zip(node.ops, node.comparators): + if not isinstance(op, (ast.Is, ast.IsNot)): + continue + if isinstance(comp, (ast.List, ast.Set, ast.Dict, ast.Tuple)): + return True + if ( + isinstance(comp, ast.Call) + and isinstance(comp.func, ast.Name) + and comp.func.id in _MUTABLE_CONSTRUCTORS + ): + return True + return False + + +def _verify_extraction( + helper_source: Optional[str], call_replacements: List[str] +) -> bool: + """Verify the extraction produces syntactically valid Python. + + Replacements are dedented and then wrapped in a dummy function before + compilation so that ``return`` / ``yield`` statements — which are legal + inside a function body — do not cause false SyntaxError rejections. + Pass helper_source=None to skip the helper compilation check (used when + replacing with an existing function rather than a newly extracted one). + """ + if helper_source is not None: + dedented_helper = textwrap.dedent(helper_source) + try: + compile(dedented_helper, "", "exec") + except SyntaxError: + return False + if _has_param_overwritten_before_read(helper_source): + return False + # Dedent before checking: helper may be indented (e.g. staticmethod). + # compile() already confirmed it's valid Python, so ast.parse will succeed. + if _has_mutable_literal_is_check(dedented_helper): + return False + for replacement in call_replacements: + dedented = textwrap.dedent(replacement) + # Wrap in a dummy function that contains a for loop so that + # ``return`` / ``yield`` (valid inside a function body) AND + # ``continue`` / ``break`` (valid inside a loop body) do not cause + # false SyntaxError rejections. Replacements are always placed back + # inside the caller's original context, which may include a loop. + wrapped = "def _check():\n for _ in []:\n" + textwrap.indent( + dedented, " " + ) + try: + compile(wrapped, "", "exec") + except SyntaxError: + # Retry with async wrapper for replacements that contain `await` + async_wrapped = "async def _check():\n for _ in []:\n" + textwrap.indent( + dedented, " " + ) + try: + compile(async_wrapped, "", "exec") + except SyntaxError: + return False + wrapped = async_wrapped + # Check the wrapped form so that indented/return-containing replacements + # parse successfully and give a definitive True/False answer. + if _has_mutable_literal_is_check(wrapped): + return False + return True + + +def _pyflakes_new_undefined_names(original: str, candidate: str) -> set: + """Return undefined names (F821) introduced by the edit. + + Compares pyflakes output before and after the edit and returns only names + that are newly undefined in the candidate — not ones already present in the + original source. This avoids false positives from pre-existing bare function + calls or module-level references that are valid in context but not resolvable + from a standalone snippet. + """ + import pyflakes.api + import pyflakes.messages + + class _Collector: + def __init__(self): + self.names: set = set() + + def unexpectedError(self, filename, msg): # pragma: no cover + pass + + def syntaxError(self, filename, msg, lineno, offset, text): # pragma: no cover + pass + + def flake(self, msg): + if isinstance(msg, pyflakes.messages.UndefinedName): + self.names.add(msg.message_args[0]) + + before = _Collector() + pyflakes.api.check(original, "", reporter=before) + after = _Collector() + pyflakes.api.check(candidate, "", reporter=after) + return after.names - before.names + + +def _pyflakes_strip_unused_simple_assigns(source: str, allowed_names: set) -> str: + """Remove simple literal initializations that became unused after extraction. + + Only considers assignments whose target name is in *allowed_names* — the + set of variable names that the extraction actually touched. This prevents + the cleaner from making unrelated changes to variables that were already + unused before the extraction ran. + + Runs pyflakes ``UnusedVariable`` (F841) detection on *source* and strips + any ``Assign`` statement whose right-hand side is a pure literal (no + function calls, no attribute accesses), so we never discard side effects. + + A ``compile()`` check guards against the rare case where the removed line + was the only statement in its block — if the result is invalid Python the + original source is returned unchanged. + """ + import pyflakes.api + import pyflakes.messages + + class _Collector: + def __init__(self): + self.linenos: set = set() + + def unexpectedError(self, filename, msg): # pragma: no cover + pass + + def syntaxError(self, filename, msg, lineno, offset, text): # pragma: no cover + pass + + def flake(self, msg): + if isinstance(msg, pyflakes.messages.UnusedVariable): + self.linenos.add(msg.lineno) + + reporter = _Collector() + pyflakes.api.check(source, "", reporter=reporter) + if not reporter.linenos: + return source + + try: + tree = ast.parse(source) + except SyntaxError: # pragma: no cover + return source # pragma: no cover + + lines_to_remove: set = set() + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + if node.lineno not in reporter.linenos: + continue + # Restrict to names the extraction actually touched. + assigned: List[str] = [] + _collect_ast_store_names(node.targets[0], assigned) + if not assigned or not set(assigned).issubset(allowed_names): + continue + if _is_pure_literal(node.value): + lines_to_remove.update(range(node.lineno, node.end_lineno + 1)) + + if not lines_to_remove: + return source + + lines = source.splitlines(keepends=True) + cleaned = "".join( + line for i, line in enumerate(lines, 1) if i not in lines_to_remove + ) + try: + compile(cleaned, "", "exec") + except SyntaxError: + return source + return cleaned + + +def _would_create_proxy_wrappers( + group: List[_SeqInfo], all_functions: List[_FunctionInfo] +) -> bool: + """Return True if extracting this group would leave *some but not all* members + as trivial proxy wrappers. + + A function becomes a trivial proxy wrapper when its entire body is the + extracted block — after extraction it would contain only a single call to + the new helper, with no meaningful logic of its own. + + When *every* member of the group would become a proxy, extraction is still + worthwhile: all functions delegate to the same helper, which eliminates the + duplication. The problematic case is a mixed group where some members lose + all their logic while others keep meaningful bodies. + """ + proxy_count = 0 + non_module_count = 0 + for seq in group: + if seq.scope == "": + continue + non_module_count += 1 + func_outer_scope = ( + seq.class_scope if seq.class_scope is not None else "" + ) + for func in all_functions: + if func.name == seq.scope and func.scope == func_outer_scope: + if len(seq.stmts) == func.body_stmt_count: + proxy_count += 1 + break + return 0 < proxy_count < non_module_count diff --git a/tests/advisor/__init__.py b/tests/advisor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/advisor/test_plan.py b/tests/advisor/test_plan.py new file mode 100644 index 0000000..4e41f41 --- /dev/null +++ b/tests/advisor/test_plan.py @@ -0,0 +1,1002 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch +from crispen.config import CrispenConfig +from crispen.errors import CrispenAPIError +from crispen.file_limiter.advisor import ( + _PLACEMENT_CHUNK_SIZE, + _advise_set3, + _assign_placements_chunk, + advise_file_limiter, +) +import pytest +from .test_unit import ( + _CONFIG, + _PATCH_CLIENT, + _PATCH_KEY, + _classified, + _make_entity, + _make_llm_result, + _propose_ok, +) + + +def test_plan_abort_when_classified_abort(): + """classified.abort=True → FileLimiterPlan(abort=True), no LLM calls.""" + c = _classified(abort=True) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + assert plan.abort is True + assert plan.set3_migrate == [] + assert plan.placements == [] + + +def test_plan_no_movable_groups(): + """set_2=[], set_3=[] → empty plan, no LLM calls.""" + c = _classified() + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + assert plan.abort is False + assert plan.placements == [] + + +def test_plan_api_key_error_propagates(monkeypatch): + """Missing API key raises CrispenAPIError before any LLM call.""" + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + c = _classified( + entities=[_make_entity("foo", 1, 5)], + set_2_groups=[["foo"]], + ) + with pytest.raises(CrispenAPIError): + advise_file_limiter(c, "src/big.py", _CONFIG) + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_set2_only_skips_set3_call(mock_key, mock_client, mock_call): + """set_2 groups only: no set3 call; propose + assign = 2 LLM calls.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.side_effect = [ + _propose_ok("utils.py"), + _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), + ] + c = _classified( + entities=[_make_entity("foo", 1, 10)], + set_2_groups=[["foo"]], + ) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + + assert plan.abort is False + assert plan.set3_migrate == [] + assert len(plan.placements) == 1 + assert plan.placements[0].group == ["foo"] + assert plan.placements[0].target_file == "utils.py" + assert ( + mock_call.call_count == 2 + ) # propose + assign (no refinement: only 1 tiny file) + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_set3_all_stay_no_placement(mock_key, mock_client, mock_call): + """All Set 3 groups stay → no propose/assign call, empty plan.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + {"decisions": [{"group_id": 0, "action": "stay"}]} + ) + + c = _classified( + entities=[_make_entity("bar", 1, 10)], + set_3_groups=[["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + + assert plan.abort is False + assert plan.set3_migrate == [] + assert plan.placements == [] + assert mock_call.call_count == 1 # only set3 advice call + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_set3_migrate( + mock_key, mock_client, mock_call_helpers, mock_call_placement +): + """Set 3 group migrates → set3 + propose + assign = 3 LLM calls.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # Call order: _advise_set3 (helpers) → _propose_files_step (placement) → _assign_placements_chunk (placement) + mock_call_helpers.side_effect = [ + _make_llm_result({"decisions": [{"group_id": 0, "action": "migrate"}]}), + ] + mock_call_placement.side_effect = [ + _propose_ok("helpers.py"), + _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "helpers.py"}]} + ), + ] + c = _classified( + entities=[_make_entity("bar", 1, 10)], + set_3_groups=[["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + + assert plan.abort is False + assert plan.set3_migrate == [["bar"]] + assert len(plan.placements) == 1 + assert plan.placements[0].group == ["bar"] + assert plan.placements[0].target_file == "helpers.py" + assert mock_call_helpers.call_count + mock_call_placement.call_count == 3 + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_set3_test_subdir_skips_advise_call( + mock_key, mock_client, mock_call_helpers, mock_call_placement +): + """Test-file subdir split: set-3 groups migrate without an LLM advice call.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # Only propose + assign calls; no set3-advice call. + # call order: _propose_files_step (placement), _assign_placements_chunk (placement) + mock_call_placement.side_effect = [ + _propose_ok("test_helpers.py"), + _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "test_helpers.py"}]} + ), + ] + c = _classified( + entities=[_make_entity("test_bar", 1, 10)], + set_3_groups=[["test_bar"]], + ) + plan = advise_file_limiter(c, "tests/test_big.py", _CONFIG, subdir_name="big") + + assert plan.abort is False + assert plan.set3_migrate == [["test_bar"]] + assert len(plan.placements) == 1 + assert plan.placements[0].target_file == "test_helpers.py" + assert mock_call_placement.call_count == 2 # no set3-advice call + assert ( + mock_call_helpers.call_count == 0 + ) # helpers.call_with_tool not called in test subdir path + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_set2_and_set3_migrate( + mock_key, mock_client, mock_call_placement, mock_call_helpers +): + """set_2 + migrating set_3 → both groups in placement call.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # call_with_tool is called in both helpers (for set3) and placement (for placement) + # Order: first _advise_set3 in helpers, then _propose_files_step, _assign_placements_chunk in placement + mock_call_helpers.side_effect = [ + _make_llm_result({"decisions": [{"group_id": 0, "action": "migrate"}]}), + ] + mock_call_placement.side_effect = [ + _propose_ok("new_stuff.py", "changed.py"), + _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "new_stuff.py"}, + {"group_id": 1, "target_file": "changed.py"}, + ] + } + ), + ] + c = _classified( + entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 15)], + set_2_groups=[["foo"]], + set_3_groups=[["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + + assert plan.abort is False + assert plan.set3_migrate == [["bar"]] + assert len(plan.placements) == 2 + targets = {p.target_file for p in plan.placements} + assert targets == {"new_stuff.py", "changed.py"} + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_set3_call_returns_none_aborts(mock_key, mock_client, mock_call): + """Call 1 (set3 advice) returns None → abort.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result(None) + + c = _classified( + entities=[_make_entity("bar", 1, 10)], + set_3_groups=[["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + assert plan.abort is True + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_placement_call_returns_none_aborts( + mock_key, mock_client, mock_call_placement, mock_call_helpers +): + """Propose succeeds then assignment chunk exhausts retries → abort.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # file_limiter_retries=0 → 1 attempt for propose, 1 attempt for assign. + # call_with_tool is used in both helpers (for _advise_set3) and placement (for _propose_files_step, _assign_placements_chunk) + # Based on test inputs: set_3_groups is non-empty and not a test subdir, so _advise_set3 is called first (helpers) + # Then _propose_files_step and _assign_placements_chunk are called (placement) + mock_call_helpers.side_effect = [ + _make_llm_result({"decisions": [{"group_id": 0, "action": "migrate"}]}), + ] + mock_call_placement.side_effect = [ + _propose_ok("helpers.py"), + _make_llm_result(None), # assignment fails + ] + c = _classified( + entities=[_make_entity("bar", 1, 10)], + set_3_groups=[["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) + assert plan.abort is True + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_set3_invalid_group_id_treated_as_stay(mock_key, mock_client, mock_call): + """Out-of-range group_id in set3 advice → skipped (treated as stay).""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + { + "decisions": [ + {"group_id": 99, "action": "migrate"}, # invalid — out of range + {"group_id": 0, "action": "stay"}, + ] + } + ) + c = _classified( + entities=[_make_entity("bar", 1, 10)], + set_3_groups=[["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + assert plan.abort is False + assert plan.set3_migrate == [] + assert plan.placements == [] + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_set3_non_int_group_id_treated_as_stay(mock_key, mock_client, mock_call): + """Non-integer group_id in set3 advice → isinstance check fails → stay.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + {"decisions": [{"group_id": "zero", "action": "migrate"}]} + ) + c = _classified( + entities=[_make_entity("bar", 1, 10)], + set_3_groups=[["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + assert plan.set3_migrate == [] + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_set3_unknown_action_treated_as_stay(mock_key, mock_client, mock_call): + """Unknown action value in set3 advice → action != 'migrate' → stay.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + {"decisions": [{"group_id": 0, "action": "delete"}]} # not in enum + ) + c = _classified( + entities=[_make_entity("bar", 1, 10)], + set_3_groups=[["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + assert plan.set3_migrate == [] + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_placement_incomplete_aborts(mock_key, mock_client, mock_call): + """Placement missing some group_ids → len mismatch → abort.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # Two groups but only one placement returned; retries=0 → immediate abort. + mock_call.side_effect = [ + _propose_ok("utils.py"), + _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), + ] + c = _classified( + entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)], + set_2_groups=[["foo"], ["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) + assert plan.abort is True + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_placement_duplicate_group_id_aborts( + mock_key, mock_client, mock_call_placement, mock_call_helpers +): + """Duplicate group_id in placement → only first counted → len mismatch → abort.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # First call is to _propose_files_step in placement.py, second call is to _assign_placements_chunk in placement.py + mock_call_placement.side_effect = [ + _propose_ok("utils.py", "other.py"), + _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "utils.py"}, + {"group_id": 0, "target_file": "other.py"}, # duplicate + ] + } + ), + ] + c = _classified( + entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)], + set_2_groups=[["foo"], ["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) + assert plan.abort is True + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_placement_empty_target_aborts(mock_key, mock_client, mock_call): + """Empty target_file → falsy check fails → treated as missing → abort.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.side_effect = [ + _propose_ok("utils.py"), + _make_llm_result({"placements": [{"group_id": 0, "target_file": ""}]}), + ] + c = _classified( + entities=[_make_entity("foo", 1, 5)], + set_2_groups=[["foo"]], + ) + plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) + assert plan.abort is True + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_placement_out_of_range_group_id_aborts(mock_key, mock_client, mock_call): + """Out-of-range group_id in placement → skipped → len mismatch → abort.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.side_effect = [ + _propose_ok("utils.py"), + _make_llm_result({"placements": [{"group_id": 99, "target_file": "utils.py"}]}), + ] + c = _classified( + entities=[_make_entity("foo", 1, 5)], + set_2_groups=[["foo"]], + ) + plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) + assert plan.abort is True + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_placement_non_int_group_id_aborts( + mock_key, mock_client, mock_call_placement, mock_call_helpers +): + """Non-integer group_id in placement → isinstance check fails → abort.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # First call is in helpers (_advise_set3), second call is in placement (_assign_placements) + mock_call_helpers.return_value = ( + None # Not reached due to early return (no set_3_groups) + ) + mock_call_placement.side_effect = [ + _propose_ok("utils.py"), + _make_llm_result( + {"placements": [{"group_id": "zero", "target_file": "utils.py"}]} + ), + ] + c = _classified( + entities=[_make_entity("foo", 1, 5)], + set_2_groups=[["foo"]], + ) + plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) + assert plan.abort is True + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_placement_targets_outside_proposed_aborts( + mock_key, mock_client, mock_call_helpers, mock_call_placement +): + """LLM returns target not in proposed list → constrained check fails → abort.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # Propose "utils.py" but assignment tries to use "existing.py" (not proposed). + # First call is in helpers (_advise_set3) - returns propose result + mock_call_helpers.return_value = _propose_ok("utils.py") + # Second call is in placement (_assign_placements_chunk) - returns conflicting placement + mock_call_placement.return_value = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "existing.py"}]} + ) + c = _classified( + entities=[_make_entity("foo", 1, 5)], + set_2_groups=[["foo"]], + ) + plan = advise_file_limiter( + c, + "src/big.py", + CrispenConfig(file_limiter_retries=0), + existing_files=frozenset({"existing.py"}), + ) + assert plan.abort is True + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_entity_not_in_entity_map(mock_key, mock_client, mock_call): + """Group contains name absent from entity list → falls back to name-only display.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.side_effect = [ + _propose_ok("utils.py"), + _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), + ] + # "ghost" is not in entities list, so entity_map lookup fails. + c = _classified( + entities=[], + set_2_groups=[["ghost"]], + ) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + assert plan.abort is False + assert plan.placements[0].target_file == "utils.py" + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_set3_prev_failure_appended_to_prompt(mock_key, mock_client, mock_call): + """prev_set3_failure is appended to the set3 advice prompt.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + {"decisions": [{"group_id": 0, "action": "stay"}]} + ) + + c = _classified( + entities=[_make_entity("bar", 1, 10)], + set_3_groups=[["bar"]], + ) + advise_file_limiter(c, "src/big.py", _CONFIG, prev_set3_failure="sentinel text") + + # messages is positional arg index 6 in call_with_tool + messages = mock_call.call_args[0][6] + assert "sentinel text" in messages[0]["content"] + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_placement_prev_failure_appended_to_prompt( + mock_key, mock_client, mock_call_placement, mock_call_helpers +): + """prev_placement_failure is appended to the assignment prompt (last call).""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # First call is in placement._propose_files_step, second is in placement._assign_placements_chunk + mock_call_placement.side_effect = [ + _propose_ok("utils.py"), + _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), + ] + mock_call_helpers.return_value = None # helpers not reached in this test + + c = _classified( + entities=[_make_entity("foo", 1, 10)], + set_2_groups=[["foo"]], + ) + advise_file_limiter( + c, "src/big.py", _CONFIG, prev_placement_failure="sentinel text" + ) + + assert mock_call_placement.call_count == 2 # propose + assign + # The assign call is the last call; it receives prev_placement_failure. + messages = mock_call_placement.call_args[0][6] + assert "sentinel text" in messages[0]["content"] + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_chunked_placement_makes_multiple_calls(mock_key, mock_client, mock_call): + """More than _PLACEMENT_CHUNK_SIZE groups → propose + multiple assign calls.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + + # Build _PLACEMENT_CHUNK_SIZE + 1 groups so two chunks are needed. + n = _PLACEMENT_CHUNK_SIZE + 1 + entities = [_make_entity(f"f{i}", i * 2 + 1, i * 2 + 2) for i in range(n)] + groups = [[f"f{i}"] for i in range(n)] + + # First chunk returns placements for group_ids 0..CHUNK_SIZE-1. + first_chunk_response = _make_llm_result( + { + "placements": [ + {"group_id": j, "target_file": "file_a.py"} + for j in range(_PLACEMENT_CHUNK_SIZE) + ] + } + ) + # Second chunk has 1 group (group_id 0) → goes to file_b.py (tiny, 2 lines). + second_chunk_response = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "file_b.py"}]} + ) + # Refinement: file_b.py is tiny (2 lines < 200), reassign to file_a.py. + refine_response = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "file_a.py"}]} + ) + + mock_call.side_effect = [ + _propose_ok("file_a.py", "file_b.py"), + first_chunk_response, + second_chunk_response, + refine_response, + ] + + c = _classified(entities=entities, set_2_groups=groups) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + + assert plan.abort is False + assert len(plan.placements) == n + # propose + chunk1 + chunk2 + refine = 4 calls. + assert mock_call.call_count == 4 + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_chunked_placement_second_chunk_fails_aborts( + mock_key, mock_client, mock_call, capsys +): + """Second chunk exhausts all per-chunk retries → placement returns None → abort.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + + n = _PLACEMENT_CHUNK_SIZE + 1 + entities = [_make_entity(f"f{i}", i * 2 + 1, i * 2 + 2) for i in range(n)] + groups = [[f"f{i}"] for i in range(n)] + + first_chunk_response = _make_llm_result( + { + "placements": [ + {"group_id": j, "target_file": "file_a.py"} + for j in range(_PLACEMENT_CHUNK_SIZE) + ] + } + ) + + cfg = CrispenConfig(file_limiter_retries=1) # 2 attempts per chunk + # propose + chunk 1 (1 call) + chunk 2 (2 failed attempts) = 4 calls. + mock_call.side_effect = [ + _propose_ok("file_a.py", "file_b.py"), + first_chunk_response, + _make_llm_result(None), + _make_llm_result(None), + ] + + c = _classified(entities=entities, set_2_groups=groups) + plan = advise_file_limiter(c, "src/big.py", cfg, verbose=True) + + assert plan.abort is True + assert "LLM failed to assign file placements" in plan.abort_reason + assert mock_call.call_count == 4 # propose + chunk1 + 2 failed chunk2 attempts + assert "failed to assign file placements" in capsys.readouterr().err + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_chunked_placement_chunk_retry_succeeds(mock_key, mock_client, mock_call): + """A chunk that fails once is retried; on success the full plan is returned.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + + n = _PLACEMENT_CHUNK_SIZE + 1 + entities = [_make_entity(f"f{i}", i * 2 + 1, i * 2 + 2) for i in range(n)] + groups = [[f"f{i}"] for i in range(n)] + + first_chunk_response = _make_llm_result( + { + "placements": [ + {"group_id": j, "target_file": "file_a.py"} + for j in range(_PLACEMENT_CHUNK_SIZE) + ] + } + ) + second_chunk_response = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "file_b.py"}]} + ) + # Refinement: file_b.py is tiny, reassign to file_a.py. + refine_response = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "file_a.py"}]} + ) + + cfg = CrispenConfig(file_limiter_retries=1) # 2 attempts per chunk + # propose + chunk1 + chunk2 (fail) + chunk2 (succeed) + refine = 5 calls. + mock_call.side_effect = [ + _propose_ok("file_a.py", "file_b.py"), + first_chunk_response, + _make_llm_result(None), + second_chunk_response, + refine_response, + ] + + c = _classified(entities=entities, set_2_groups=groups) + plan = advise_file_limiter(c, "src/big.py", cfg) + + assert plan.abort is False + assert len(plan.placements) == n + assert mock_call.call_count == 5 + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_plan_chunked_placement_zero_total_lines(mock_key, mock_client, mock_call): + """Groups whose names are absent from entity_map → total_lines==0 → fallback.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + + # Groups reference names not present in entity_map (entities=[]). + # Projected lines = 0 for all files → no tiny files → no refinement. + groups = [["orphan_a"], ["orphan_b"]] + mock_call.side_effect = [ + _propose_ok("a.py", "b.py"), + _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "a.py"}, + {"group_id": 1, "target_file": "b.py"}, + ] + } + ), + ] + + c = _classified(entities=[], set_2_groups=groups) + plan = advise_file_limiter(c, "src/big.py", _CONFIG) + + assert plan.abort is False + assert len(plan.placements) == 2 + assert mock_call.call_count == 2 # propose + assign (no refinement) + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_placement_prompt_includes_mermaid_when_deps_exist( + mock_key, mock_client, mock_call +): + """Inter-group deps exist → Mermaid diagram included in the assignment prompt.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.side_effect = [ + _propose_ok("utils.py", "models.py"), + _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "utils.py"}, + {"group_id": 1, "target_file": "models.py"}, + ] + } + ), + ] + c = _classified( + entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)], + set_2_groups=[["foo"], ["bar"]], + graph={"foo": {"bar"}, "bar": set()}, + ) + advise_file_limiter(c, "src/big.py", _CONFIG) + + # The assignment call is the last call; it has the Mermaid diagram. + messages = mock_call.call_args[0][6] + assert "```mermaid" in messages[0]["content"] + assert "G0 --> G1" in messages[0]["content"] + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_advise_verbose_set3_and_placement( + mock_key, mock_client, mock_call_placement, mock_call_helpers, capsys +): + """verbose=True exercises the print + _counter branches in _advise_set3, + _propose_files_step, and _assign_placements_chunk.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # set3 call → propose call → assign call. + # Order: _advise_set3 (helpers) → _propose_files_step (placement) → _assign_placements_chunk (placement) + mock_call_helpers.side_effect = [ + _make_llm_result({"decisions": [{"group_id": 0, "action": "migrate"}]}), + ] + mock_call_placement.side_effect = [ + _propose_ok("utils.py"), + _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), + ] + c = _classified( + entities=[_make_entity("bar", 1, 10)], + set_3_groups=[["bar"]], + ) + plan = advise_file_limiter(c, "src/big.py", _CONFIG, verbose=True) + + assert plan.abort is False + assert plan.llm_calls == 3 + err = capsys.readouterr().err + assert "set-3 group" in err + assert "file placements" in err + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_advise_verbose_detailed_timing_prints( + mock_key, mock_client, mock_call_helpers, mock_call_placement, capsys +): + """timing='detailed' prints per-call → done lines for set3, propose, and assign.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # set3 call → propose call → assign call. + # set3 is handled by helpers, propose and assign are handled by placement + mock_call_helpers.side_effect = [ + _make_llm_result({"decisions": [{"group_id": 0, "action": "migrate"}]}), + ] + mock_call_placement.side_effect = [ + _propose_ok("utils.py"), + _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), + ] + c = _classified( + entities=[_make_entity("bar", 1, 10)], + set_3_groups=[["bar"]], + ) + plan = advise_file_limiter( + c, "src/big.py", _CONFIG, verbose=True, timing="detailed" + ) + + assert plan.abort is False + err = capsys.readouterr().err + assert "→ done [" in err + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +def test_advise_set3_no_counter(mock_call): + """_advise_set3 called without _counter covers the None-counter branch.""" + mock_call.return_value = _make_llm_result( + {"decisions": [{"group_id": 0, "action": "migrate"}]} + ) + c = _classified( + entities=[_make_entity("foo", 1, 5)], + set_3_groups=[["foo"]], + ) + result = _advise_set3(c, "big.py", MagicMock(), _CONFIG) + assert result == [["foo"]] + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +def test_advise_set3_with_dep_graph(mock_call): + """_advise_set3 with inter-group dependencies includes mermaid graph in prompt.""" + mock_call.return_value = _make_llm_result( + {"decisions": [{"group_id": 0, "action": "migrate"}]} + ) + # graph["foo"] = {"bar"} means foo depends on bar → two groups have an edge + c = _classified( + entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)], + graph={"foo": {"bar"}}, + set_3_groups=[["foo"], ["bar"]], + ) + result = _advise_set3(c, "big.py", MagicMock(), _CONFIG) + assert result == [["foo"]] + # Verify the mermaid graph was injected into the prompt. + prompt = mock_call.call_args[0][6][0]["content"] + assert "graph TD" in prompt + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_assign_placements_chunk_no_counter(mock_call): + """_assign_placements_chunk without _counter covers the None-counter branch.""" + mock_call.return_value = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "utils.py"}]} + ) + c = _classified(entities=[_make_entity("foo", 1, 5)]) + result = _assign_placements_chunk( + [["foo"]], c, "big.py", frozenset(), MagicMock(), _CONFIG + ) + assert result is not None + assert result[0].target_file == "utils.py" + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_assign_placements_chunk_subdir_name(mock_call): + """subdir_name is included in the prompt and suppresses the plain directory rule.""" + mock_call.return_value = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "detection_flow.py"}]} + ) + c = _classified(entities=[_make_entity("foo", 1, 5)]) + result = _assign_placements_chunk( + [["foo"]], + c, + "tests/test_duplicate_extractor.py", + frozenset(), + MagicMock(), + _CONFIG, + subdir_name="duplicate_extractor", + ) + assert result is not None + assert result[0].target_file == "detection_flow.py" + # The prompt should mention the subdirectory and warn against repeating its name. + prompt = mock_call.call_args[0][6][0]["content"] + assert "duplicate_extractor/" in prompt + assert "do not repeat" in prompt.lower() + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_assign_placements_chunk_strips_subdir_prefix(mock_call): + """LLM returns 'subdir/file.py' — the leading subdir/ should be stripped.""" + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "duplicate_extractor/detection_flow.py"} + ] + } + ) + c = _classified(entities=[_make_entity("foo", 1, 5)]) + result = _assign_placements_chunk( + [["foo"]], + c, + "tests/test_duplicate_extractor.py", + frozenset(), + MagicMock(), + _CONFIG, + subdir_name="duplicate_extractor", + ) + assert result is not None + assert result[0].target_file == "detection_flow.py" + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_propose_retry_succeeds_on_second_attempt( + mock_key, mock_client, mock_call_placement, mock_call_helpers +): + """Propose returns None once, then succeeds on retry (lines 862->882, 878).""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + entity = _make_entity("foo", 1, 50) + c = _classified(entities=[entity], set_2_groups=[["foo"]]) + # Call order: _propose_files_step (placement), then _assign_placements_chunk (placement) + mock_call_placement.side_effect = [ + _make_llm_result(None), # propose fails first attempt + _propose_ok("helpers.py"), # propose succeeds on retry + _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "helpers.py"}]} + ), # assign + # no refinement: 50 lines is not tiny (>= 200 is fine, 50 < 200 but only file) + ] + # _advise_set3 is not called (no set_3_groups), so helpers.call_with_tool is not exercised + mock_call_helpers.return_value = None + plan = advise_file_limiter( + c, + "src/big.py", + CrispenConfig(file_limiter_retries=1), # allow 1 retry + ) + assert plan.abort is False + assert plan.placements[0].target_file == "helpers.py" + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_propose_all_retries_exhausted_aborts(mock_key, mock_client, mock_call): + """All propose retries fail → _assign_placements returns None → abort (line 883).""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + entity = _make_entity("foo", 1, 50) + c = _classified(entities=[entity], set_2_groups=[["foo"]]) + mock_call.return_value = _make_llm_result(None) # propose always fails + plan = advise_file_limiter( + c, + "src/big.py", + CrispenConfig(file_limiter_retries=0), # no retries + ) + assert plan.abort is True + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_propose_no_tool_call_verbose(mock_key, mock_client, mock_call, capsys): + """tool_input=None + verbose=True → logs 'no tool call in response'.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + entity = _make_entity("foo", 1, 50) + c = _classified(entities=[entity], set_2_groups=[["foo"]]) + mock_call.return_value = _make_llm_result(None) + plan = advise_file_limiter( + c, "src/big.py", CrispenConfig(file_limiter_retries=0), verbose=True + ) + assert plan.abort is True + assert "no tool call" in capsys.readouterr().err + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_propose_empty_files_list_verbose(mock_key, mock_client, mock_call, capsys): + """tool_input={"files": []} + verbose=True → logs 'empty files list'.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + entity = _make_entity("foo", 1, 50) + c = _classified(entities=[entity], set_2_groups=[["foo"]]) + mock_call.return_value = _make_llm_result({"files": []}) + plan = advise_file_limiter( + c, "src/big.py", CrispenConfig(file_limiter_retries=0), verbose=True + ) + assert plan.abort is True + assert "empty files list" in capsys.readouterr().err + + +@patch("crispen.file_limiter.advisor.helpers.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch(_PATCH_CLIENT) +@patch(_PATCH_KEY) +def test_propose_all_filenames_filtered_verbose( + mock_key, mock_client, mock_call_placement, mock_call_helpers, capsys +): + """All proposed filenames in existing_files + verbose=True → logs filtered names.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + entity = _make_entity("foo", 1, 50) + c = _classified(entities=[entity], set_2_groups=[["foo"]]) + # Propose "taken.py" which is already in existing_files. + mock_call_placement.return_value = _make_llm_result( + {"files": [{"filename": "taken.py", "description": "existing"}]} + ) + plan = advise_file_limiter( + c, + "src/big.py", + CrispenConfig(file_limiter_retries=0), + existing_files=frozenset({"taken.py"}), + verbose=True, + ) + assert plan.abort is True + assert "filtered" in capsys.readouterr().err diff --git a/tests/advisor/test_resolve.py b/tests/advisor/test_resolve.py new file mode 100644 index 0000000..b44de2f --- /dev/null +++ b/tests/advisor/test_resolve.py @@ -0,0 +1,475 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch +from crispen.config import CrispenConfig +from crispen.errors import CrispenAPIError +from crispen.file_limiter.advisor import ( + GroupPlacement, + _LLMAccumulator, + _find_conflicting_placement_indices, + resolve_naming_conflicts, +) +import pytest +from .test_unit import _CONFIG, _classified, _make_entity, _make_llm_result + + +def test_find_conflicting_idx_plan_vs_plan(): + """Flat file + subdir with same stem both appear → both indices returned.""" + placements = [ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="utils/io.py"), + GroupPlacement(group=["baz"], target_file="helpers.py"), + ] + idxs = _find_conflicting_placement_indices(placements, frozenset(), frozenset()) + assert idxs == [0, 1] + + +def test_find_conflicting_idx_file_vs_existing_dir(): + """Flat .py target whose stem matches an existing directory → index returned.""" + placements = [GroupPlacement(group=["foo"], target_file="models.py")] + idxs = _find_conflicting_placement_indices( + placements, frozenset(), frozenset({"models"}) + ) + assert idxs == [0] + + +def test_find_conflicting_idx_subdir_vs_existing_file(): + """Subdir target whose top matches an existing .py file → index returned.""" + placements = [GroupPlacement(group=["bar"], target_file="helpers/io.py")] + idxs = _find_conflicting_placement_indices( + placements, frozenset({"helpers.py"}), frozenset() + ) + assert idxs == [0] + + +def test_find_conflicting_idx_flat_target_in_existing_files(): + """Flat target in existing_files (e.g. conftest.py) → index returned.""" + placements = [ + GroupPlacement(group=["fix"], target_file="conftest.py"), + GroupPlacement(group=["bar"], target_file="helpers.py"), + ] + idxs = _find_conflicting_placement_indices( + placements, frozenset({"conftest.py"}), frozenset() + ) + assert idxs == [0] + + +def test_find_conflicting_idx_no_conflict(): + """Clean plan with no conflicts → empty list.""" + placements = [ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="helpers.py"), + ] + assert ( + _find_conflicting_placement_indices(placements, frozenset(), frozenset()) == [] + ) + + +_CONFLICTING_PLACEMENTS = [ + GroupPlacement(group=["foo"], target_file="utils.py"), # plan-vs-plan conflict + GroupPlacement(group=["bar"], target_file="utils/io.py"), # plan-vs-plan conflict + GroupPlacement(group=["baz"], target_file="helpers.py"), # not conflicting +] + +_CLEAN_PLACEMENTS = [ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="helpers.py"), +] + + +def test_resolve_no_conflicts_returns_unchanged(): + """No conflicts → returns a copy of the input list; no LLM calls needed.""" + c = _classified(entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)]) + result = resolve_naming_conflicts( + _CLEAN_PLACEMENTS, c, "src/big.py", frozenset(), frozenset(), _CONFIG + ) + assert result == _CLEAN_PLACEMENTS + assert result is not _CLEAN_PLACEMENTS + + +def test_resolve_api_key_error_propagates(monkeypatch): + """Missing API key raises CrispenAPIError before any LLM call.""" + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + c = _classified(entities=[_make_entity("foo", 1, 5)]) + with pytest.raises(CrispenAPIError): + resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, c, "src/big.py", frozenset(), frozenset(), _CONFIG + ) + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_success(mock_key, mock_client, mock_call): + """Happy path: forbidden_dir_stems and existing_file_stems both non-empty; + prev_failure is False on the first (successful) attempt.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "models.py"}, + {"group_id": 1, "target_file": "services.py"}, + ] + } + ) + c = _classified( + entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)], + ) + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + existing_files=frozenset({"other.py"}), # non-empty → existing_file_stems + existing_dirs=frozenset({"mydir"}), # non-empty → forbidden_dir_stems + config=_CONFIG, + ) + assert result is not None + assert result[0].target_file == "models.py" + assert result[1].target_file == "services.py" + assert result[2].target_file == "helpers.py" # non-conflicting, unchanged + assert mock_call.call_count == 1 + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_llm_none_returns_none(mock_key, mock_client, mock_call): + """LLM returns None → resolve returns None.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result(None) + c = _classified() + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + frozenset(), + frozenset(), + CrispenConfig(file_limiter_retries=0), + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_forbidden_target_returns_none(mock_key, mock_client, mock_call): + """LLM picks a target that is in forbidden_files → resolve returns None.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + # "helpers.py" is a non-conflicting target → included in forbidden_files. + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "helpers.py"}, + {"group_id": 1, "target_file": "services.py"}, + ] + } + ) + c = _classified() + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + frozenset(), + frozenset(), + CrispenConfig(file_limiter_retries=0), + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_incomplete_response_returns_none(mock_key, mock_client, mock_call): + """LLM returns fewer placements than groups → len mismatch → None.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "models.py"}]} # only 1 of 2 + ) + c = _classified() + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + frozenset(), + frozenset(), + CrispenConfig(file_limiter_retries=0), + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_retry_succeeds(mock_key, mock_client, mock_call): + """First attempt None, second succeeds; covers if prev_failure: True branch.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.side_effect = [ + _make_llm_result(None), + _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "models.py"}, + {"group_id": 1, "target_file": "services.py"}, + ] + } + ), + ] + c = _classified() + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + frozenset(), + frozenset(), + CrispenConfig(file_limiter_retries=1), + ) + assert result is not None + assert result[0].target_file == "models.py" + assert mock_call.call_count == 2 + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_empty_forbidden_dir_stems(mock_key, mock_client, mock_call): + """existing_dirs empty → forbidden_dir_stems empty → branch False.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "models.py"}, + {"group_id": 1, "target_file": "services.py"}, + ] + } + ) + c = _classified() + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + existing_files=frozenset({"other.py"}), # file_stems non-empty + existing_dirs=frozenset(), # dir_stems empty + config=_CONFIG, + ) + assert result is not None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_empty_existing_file_stems(mock_key, mock_client, mock_call): + """existing_files=frozenset() → file_stems empty → if existing_file_stems: False.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "models.py"}, + {"group_id": 1, "target_file": "services.py"}, + ] + } + ) + c = _classified() + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + existing_files=frozenset(), # file_stems empty + existing_dirs=frozenset({"mydir"}), # dir_stems non-empty + config=_CONFIG, + ) + assert result is not None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_non_int_group_id(mock_key, mock_client, mock_call): + """Non-integer group_id → isinstance check fails → skipped → len mismatch → None.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": "zero", "target_file": "models.py"}, + {"group_id": 1, "target_file": "services.py"}, + ] + } + ) + c = _classified() + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + frozenset(), + frozenset(), + CrispenConfig(file_limiter_retries=0), + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_out_of_range_group_id(mock_key, mock_client, mock_call): + """Out-of-range group_id → range check fails → skipped → len mismatch → None.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": 99, "target_file": "models.py"}, + {"group_id": 1, "target_file": "services.py"}, + ] + } + ) + c = _classified() + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + frozenset(), + frozenset(), + CrispenConfig(file_limiter_retries=0), + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_duplicate_group_id(mock_key, mock_client, mock_call): + """Duplicate group_id → second entry skipped → len mismatch → None.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "models.py"}, + {"group_id": 0, "target_file": "other.py"}, # duplicate + ] + } + ) + c = _classified() + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + frozenset(), + frozenset(), + CrispenConfig(file_limiter_retries=0), + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_empty_target(mock_key, mock_client, mock_call): + """Empty target_file → falsy check fails → skipped → len mismatch → None.""" + mock_key.return_value = "key" + mock_client.return_value = MagicMock() + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": ""}, + {"group_id": 1, "target_file": "services.py"}, + ] + } + ) + c = _classified() + result = resolve_naming_conflicts( + _CONFLICTING_PLACEMENTS, + c, + "src/big.py", + frozenset(), + frozenset(), + CrispenConfig(file_limiter_retries=0), + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_verbose(mock_key, mock_client, mock_call, capsys): + """verbose=True exercises the print + _counter branches in + _rename_conflicting_chunk (with _counter passed to cover the increment).""" + mock_key.return_value = "key" + # Both placements conflict (utils.py vs utils/io.py share stem "utils"), + # so the chunk sent to LLM has 2 groups; return both renamed. + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "models.py"}, + {"group_id": 1, "target_file": "helpers.py"}, + ] + } + ) + entity = _make_entity("foo", 1, 5) + c = _classified(entities=[entity]) + placements = [ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="utils/io.py"), # conflict + ] + acc = _LLMAccumulator() + result = resolve_naming_conflicts( + placements, + c, + "src/big.py", + frozenset(), + frozenset(), + _CONFIG, + verbose=True, + _acc=acc, + ) + + assert result is not None + assert acc.calls == 1 # one LLM call was counted + err = capsys.readouterr().err + assert "naming conflicts" in err + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +@patch("crispen.file_limiter.advisor.placement.make_client") +@patch("crispen.file_limiter.advisor.placement.get_api_key") +def test_resolve_verbose_detailed_timing_print( + mock_key, mock_client, mock_call, capsys +): + """timing='detailed' prints per-call → done line in resolve_naming_conflicts.""" + mock_key.return_value = "key" + mock_call.return_value = _make_llm_result( + { + "placements": [ + {"group_id": 0, "target_file": "models.py"}, + {"group_id": 1, "target_file": "helpers.py"}, + ] + } + ) + entity = _make_entity("foo", 1, 5) + c = _classified(entities=[entity]) + placements = [ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="utils/io.py"), # conflict + ] + acc = _LLMAccumulator() + result = resolve_naming_conflicts( + placements, + c, + "src/big.py", + frozenset(), + frozenset(), + _CONFIG, + verbose=True, + timing="detailed", + _acc=acc, + ) + + assert result is not None + err = capsys.readouterr().err + assert "→ done [" in err diff --git a/tests/advisor/test_unit.py b/tests/advisor/test_unit.py new file mode 100644 index 0000000..e2cc321 --- /dev/null +++ b/tests/advisor/test_unit.py @@ -0,0 +1,644 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch +from crispen.config import CrispenConfig +from crispen.llm_client import LLMCallResult +from crispen.file_limiter.advisor import ( + GroupPlacement, + _LLMAccumulator, + _assign_placements_chunk, + _build_group_mermaid, + _compute_projected_lines, + _group_summary, + _propose_files_step, + _refine_merge_tiny, +) +from crispen.file_limiter.classifier import ClassifiedEntities +from crispen.file_limiter.entity_parser import Entity, EntityKind + + +def _make_entity( + name: str, + start: int, + end: int, + *, + docstring=None, + params=None, +) -> Entity: + return Entity( + EntityKind.FUNCTION, + name, + start, + end, + [name], + docstring=docstring, + params=params or [], + ) + + +def _classified( + *, + entities=None, + entity_class=None, + graph=None, + set_1=None, + set_2_groups=None, + set_3_groups=None, + abort=False, +) -> ClassifiedEntities: + return ClassifiedEntities( + entities=entities or [], + entity_class=entity_class or {}, + graph=graph if graph is not None else {}, + set_1=set_1 or [], + set_2_groups=set_2_groups or [], + set_3_groups=set_3_groups or [], + abort=abort, + ) + + +def _make_llm_result(tool_input) -> LLMCallResult: + """Wrap a dict (or None) in LLMCallResult for mock_call.return_value.""" + return LLMCallResult( + tool_input=tool_input, elapsed=0.01, input_tokens=10, output_tokens=5 + ) + + +def _propose_ok(*filenames: str) -> LLMCallResult: + """Return a valid propose_output_files LLM response for the given filenames.""" + return _make_llm_result( + {"files": [{"filename": f, "description": "auto-generated"} for f in filenames]} + ) + + +_CONFIG = CrispenConfig() +_PATCH_KEY = "crispen.file_limiter.advisor.get_api_key" +_PATCH_CLIENT = "crispen.file_limiter.advisor.make_client" +_PATCH_CALL = "crispen.file_limiter.advisor.call_with_tool" + + +def test_group_summary_with_docstring_and_params(): + """Entity with docstring and params → both appear in summary.""" + ent = _make_entity( + "foo", + 1, + 10, + docstring="Parse the config file. More details here.", + params=["path: str", "strict: bool"], + ) + summary = _group_summary(["foo"], {"foo": ent}) + assert "foo (10 lines)" in summary + assert '"Parse the config file."' in summary + assert "params: path: str, strict: bool" in summary + + +def test_group_summary_with_params_only(): + """Entity with params but no docstring → params appear, no docstring quote.""" + ent = _make_entity("bar", 1, 5, params=["x: int", "y"]) + summary = _group_summary(["bar"], {"bar": ent}) + assert "params: x: int, y" in summary + assert '"' not in summary + + +def test_group_summary_docstring_no_period(): + """Docstring with no period → full text used as first sentence.""" + ent = _make_entity("baz", 1, 3, docstring="No period here") + summary = _group_summary(["baz"], {"baz": ent}) + assert '"No period here"' in summary + + +def test_group_summary_with_section_header(): + """Entity with section_header → section appears first in extras.""" + + ent = Entity( + EntityKind.FUNCTION, + "foo", + 1, + 5, + ["foo"], + section_header="Helpers", + ) + summary = _group_summary(["foo"], {"foo": ent}) + assert 'section: "Helpers"' in summary + + +def test_group_summary_no_section_header(): + """Entity without section_header → no 'section:' in summary.""" + ent = _make_entity("bar", 1, 5) + summary = _group_summary(["bar"], {"bar": ent}) + assert "section:" not in summary + + +def test_build_group_mermaid_no_edges(): + """Empty graph → no inter-group deps → returns empty string.""" + c = _classified(entities=[], set_2_groups=[["foo"], ["bar"]]) + result = _build_group_mermaid([["foo"], ["bar"]], c) + assert result == "" + + +def test_build_group_mermaid_with_inter_group_dep(): + """G0 depends on G1 → Mermaid text with that edge is returned.""" + c = _classified(graph={"foo": {"bar"}, "bar": set()}) + result = _build_group_mermaid([["foo"], ["bar"]], c) + assert "```mermaid" in result + assert "G0 --> G1" in result + + +def test_build_group_mermaid_dep_outside_chunk(): + """Dep to entity outside chunk → dep_gid is None → not added → empty.""" + c = _classified(graph={"foo": {"external"}, "bar": set()}) + result = _build_group_mermaid([["foo"], ["bar"]], c) + assert result == "" + + +def test_build_group_mermaid_intra_group_dep(): + """Dep within same SCC group → dep_gid == gid → not added as edge.""" + # foo and baz are in the same group; foo depends on baz (intra-SCC edge) + c = _classified(graph={"foo": {"baz"}, "baz": {"foo"}, "bar": set()}) + result = _build_group_mermaid([["foo", "baz"], ["bar"]], c) + assert result == "" + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_assign_placements_chunk_constrained_success(mock_call): + """Constrained mode: target in proposed_filenames → placement accepted.""" + mock_call.return_value = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "utils.py"}]} + ) + c = _classified(entities=[_make_entity("foo", 1, 5)]) + proposed = [("utils.py", "general utilities"), ("models.py", "data models")] + result = _assign_placements_chunk( + [["foo"]], + c, + "src/big.py", + frozenset(), + MagicMock(), + _CONFIG, + proposed_files=proposed, + ) + assert result is not None + assert result[0].target_file == "utils.py" + # Prompt should list proposed files and instruct constrained choice. + prompt = mock_call.call_args[0][6][0]["content"] + assert "Proposed output files" in prompt + assert "utils.py" in prompt + assert "models.py" in prompt + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_assign_placements_chunk_constrained_invalid_target(mock_call): + """Constrained mode: target not in proposed_filenames → immediate None return.""" + mock_call.return_value = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "rogue_file.py"}]} + ) + c = _classified(entities=[_make_entity("foo", 1, 5)]) + proposed = [("utils.py", "general utilities")] + result = _assign_placements_chunk( + [["foo"]], + c, + "src/big.py", + frozenset(), + MagicMock(), + _CONFIG, + proposed_files=proposed, + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_success(mock_call): + """Basic success: valid filenames are returned.""" + mock_call.return_value = _make_llm_result( + { + "files": [ + {"filename": "utils.py", "description": "utility functions"}, + {"filename": "models.py", "description": "data models"}, + ] + } + ) + c = _classified(entities=[_make_entity("foo", 1, 50)]) + result = _propose_files_step( + [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG + ) + assert result is not None + assert len(result) == 2 + assert result[0] == ("utils.py", "utility functions") + assert result[1] == ("models.py", "data models") + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_llm_none(mock_call): + """call_with_tool returns None → _propose_files_step returns None.""" + mock_call.return_value = _make_llm_result(None) + c = _classified() + result = _propose_files_step( + [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_empty_files_list(mock_call): + """LLM returns empty files list → returns None (not proposed).""" + mock_call.return_value = _make_llm_result({"files": []}) + c = _classified() + result = _propose_files_step( + [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_strips_existing_files(mock_call): + """Filename in existing_files is stripped; remaining valid ones returned.""" + mock_call.return_value = _make_llm_result( + { + "files": [ + {"filename": "taken.py", "description": "already exists"}, + {"filename": "utils.py", "description": "new file"}, + ] + } + ) + c = _classified() + result = _propose_files_step( + [["foo"]], + c, + "src/big.py", + 2, + frozenset({"taken.py"}), + MagicMock(), + _CONFIG, + ) + assert result is not None + assert len(result) == 1 + assert result[0][0] == "utils.py" + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_all_in_existing_files(mock_call): + """All proposed filenames are in existing_files → stripped → returns None.""" + mock_call.return_value = _make_llm_result( + {"files": [{"filename": "taken.py", "description": "existing"}]} + ) + c = _classified() + result = _propose_files_step( + [["foo"]], + c, + "src/big.py", + 2, + frozenset({"taken.py"}), + MagicMock(), + _CONFIG, + ) + assert result is None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_strips_duplicates(mock_call): + """Duplicate filenames are stripped; only first occurrence kept.""" + mock_call.return_value = _make_llm_result( + { + "files": [ + {"filename": "utils.py", "description": "first"}, + {"filename": "utils.py", "description": "duplicate"}, + ] + } + ) + c = _classified() + result = _propose_files_step( + [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG + ) + assert result is not None + assert len(result) == 1 + assert result[0] == ("utils.py", "first") + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_strips_empty_filename(mock_call): + """Empty filename string is skipped; valid ones returned.""" + mock_call.return_value = _make_llm_result( + { + "files": [ + {"filename": "", "description": "empty"}, + {"filename": "utils.py", "description": "valid"}, + ] + } + ) + c = _classified() + result = _propose_files_step( + [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG + ) + assert result is not None + assert len(result) == 1 + assert result[0][0] == "utils.py" + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_verbose(mock_call, capsys): + """verbose=True prints propose message to stderr and increments counter.""" + mock_call.return_value = _make_llm_result( + {"files": [{"filename": "utils.py", "description": "utilities"}]} + ) + c = _classified() + acc = _LLMAccumulator() + result = _propose_files_step( + [["foo"]], + c, + "src/big.py", + 2, + frozenset(), + MagicMock(), + _CONFIG, + verbose=True, + _acc=acc, + ) + assert result is not None + assert acc.calls == 1 + err = capsys.readouterr().err + assert "propose" in err.lower() + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_no_counter(mock_call): + """_counter=None covers the None-counter branch (no increment).""" + mock_call.return_value = _make_llm_result( + {"files": [{"filename": "utils.py", "description": "utilities"}]} + ) + c = _classified() + result = _propose_files_step( + [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG + ) + assert result is not None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_subdir_name(mock_call): + """subdir_name triggers the subdir placement_rule branch in the prompt.""" + mock_call.return_value = _make_llm_result( + {"files": [{"filename": "handlers.py", "description": "request handlers"}]} + ) + c = _classified() + result = _propose_files_step( + [["foo"]], + c, + "src/service.py", + 2, + frozenset(), + MagicMock(), + _CONFIG, + subdir_name="service", + ) + assert result is not None + prompt = mock_call.call_args[0][6][0]["content"] + assert "service/" in prompt + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_no_existing_files(mock_call): + """existing_files=frozenset() → exclude_section empty (branch False).""" + mock_call.return_value = _make_llm_result( + {"files": [{"filename": "utils.py", "description": "utilities"}]} + ) + c = _classified() + result = _propose_files_step( + [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG + ) + prompt = mock_call.call_args[0][6][0]["content"] + assert "already exist" not in prompt + assert result is not None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_with_existing_files(mock_call): + """existing_files non-empty → exclude_section added to prompt (branch True).""" + mock_call.return_value = _make_llm_result( + {"files": [{"filename": "utils.py", "description": "utilities"}]} + ) + c = _classified() + result = _propose_files_step( + [["foo"]], + c, + "src/big.py", + 2, + frozenset({"other.py"}), + MagicMock(), + _CONFIG, + ) + prompt = mock_call.call_args[0][6][0]["content"] + assert "already exist" in prompt + assert result is not None + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_propose_files_step_prev_failure(mock_call): + """prev_failure is appended to the propose prompt.""" + mock_call.return_value = _make_llm_result( + {"files": [{"filename": "utils.py", "description": "utilities"}]} + ) + c = _classified() + _propose_files_step( + [["foo"]], + c, + "src/big.py", + 2, + frozenset(), + MagicMock(), + _CONFIG, + prev_failure="sentinel_propose_failure", + ) + prompt = mock_call.call_args[0][6][0]["content"] + assert "sentinel_propose_failure" in prompt + + +def test_compute_projected_lines_basic(): + """Entities found in map → lines counted per target file.""" + entity_a = _make_entity("func_a", 1, 50) # 50 lines + entity_b = _make_entity("func_b", 51, 100) # 50 lines + entity_map = {"func_a": entity_a, "func_b": entity_b} + placements = [ + GroupPlacement(group=["func_a"], target_file="utils.py"), + GroupPlacement(group=["func_b"], target_file="utils.py"), + ] + projected = _compute_projected_lines(placements, entity_map) + assert projected == {"utils.py": 100} + + +def test_compute_projected_lines_unknown_entity(): + """Entity name not in map → no lines added for that entity (skipped).""" + entity_map = {} # nothing in the map + placements = [GroupPlacement(group=["ghost"], target_file="utils.py")] + projected = _compute_projected_lines(placements, entity_map) + assert projected == {} + + +def test_compute_projected_lines_multiple_files(): + """Entities across multiple target files → separate line counts.""" + entity_a = _make_entity("func_a", 1, 100) # 100 lines + entity_b = _make_entity("func_b", 101, 200) # 100 lines + entity_map = {"func_a": entity_a, "func_b": entity_b} + placements = [ + GroupPlacement(group=["func_a"], target_file="module_a.py"), + GroupPlacement(group=["func_b"], target_file="module_b.py"), + ] + projected = _compute_projected_lines(placements, entity_map) + assert projected == {"module_a.py": 100, "module_b.py": 100} + + +def test_refine_merge_tiny_no_tiny_files(): + """All projected files are above the tiny threshold → no merge, return unchanged.""" + # Entity with 300 lines is well above min_size (200 for 1000-line limit). + entity = _make_entity("large_func", 1, 300) + c = _classified(entities=[entity]) + placements = [GroupPlacement(group=["large_func"], target_file="utils.py")] + proposed_files = [("utils.py", "large functions"), ("models.py", "models")] + + result = _refine_merge_tiny( + placements, proposed_files, c, "src/big.py", MagicMock(), _CONFIG + ) + assert result == placements + assert result is not placements + + +def test_refine_merge_tiny_no_ok_proposed(): + """All proposed files are tiny → ok_proposed is empty → return unchanged.""" + # Two tiny entities, both below threshold. + entity_a = _make_entity("tiny_a", 1, 10) + entity_b = _make_entity("tiny_b", 11, 20) + c = _classified(entities=[entity_a, entity_b]) + placements = [ + GroupPlacement(group=["tiny_a"], target_file="a.py"), + GroupPlacement(group=["tiny_b"], target_file="b.py"), + ] + proposed_files = [("a.py", "tiny a"), ("b.py", "tiny b")] + # Both files are tiny (10 and 10 lines < 200); no ok_proposed → no merge. + + result = _refine_merge_tiny( + placements, proposed_files, c, "src/big.py", MagicMock(), _CONFIG + ) + assert result == placements + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_refine_merge_tiny_success(mock_call): + """Tiny file group is merged into a larger file successfully.""" + # LLM reassigns the tiny group to the large file. + mock_call.return_value = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "large.py"}]} + ) + + entity_small = _make_entity("small_func", 1, 10) # 10 lines (tiny) + entity_large = _make_entity("large_func", 11, 310) # 300 lines (not tiny) + c = _classified(entities=[entity_small, entity_large]) + + placements = [ + GroupPlacement(group=["small_func"], target_file="small.py"), + GroupPlacement(group=["large_func"], target_file="large.py"), + ] + proposed_files = [("small.py", "small"), ("large.py", "large")] + # small.py: 10 lines (tiny <200); large.py: 300 lines (not tiny). + # ok_proposed = [("large.py", "large")]; refinement merges small.py into large.py. + + result = _refine_merge_tiny( + placements, proposed_files, c, "src/big.py", MagicMock(), _CONFIG + ) + assert len(result) == 2 + # small_func should now be in large.py. + small_placement = next(r for r in result if "small_func" in r.group) + assert small_placement.target_file == "large.py" + # large_func remains in large.py. + large_placement = next(r for r in result if "large_func" in r.group) + assert large_placement.target_file == "large.py" + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_refine_merge_tiny_llm_fails(mock_call): + """Reassignment LLM returns None → original placements returned (best-effort).""" + mock_call.return_value = _make_llm_result(None) # LLM fails + + entity_small = _make_entity("small_func", 1, 10) + entity_large = _make_entity("large_func", 11, 310) + c = _classified(entities=[entity_small, entity_large]) + + placements = [ + GroupPlacement(group=["small_func"], target_file="small.py"), + GroupPlacement(group=["large_func"], target_file="large.py"), + ] + proposed_files = [("small.py", "small"), ("large.py", "large")] + + result = _refine_merge_tiny( + placements, proposed_files, c, "src/big.py", MagicMock(), _CONFIG + ) + # Best-effort: return original placements unchanged. + assert len(result) == 2 + assert result[0].target_file == "small.py" + assert result[1].target_file == "large.py" + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_refine_merge_tiny_verbose(mock_call, capsys): + """verbose=True prints the refining message to stderr.""" + mock_call.return_value = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "large.py"}]} + ) + + entity_small = _make_entity("small_func", 1, 10) + entity_large = _make_entity("large_func", 11, 310) + c = _classified(entities=[entity_small, entity_large]) + + placements = [ + GroupPlacement(group=["small_func"], target_file="small.py"), + GroupPlacement(group=["large_func"], target_file="large.py"), + ] + proposed_files = [("small.py", "small"), ("large.py", "large")] + + _refine_merge_tiny( + placements, + proposed_files, + c, + "src/big.py", + MagicMock(), + _CONFIG, + verbose=True, + ) + err = capsys.readouterr().err + assert "refining" in err.lower() + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_assign_placements_chunk_existing_files_exclude_section(mock_call): + """Free-form mode with non-empty existing_files builds the exclude section.""" + mock_call.return_value = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "helpers.py"}]} + ) + entity = _make_entity("foo", 1, 10) + c = _classified(entities=[entity], set_2_groups=[["foo"]]) + result = _assign_placements_chunk( + [["foo"]], + c, + "src/big.py", + frozenset({"existing.py"}), # non-empty existing_files + MagicMock(), + _CONFIG, + proposed_files=None, # free-form mode + ) + assert result is not None + assert result[0].target_file == "helpers.py" + + +@patch("crispen.file_limiter.advisor.placement.call_with_tool") +def test_assign_placements_chunk_target_in_existing_files_returns_none(mock_call): + """Free-form mode: target_file in existing_files → return None (line 589).""" + mock_call.return_value = _make_llm_result( + {"placements": [{"group_id": 0, "target_file": "existing.py"}]} + ) + entity = _make_entity("foo", 1, 10) + c = _classified(entities=[entity], set_2_groups=[["foo"]]) + result = _assign_placements_chunk( + [["foo"]], + c, + "src/big.py", + frozenset({"existing.py"}), # target collides with existing file + MagicMock(), + _CONFIG, + proposed_files=None, # free-form mode + ) + assert result is None diff --git a/tests/code_gen/__init__.py b/tests/code_gen/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/code_gen/generate_file_splits/__init__.py b/tests/code_gen/generate_file_splits/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/code_gen/generate_file_splits/test_basic_generation.py b/tests/code_gen/generate_file_splits/test_basic_generation.py new file mode 100644 index 0000000..7b94144 --- /dev/null +++ b/tests/code_gen/generate_file_splits/test_basic_generation.py @@ -0,0 +1,304 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.code_gen import ( + _find_main_block_entity, + _find_main_direct_callees, + generate_file_splits, +) +from crispen.file_limiter.entity_parser import Entity, EntityKind +from ..helpers import _classified, _make_entity, _plan + + +def test_generate_empty_placements(): + plan = _plan() # placements=[] + c = _classified() + source = "def foo():\n pass\n" + result = generate_file_splits(c, plan, source, "big.py") + assert result.abort is False + assert result.new_files == {} + assert result.original_source == source + + +def test_generate_single_entity_migration(): + source = "import os\n\ndef foo():\n os.getcwd()\n" + entity = _make_entity("foo", 3, 4) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["foo"], target_file="utils.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + assert result.abort is False + assert "utils.py" in result.new_files + new_src = result.new_files["utils.py"] + assert "import os" in new_src + assert "def foo():" in new_src + # Original should not have foo's def anymore + assert "def foo():" not in result.original_source + # But should have a re-export + assert "from .utils import foo" in result.original_source + + +def test_generate_private_entity_no_reexport(): + source = "def _helper():\n pass\n" + entity = _make_entity("_helper", 1, 2) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["_helper"], target_file="private.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + assert "from .private import" not in result.original_source + + +def test_generate_entity_not_in_source_map(): + # Group has entity name not in classified.entities → entity skipped in new file. + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + c = _classified(entities=[entity]) + # "ghost" is in the group but has no matching entity + plan = _plan([GroupPlacement(group=["foo", "ghost"], target_file="utils.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + assert "utils.py" in result.new_files + # "ghost" produces no source so only "foo" appears + new_src = result.new_files["utils.py"] + assert "def foo():" in new_src + + +def test_generate_no_imports_needed(): + # Entity uses no imports → no import section in new file. + source = "def add(a, b):\n return a + b\n" + entity = _make_entity("add", 1, 2) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["add"], target_file="math_utils.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + new_src = result.new_files["math_utils.py"] + # No "import" prefix expected + assert not new_src.startswith("import") + assert "def add" in new_src + + +def test_generate_multiple_groups_same_file(): + source = textwrap.dedent( + """\ + import os + + def foo(): + pass + + def bar(): + pass + """ + ) + e_foo = _make_entity("foo", 3, 4) + e_bar = _make_entity("bar", 6, 7) + c = _classified(entities=[e_foo, e_bar]) + plan = _plan( + [ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="utils.py"), + ] + ) + result = generate_file_splits(c, plan, source, "big.py") + + new_src = result.new_files["utils.py"] + assert "def foo():" in new_src + assert "def bar():" in new_src + + +def test_generate_multiple_different_target_files(): + source = "def foo():\n pass\n\ndef bar():\n pass\n" + e_foo = _make_entity("foo", 1, 2) + e_bar = _make_entity("bar", 4, 5) + c = _classified(entities=[e_foo, e_bar]) + plan = _plan( + [ + GroupPlacement(group=["foo"], target_file="foo_module.py"), + GroupPlacement(group=["bar"], target_file="bar_module.py"), + ] + ) + result = generate_file_splits(c, plan, source, "big.py") + + assert "foo_module.py" in result.new_files + assert "bar_module.py" in result.new_files + assert "def foo():" in result.new_files["foo_module.py"] + assert "def bar():" in result.new_files["bar_module.py"] + assert "from .bar_module import bar" in result.original_source + assert "from .foo_module import foo" in result.original_source + + +def test_generate_future_import_not_duplicated_when_in_entity_source(): + # Entity source itself contains `from __future__ import annotations` + # (e.g. the _block_1 TOP_LEVEL entity which IS the file's import block). + # It must appear only once at the top of the new file, not again inside + # the entity source, which would cause a SyntaxError. + source = textwrap.dedent( + """\ + from __future__ import annotations + + \"\"\"Module docstring.\"\"\" + + from __future__ import annotations + + import os + + _CONST = 42 + """ + ) + # _block_1 spans the whole file and contains the future import + constants. + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 9, ["_CONST"]) + c = _classified(entities=[e_block]) + plan = _plan([GroupPlacement(group=["_block_1"], target_file="constants.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + new_src = result.new_files["constants.py"] + assert new_src.count("from __future__ import annotations") == 1 + # Must be at the very start of the file (before any other code). + first_non_blank = next(line for line in new_src.splitlines() if line.strip()) + assert first_non_blank == "from __future__ import annotations" + + +def test_generate_future_import_always_included(): + source = "from __future__ import annotations\n\ndef foo():\n pass\n" + entity = _make_entity("foo", 3, 4) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["foo"], target_file="utils.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + new_src = result.new_files["utils.py"] + assert "from __future__ import annotations" in new_src + + +def test_find_main_block_entity_present(): + from crispen.file_limiter.entity_parser import parse_entities + + source = textwrap.dedent( + """\ + def run(): + pass + + if __name__ == "__main__": + run() + """ + ) + entities = parse_entities(source) + esmap = {e.name: source.splitlines(keepends=True) for e in entities} + # Rebuild entity_source_map properly + lines = source.splitlines(keepends=True) + esmap = { + e.name: "".join(lines[e.start_line - 1 : e.end_line]).rstrip() for e in entities + } + result = _find_main_block_entity(entities, esmap) + assert result is not None + assert result.startswith("_block_") + + +def test_find_main_block_entity_absent(): + from crispen.file_limiter.entity_parser import parse_entities + + source = "def foo():\n pass\n" + entities = parse_entities(source) + lines = source.splitlines(keepends=True) + esmap = { + e.name: "".join(lines[e.start_line - 1 : e.end_line]).rstrip() for e in entities + } + assert _find_main_block_entity(entities, esmap) is None + + +def test_find_main_block_entity_syntax_error_skipped(): + + # Entity whose source is invalid Python: should be skipped gracefully. + entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, []) + result = _find_main_block_entity([entity], {"_block_1": "def (invalid"}) + assert result is None + + +def test_find_main_direct_callees_basic(): + src = 'if __name__ == "__main__":\n run_tests()\n' + callees = _find_main_direct_callees(src, {"run_tests", "other"}) + assert callees == {"run_tests"} + + +def test_find_main_direct_callees_not_in_entity_names(): + src = 'if __name__ == "__main__":\n unknown()\n' + callees = _find_main_direct_callees(src, {"run_tests"}) + assert callees == set() + + +def test_find_main_direct_callees_syntax_error(): + assert _find_main_direct_callees("def (invalid", {"foo"}) == set() + + +def test_find_main_direct_callees_no_main_block(): + src = "run_tests()\n" + assert _find_main_direct_callees(src, {"run_tests"}) == set() + + +def test_generate_main_block_stays_in_original(): + source = textwrap.dedent( + """\ + def run(): + pass + + def other(): + pass + + if __name__ == "__main__": + run() + """ + ) + e_run = Entity(EntityKind.FUNCTION, "run", 1, 2, ["run"]) + e_other = Entity(EntityKind.FUNCTION, "other", 4, 5, ["other"]) + e_main = Entity(EntityKind.TOP_LEVEL, "_block_7", 7, 8, []) + c = _classified(entities=[e_run, e_other, e_main]) + # Plan tries to migrate run + __main__ block and other. + plan = _plan( + [ + GroupPlacement(group=["run", "_block_7"], target_file="helpers.py"), + GroupPlacement(group=["other"], target_file="helpers.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + # __main__ block stays in original. + assert 'if __name__ == "__main__"' in result.original_source + assert 'if __name__ == "__main__"' not in result.new_files.get("helpers.py", "") + + +def test_generate_main_callee_stays_in_original(): + source = textwrap.dedent( + """\ + def run(): + pass + + def other(): + pass + + if __name__ == "__main__": + run() + """ + ) + e_run = Entity(EntityKind.FUNCTION, "run", 1, 2, ["run"]) + e_other = Entity(EntityKind.FUNCTION, "other", 4, 5, ["other"]) + e_main = Entity(EntityKind.TOP_LEVEL, "_block_7", 7, 8, []) + c = _classified(entities=[e_run, e_other, e_main]) + # Plan tries to migrate run (the direct callee of __main__). + plan = _plan( + [ + GroupPlacement(group=["run"], target_file="helpers.py"), + GroupPlacement(group=["other"], target_file="helpers.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + # run() is a direct __main__ callee — must stay in original. + assert "def run():" in result.original_source + # other() is not a callee — may be migrated. + assert "helpers.py" in result.new_files diff --git a/tests/code_gen/generate_file_splits/test_cross_file_imports.py b/tests/code_gen/generate_file_splits/test_cross_file_imports.py new file mode 100644 index 0000000..6a43028 --- /dev/null +++ b/tests/code_gen/generate_file_splits/test_cross_file_imports.py @@ -0,0 +1,346 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.code_gen import generate_file_splits +from crispen.file_limiter.entity_parser import Entity, EntityKind +from ..helpers import _classified, _make_entity, _plan + + +def test_generate_cross_file_import(): + # fn_a goes to fn_module.py; _block_1 (defining _CONST) goes to constants.py. + # _CONST is a TOP_LEVEL variable that is never reassigned → fn_module.py uses + # a plain "from .constants import _CONST" (idiomatic Python; no module alias). + source = "_CONST = 42\n\ndef fn_a():\n return _CONST\n" + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) + e_fn = _make_entity("fn_a", 3, 4) + c = _classified(entities=[e_block, e_fn]) + plan = _plan( + [ + GroupPlacement(group=["_block_1"], target_file="constants.py"), + GroupPlacement(group=["fn_a"], target_file="fn_module.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + fn_src = result.new_files["fn_module.py"] + assert "from .constants import _CONST" in fn_src + assert "from . import constants" not in fn_src + assert "constants._CONST" not in fn_src + # constants.py should NOT have a cross-import (it defines _CONST, not uses it) + const_src = result.new_files["constants.py"] + assert "from .fn_module" not in const_src + + +def test_generate_cross_file_import_no_duplicate_names(): + # Two entities (fn_a and fn_b) migrate to the same new file. + # fn_a uses X and Z from helpers; fn_b uses Y and Z from helpers. + # X, Y, Z are TOP_LEVEL variables that are never reassigned → the new file + # gets ONE "from .constants import X, Y, Z" (no module alias needed). + source = textwrap.dedent( + """\ + X = 1 + Y = 2 + Z = 3 + + def fn_a(): + return X + Z + + def fn_b(): + return Y + Z + """ + ) + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["X", "Y", "Z"]) + e_a = _make_entity("fn_a", 5, 6) + e_b = _make_entity("fn_b", 8, 9) + c = _classified(entities=[e_block, e_a, e_b]) + plan = _plan( + [ + GroupPlacement(group=["_block_1"], target_file="constants.py"), + GroupPlacement(group=["fn_a", "fn_b"], target_file="funcs.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + funcs_src = result.new_files["funcs.py"] + # Both fn_a and fn_b are present + assert "def fn_a" in funcs_src + assert "def fn_b" in funcs_src + # Plain from-import (no module alias) since none of X/Y/Z are reassigned + assert "from .constants import" in funcs_src + assert "from . import constants" not in funcs_src + # Variables are referenced by their bare names, not as module attributes + assert "constants.X" not in funcs_src + assert "constants.Y" not in funcs_src + assert "constants.Z" not in funcs_src + + +def test_generate_cross_file_import_reassigned_uses_module_alias(): + # _CONST is defined by _block_1 (→ constants.py) AND reassigned by _block_2 + # (non-migrated, stays in big.py). Because _CONST is stored by a different + # entity, fn_module.py must use the module-alias form so that any mutation of + # _CONST propagates through the module reference rather than a stale copy. + source = textwrap.dedent( + """\ + _CONST = 42 + _CONST = int("99") + + def fn_a(): + return _CONST + """ + ) + e_block1 = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) + e_block2 = Entity(EntityKind.TOP_LEVEL, "_block_2", 2, 2, ["_CONST"]) + e_fn = _make_entity("fn_a", 4, 5) + c = _classified(entities=[e_block1, e_block2, e_fn]) + plan = _plan( + [ + GroupPlacement(group=["_block_1"], target_file="constants.py"), + GroupPlacement(group=["fn_a"], target_file="fn_module.py"), + # _block_2 stays (non-migrated) — its store makes _CONST "reassigned" + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + fn_src = result.new_files["fn_module.py"] + # _CONST is reassigned → module-alias import so mutations propagate. + assert "from . import constants" in fn_src + assert "constants._CONST" in fn_src + assert "from .constants import _CONST" not in fn_src + + +def test_generate_cross_file_reassigned_original_file_uses_module_alias(): + # _CONST is defined by _block_1 (migrated) and reassigned by _block_2 + # (non-migrated). + # The original file must rewrite both the load in fn_a AND the module-level + # store in _block_2 to constants._CONST so that the reassignment updates the + # value in constants.py rather than creating an orphaned local binding. + source = textwrap.dedent( + """\ + _CONST = 42 + _CONST = int("99") + + def fn_a(): + return _CONST + """ + ) + e_block1 = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) + e_block2 = Entity(EntityKind.TOP_LEVEL, "_block_2", 2, 2, ["_CONST"]) + e_fn = _make_entity("fn_a", 4, 5) + c = _classified(entities=[e_block1, e_block2, e_fn]) + plan = _plan( + [ + GroupPlacement(group=["_block_1"], target_file="constants.py"), + # _block_2 and fn_a stay (non-migrated) + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + assert not result.abort + orig = result.original_source + # Module-level import added for the module alias. + assert "from . import constants" in orig + # Both the store (_block_2) and the load (fn_a) are rewritten. + assert 'constants._CONST = int("99")' in orig + assert "return constants._CONST" in orig + # Must NOT bind _CONST as a bare name via from-import (would shadow the rewrite) + assert "from .constants import _CONST" not in orig + + +def test_generate_reassigned_all_entities_migrated_no_original_processing(): + # When ALL entities are migrated, non_migrated_entity_names is empty and the + # original-file module-alias processing block must be skipped without error. + source = "_CONST = 42\n_CONST = 99\n\ndef fn_a():\n return _CONST\n" + e_block1 = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) + e_block2 = Entity(EntityKind.TOP_LEVEL, "_block_2", 2, 2, ["_CONST"]) + e_fn = _make_entity("fn_a", 4, 5) + c = _classified(entities=[e_block1, e_block2, e_fn]) + plan = _plan( + [ + GroupPlacement(group=["_block_1"], target_file="constants.py"), + GroupPlacement(group=["_block_2", "fn_a"], target_file="funcs.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + # Does not abort or crash; original source may be minimal. + assert not result.abort + + +def test_generate_reassigned_two_entities_same_file_single_module_import(): + # Two entities in the same new file both reference a reassigned constant. + # The same "from . import constants" import must appear only once + # (seen_top_cross deduplication). + source = textwrap.dedent( + """\ + _CONST = 42 + _CONST = 99 + + def fn_a(): + return _CONST + + def fn_b(): + return _CONST + """ + ) + e_block1 = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) + e_block2 = Entity(EntityKind.TOP_LEVEL, "_block_2", 2, 2, ["_CONST"]) + e_fn_a = _make_entity("fn_a", 4, 5) + e_fn_b = _make_entity("fn_b", 7, 8) + c = _classified(entities=[e_block1, e_block2, e_fn_a, e_fn_b]) + plan = _plan( + [ + GroupPlacement(group=["_block_1"], target_file="constants.py"), + GroupPlacement(group=["fn_a", "fn_b"], target_file="funcs.py"), + # _block_2 stays non-migrated → makes _CONST "reassigned" + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + funcs_src = result.new_files["funcs.py"] + # The module import must appear exactly once despite two entities needing it. + import_lines = [ln for ln in funcs_src.splitlines() if "import constants" in ln] + assert len(import_lines) == 1 + + +def test_generate_aborts_on_cross_file_import_cycle(): + # fn_a references fn_b (in b.py) and fn_b references fn_a (in a.py). + # This creates a circular import a.py ↔ b.py that Python cannot load. + # generate_file_splits must detect the cycle and abort rather than emit + # broken code. + source = "def fn_a():\n return fn_b()\n\ndef fn_b():\n return fn_a()\n" + e_a = _make_entity("fn_a", 1, 2) + e_b = _make_entity("fn_b", 4, 5) + c = _classified(entities=[e_a, e_b]) + plan = _plan( + [ + GroupPlacement(group=["fn_a"], target_file="a.py"), + GroupPlacement(group=["fn_b"], target_file="b.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + assert result.abort is True + assert result.new_files == {} + + +def test_generate_aborts_on_cycle_through_original(): + # _CONST is a TOP_LEVEL constant (stays in original). + # _worker is migrated to helpers.py and references _CONST. + # main() (non-migrated) calls _worker → original will re-export _worker. + # Cycle: original → helpers.py (re-export of _worker) + # → original (via `from .original import _CONST`). + source = textwrap.dedent( + """\ + _CONST = "value" + + def _worker(): + return _CONST + + def main(): + return _worker() + """ + ) + e_const = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) + e_worker = _make_entity("_worker", 3, 4) + e_main = _make_entity("main", 6, 7) + c = _classified(entities=[e_const, e_worker, e_main]) + plan = _plan([GroupPlacement(group=["_worker"], target_file="helpers.py")]) + + result = generate_file_splits(c, plan, source, "original.py") + + # helpers.py would need `from .original import _CONST` while original + # re-exports _worker from helpers.py → circular import → must abort. + assert result.abort is True + assert result.new_files == {} + + +def test_generate_aborts_on_cycle_through_original_test_subdir(): + # In a test-file subdir split non_migrated_home ("test_svc.py") differs + # from original_basename ("svc/__init__.py"). The cycle detection must + # treat the original test file as its own graph node: + # + # _CONFIG stays in test_svc.py (TOP_LEVEL, non-migrated). + # _helper is migrated to svc/test_helpers.py and references _CONFIG. + # test_fn (non-migrated) calls _helper → test_svc.py re-exports _helper. + # Cycle: test_svc.py → svc/test_helpers.py (re-export of _helper) + # → test_svc.py (via `from ..test_svc import _CONFIG`). + source = textwrap.dedent( + """\ + _CONFIG = "value" + + def _helper(): + return _CONFIG + + def test_fn(): + return _helper() + """ + ) + e_config = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONFIG"]) + e_helper = _make_entity("_helper", 3, 4) + e_test = _make_entity("test_fn", 6, 7) + c = _classified(entities=[e_config, e_helper, e_test]) + plan = _plan([GroupPlacement(group=["_helper"], target_file="svc/test_helpers.py")]) + + result = generate_file_splits( + c, plan, source, "tests/test_svc.py", subdir_name="svc" + ) + + # svc/test_helpers.py imports _CONFIG from test_svc.py, and test_svc.py + # re-exports _helper from svc/test_helpers.py → circular import → abort. + assert result.abort is True + assert result.new_files == {} + + +def test_generate_test_file_reexports_use_absolute_imports(tmp_path): + # When the original is a test file, re-exports in the updated original + # must use absolute imports so pytest can load the file. + (tmp_path / "pyproject.toml").touch() + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + test_file = tests_dir / "test_engine.py" + test_file.write_text("") + + source = "import os\n\ndef foo():\n os.getcwd()\n" + entity = _make_entity("foo", 3, 4) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["foo"], target_file="test_helpers.py")]) + + result = generate_file_splits(c, plan, source, str(test_file)) + + assert "from tests.test_helpers import foo" in result.original_source + assert "from .test_helpers import foo" not in result.original_source + + +def test_generate_test_file_cross_imports_use_absolute_imports(tmp_path): + # Cross-file imports in generated test split files must also be absolute. + (tmp_path / "pyproject.toml").touch() + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + test_file = tests_dir / "test_engine.py" + test_file.write_text("") + + source = "_CONST = 42\n\ndef test_fn():\n return _CONST\n" + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) + e_fn = _make_entity("test_fn", 3, 4) + c = _classified(entities=[e_block, e_fn]) + plan = _plan( + [ + GroupPlacement(group=["_block_1"], target_file="test_constants.py"), + GroupPlacement(group=["test_fn"], target_file="test_fns.py"), + ] + ) + + result = generate_file_splits(c, plan, source, str(test_file)) + + fn_src = result.new_files["test_fns.py"] + # _CONST is a TOP_LEVEL variable that is never reassigned → plain absolute + # from-import (idiomatic Python; module alias only needed if reassigned). + assert "from tests.test_constants import _CONST" in fn_src + assert "import tests.test_constants as test_constants" not in fn_src + assert "test_constants._CONST" not in fn_src diff --git a/tests/code_gen/generate_file_splits/test_edge_cases.py b/tests/code_gen/generate_file_splits/test_edge_cases.py new file mode 100644 index 0000000..8c5553b --- /dev/null +++ b/tests/code_gen/generate_file_splits/test_edge_cases.py @@ -0,0 +1,162 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.code_gen import generate_file_splits +from crispen.file_limiter.entity_parser import Entity, EntityKind +from ..helpers import _abort_plan, _classified, _make_entity, _plan + + +def test_generate_abort_plan(): + plan = _abort_plan() + c = _classified() + result = generate_file_splits(c, plan, "def foo():\n pass\n", "big.py") + assert result.abort is True + assert result.new_files == {} + assert result.original_source == "def foo():\n pass\n" + + +def test_generate_aborts_when_test_class_used_in_decorator(): + # TestFixture (a Test* class) provides PARAMS used in a parametrize decorator + # on test_fn. If they are split into different files, TestFixture would need + # to be imported inline (to avoid pytest duplicate collection), but that + # import would not be in scope when the decorator is evaluated. + source = textwrap.dedent( + """\ + import pytest + + class TestFixture: + PARAMS = [1, 2, 3] + + @pytest.mark.parametrize("x", TestFixture.PARAMS) + def test_fn(x): + assert x + """ + ) + e_fixture = Entity(EntityKind.CLASS, "TestFixture", 3, 4, ["TestFixture"]) + e_fn = _make_entity("test_fn", 6, 8) + c = _classified(entities=[e_fixture, e_fn]) + plan = _plan( + [ + GroupPlacement(group=["TestFixture"], target_file="test_fixture.py"), + GroupPlacement(group=["test_fn"], target_file="test_fns.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "tests/test_original.py") + + assert result.abort + assert "TestFixture" in result.abort_reason + assert "decorator" in result.abort_reason + + +def test_generate_non_migrated_helper_extracted_to_new_file(): + # _run is non-migrated; test_fn is migrated and references _run. + # _run is extracted into test_helpers.py to prevent an O→F→O cycle. + source = textwrap.dedent( + """\ + import textwrap + + def _run(x): + return x + + def test_fn(): + return _run(1) + """ + ) + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["textwrap"]) + e_run = _make_entity("_run", 3, 4) + e_test = _make_entity("test_fn", 6, 7) + c = _classified(entities=[e_block, e_run, e_test]) + plan = _plan([GroupPlacement(group=["test_fn"], target_file="test_helpers.py")]) + + result = generate_file_splits(c, plan, source, "original.py") + + new_src = result.new_files["test_helpers.py"] + # _run is defined in the new file (extracted), not imported from original + assert "def _run" in new_src + assert "from .original import _run" not in new_src + # import textwrap is not referenced by either entity + assert "from .original import textwrap" not in new_src + + +def test_generate_self_referential_placement_dropped(): + # LLM names a target file the same as the original → would create a + # circular import. The placement must be silently dropped so the entity + # stays in the original file and no self-import is added. + source = "class Foo:\n pass\n\nclass Bar:\n pass\n" + e_foo = _make_entity("Foo", 1, 2) + e_bar = _make_entity("Bar", 4, 5) + c = _classified(entities=[e_foo, e_bar]) + # "mymodule.py" is also the original filename → self-referential + plan = _plan( + [ + GroupPlacement(group=["Foo"], target_file="mymodule.py"), + GroupPlacement(group=["Bar"], target_file="helpers.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "mymodule.py") + + # Foo stays in the original — no circular self-import + assert "from .mymodule import Foo" not in result.original_source + assert "mymodule.py" not in result.new_files + # Bar is still moved normally + assert "helpers.py" in result.new_files + assert "class Bar" in result.new_files["helpers.py"] + # Foo remains in the original source (not removed) + assert "class Foo" in result.original_source + + +def test_generate_all_placements_self_referential(): + # All placements target the original file → nothing is moved. + source = "def foo():\n pass\n" + e_foo = _make_entity("foo", 1, 2) + c = _classified(entities=[e_foo]) + plan = _plan([GroupPlacement(group=["foo"], target_file="original.py")]) + + result = generate_file_splits(c, plan, source, "original.py") + + assert result.new_files == {} + assert "from .original import foo" not in result.original_source + assert "def foo" in result.original_source + + +def test_generate_shebang_stripped_from_new_file(): + # Shebang on line 1 should NOT appear in generated new files. + source = "#!/usr/bin/env python3\n\ndef foo():\n pass\n\ndef bar():\n foo()\n" + e_foo = Entity(EntityKind.FUNCTION, "foo", 3, 4, ["foo"]) + e_bar = Entity(EntityKind.FUNCTION, "bar", 6, 7, ["bar"]) + c = _classified(entities=[e_foo, e_bar]) + plan = _plan([GroupPlacement(group=["bar"], target_file="helpers.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + assert "#!/usr/bin/env python3" not in result.new_files["helpers.py"] + + +def test_generate_shebang_preserved_in_original_when_entity_migrated(): + # When the entity owning line 1 (with shebang comment) is migrated, + # the shebang must be restored at the top of the original file. + source = "#!/usr/bin/env python3\ndef foo():\n pass\n\ndef bar():\n pass\n" + e_foo = Entity(EntityKind.FUNCTION, "foo", 1, 3, ["foo"]) + e_bar = Entity(EntityKind.FUNCTION, "bar", 5, 6, ["bar"]) + c = _classified(entities=[e_foo, e_bar]) + plan = _plan([GroupPlacement(group=["foo"], target_file="helpers.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + assert result.original_source.startswith("#!/usr/bin/env python3\n") + assert "#!/usr/bin/env python3" not in result.new_files["helpers.py"] + + +def test_generate_shebang_preserved_when_not_migrated(): + # When the shebang entity stays in the original, shebang remains at top. + source = "#!/usr/bin/env python3\ndef foo():\n pass\n\ndef bar():\n pass\n" + e_foo = Entity(EntityKind.FUNCTION, "foo", 1, 3, ["foo"]) + e_bar = Entity(EntityKind.FUNCTION, "bar", 5, 6, ["bar"]) + c = _classified(entities=[e_foo, e_bar]) + plan = _plan([GroupPlacement(group=["bar"], target_file="helpers.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + assert result.original_source.startswith("#!/usr/bin/env python3\n") diff --git a/tests/code_gen/generate_file_splits/test_import_management.py b/tests/code_gen/generate_file_splits/test_import_management.py new file mode 100644 index 0000000..6fcf7c7 --- /dev/null +++ b/tests/code_gen/generate_file_splits/test_import_management.py @@ -0,0 +1,258 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.code_gen import generate_file_splits +from crispen.file_limiter.entity_parser import Entity, EntityKind +from ..helpers import _classified, _make_entity, _plan + + +def test_generate_file_splits_type_checking_for_quoted_annotation(): + # _advise_set3 uses Optional["_LLMAccumulator"] (quoted annotation). + # _LLMAccumulator is migrated to models.py; _advise_set3 goes to placements.py. + # placements.py must get: + # from typing import TYPE_CHECKING + # if TYPE_CHECKING: + # from .models import _LLMAccumulator + source = textwrap.dedent( + """\ + from typing import Optional + + class _LLMAccumulator: + pass + + def _advise_set3(acc: Optional["_LLMAccumulator"]) -> None: + pass + """ + ) + e_acc = Entity(EntityKind.CLASS, "_LLMAccumulator", 3, 4, ["_LLMAccumulator"]) + e_fn = Entity(EntityKind.FUNCTION, "_advise_set3", 6, 7, ["_advise_set3"]) + c = _classified(entities=[e_acc, e_fn]) + plan = _plan( + [ + GroupPlacement(group=["_LLMAccumulator"], target_file="models.py"), + GroupPlacement(group=["_advise_set3"], target_file="placements.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "advisor.py") + + placements_src = result.new_files["placements.py"] + # TYPE_CHECKING may be merged into an existing "from typing import ..." line. + assert "TYPE_CHECKING" in placements_src + assert "if TYPE_CHECKING:" in placements_src + assert "from .models import _LLMAccumulator" in placements_src + + +def test_generate_file_splits_type_checking_deduplication(): + # Two functions in the same target file both reference "_LLMAccumulator" + # in quoted annotations. The TYPE_CHECKING import should appear only once + # even though both entities trigger _find_cross_file_type_checking_imports. + source = textwrap.dedent( + """\ + from typing import Optional + + class _LLMAccumulator: + pass + + def _fn_a(x: Optional["_LLMAccumulator"]) -> None: + pass + + def _fn_b(y: Optional["_LLMAccumulator"]) -> None: + pass + """ + ) + e_acc = Entity(EntityKind.CLASS, "_LLMAccumulator", 3, 4, ["_LLMAccumulator"]) + e_fna = Entity(EntityKind.FUNCTION, "_fn_a", 6, 7, ["_fn_a"]) + e_fnb = Entity(EntityKind.FUNCTION, "_fn_b", 9, 10, ["_fn_b"]) + c = _classified(entities=[e_acc, e_fna, e_fnb]) + plan = _plan( + [ + GroupPlacement(group=["_LLMAccumulator"], target_file="models.py"), + GroupPlacement(group=["_fn_a", "_fn_b"], target_file="placements.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "advisor.py") + + placements_src = result.new_files["placements.py"] + assert placements_src.count("from .models import _LLMAccumulator") == 1 + + +def test_generate_file_splits_tc_dedup_drops_when_already_in_regular(): + # Entity A uses _Acc at runtime (unquoted annotation → regular cross-file import). + # Entity B uses _Acc only in a quoted annotation → would normally get a TC import. + # Both go to workers.py. The dedup step must remove the TC import entirely since + # _Acc is already covered by the regular import. + source = textwrap.dedent( + """\ + from typing import Optional + + class _Acc: + pass + + def fn_runtime(x) -> None: + a: _Acc = x + + def fn_quoted(x: Optional["_Acc"]) -> None: + pass + """ + ) + e_acc = Entity(EntityKind.CLASS, "_Acc", 3, 4, ["_Acc"]) + e_rt = Entity(EntityKind.FUNCTION, "fn_runtime", 6, 7, ["fn_runtime"]) + e_qt = Entity(EntityKind.FUNCTION, "fn_quoted", 9, 10, ["fn_quoted"]) + c = _classified(entities=[e_acc, e_rt, e_qt]) + plan = _plan( + [ + GroupPlacement(group=["_Acc"], target_file="models.py"), + GroupPlacement(group=["fn_runtime", "fn_quoted"], target_file="workers.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "advisor.py") + + workers_src = result.new_files["workers.py"] + # Regular import must be present, TYPE_CHECKING block must NOT be. + assert "from .models import _Acc" in workers_src + assert "if TYPE_CHECKING:" not in workers_src + + +def test_generate_file_splits_tc_dedup_plain_import_branches(): + # Covers the non-from-import branches in the dedup loop: + # • "import sys" in needed → _FROM_IMPORT_RE does not match (2633->2631 branch) + # • "import typing_extensions" in needed_tc (annotation-only) → TC import is a + # plain import statement, not a from-import (2655 branch) + source = textwrap.dedent( + """\ + import sys + import typing_extensions + from typing import Optional + + class _Acc: + pass + + def fn(x: Optional["_Acc"]) -> None: + sys.exit(0) + + def fn2() -> "typing_extensions.Literal": + pass + """ + ) + e_acc = Entity(EntityKind.CLASS, "_Acc", 5, 6, ["_Acc"]) + e_fn = Entity(EntityKind.FUNCTION, "fn", 8, 9, ["fn"]) + e_fn2 = Entity(EntityKind.FUNCTION, "fn2", 11, 12, ["fn2"]) + c = _classified(entities=[e_acc, e_fn, e_fn2]) + plan = _plan( + [ + GroupPlacement(group=["_Acc"], target_file="models.py"), + GroupPlacement(group=["fn", "fn2"], target_file="workers.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "advisor.py") + + workers_src = result.new_files["workers.py"] + # TC import for _Acc (cross-file, quoted annotation) must still be present. + assert "if TYPE_CHECKING:" in workers_src + assert "_Acc" in workers_src + # Plain import for typing_extensions preserved in TC block. + assert "typing_extensions" in workers_src + + +def test_generate_prunes_unused_names_from_multiname_import(): + # foo uses only List, not Dict; the new file's import should be narrowed. + source = "from typing import Dict, List\n\ndef foo(x: List):\n return x\n" + entity = _make_entity("foo", 3, 4) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["foo"], target_file="utils.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + new_src = result.new_files["utils.py"] + assert "from typing import List" in new_src + assert "Dict" not in new_src + + +def test_generate_prunes_fully_unused_import_from_original(): + # import os is only used by foo; after foo migrates the original no longer + # needs os, so the import should be removed. + source = "import os\n\ndef foo():\n os.getcwd()\n\ndef bar():\n return 1\n" + e_foo = _make_entity("foo", 3, 4) + e_bar = _make_entity("bar", 6, 7) + c = _classified(entities=[e_foo, e_bar]) + plan = _plan([GroupPlacement(group=["foo"], target_file="utils.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + assert "from .utils import foo" in result.original_source + assert "import os" not in result.original_source + assert "def bar():" in result.original_source + + +def test_generate_narrows_partial_unused_import_in_original(): + # foo uses Dict; bar uses List. After foo migrates, Dict should be + # stripped from the original's import while List is kept. + source = ( + "from typing import Dict, List\n\n" + "def foo(x: Dict):\n return x\n\n" + "def bar(x: List):\n return x\n" + ) + e_foo = _make_entity("foo", 3, 4) + e_bar = _make_entity("bar", 6, 7) + c = _classified(entities=[e_foo, e_bar]) + plan = _plan([GroupPlacement(group=["foo"], target_file="utils.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + assert "from typing import List" in result.original_source + assert "Dict" not in result.original_source + + +def test_generate_migrated_top_level_import_names_not_in_cross_file_imports(): + # Regression: when a TOP_LEVEL entity containing "from dataclasses import + # dataclass" is migrated, the name "dataclass" must NOT be added to the + # name→target-file map. A FUNCTION entity in a separate new file that also + # uses dataclass should get "from dataclasses import dataclass" (via + # _find_needed_imports) rather than "from .constants import dataclass" (a + # spurious cross-file import that would fail at runtime because constants.py + # never exports dataclass). + source = ( + "from dataclasses import dataclass\n\n" + "_CONST = 42\n\n" + "def make():\n return dataclass\n" + ) + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["dataclass", "_CONST"]) + e_make = _make_entity("make", 5, 6) + c = _classified(entities=[e_block, e_make]) + plan = _plan( + [ + GroupPlacement(group=["_block_1"], target_file="constants.py"), + GroupPlacement(group=["make"], target_file="utils.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + utils_src = result.new_files["utils.py"] + # Must import dataclass from the stdlib, not from constants.py + assert "from dataclasses import dataclass" in utils_src + assert "from .constants import dataclass" not in utils_src + + +def test_generate_top_level_entity_imports_not_duplicated(): + # When a TOP_LEVEL entity source contains regular imports (e.g. `import os`) + # those must NOT appear twice in the generated file: once from + # _find_needed_imports and again from the entity source itself. + source = "import os\n\n_CONST = os.sep\n\ndef foo():\n return os.getcwd()\n" + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["os", "_CONST"]) + e_foo = _make_entity("foo", 5, 6) + c = _classified(entities=[e_block, e_foo]) + plan = _plan( + [ + GroupPlacement(group=["_block_1", "foo"], target_file="utils.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + new_src = result.new_files["utils.py"] + assert new_src.count("import os") == 1 diff --git a/tests/code_gen/helpers.py b/tests/code_gen/helpers.py new file mode 100644 index 0000000..325f78e --- /dev/null +++ b/tests/code_gen/helpers.py @@ -0,0 +1,46 @@ +from __future__ import annotations +from crispen.file_limiter.advisor import FileLimiterPlan +from crispen.file_limiter.classifier import ClassifiedEntities +from crispen.file_limiter.entity_parser import Entity, EntityKind + + +def _make_entity(name: str, start: int, end: int, defines=None) -> Entity: + return Entity(EntityKind.FUNCTION, name, start, end, defines or [name]) + + +def _classified( + *, entities=None, set_2_groups=None, set_3_groups=None +) -> ClassifiedEntities: + return ClassifiedEntities( + entities=entities or [], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=set_2_groups or [], + set_3_groups=set_3_groups or [], + abort=False, + ) + + +def _plan(placements=None) -> FileLimiterPlan: + return FileLimiterPlan(set3_migrate=[], placements=placements or [], abort=False) + + +def _abort_plan() -> FileLimiterPlan: + return FileLimiterPlan(set3_migrate=[], placements=[], abort=True) + + +def _make_classified(entities, migrated_names=None): + migrated = set(migrated_names or []) + return ( + ClassifiedEntities( + entities=entities, + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[], + set_3_groups=[], + abort=False, + ), + migrated, + ) diff --git a/tests/code_gen/test_add_re_exports.py b/tests/code_gen/test_add_re_exports.py new file mode 100644 index 0000000..4e31af3 --- /dev/null +++ b/tests/code_gen/test_add_re_exports.py @@ -0,0 +1,732 @@ +from __future__ import annotations +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.code_gen import ( + _add_re_exports, + _class_has_test_methods, + generate_file_splits, +) +from crispen.file_limiter.entity_parser import Entity, EntityKind +from .helpers import _classified, _make_entity, _plan + + +def test_add_re_exports_top_level_import_derived_names_not_re_exported(): + # A TOP_LEVEL entity that includes import statements: the names introduced + # by those imports must NOT appear in re-exports because they are preserved + # in the original file by _remove_entity_lines, not moved to the new file. + source = "import os\n\nMY_CONST\n" # MY_CONST still loaded + entity_src = "from typing import Dict\n\nMY_CONST = 42\n" + entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["Dict", "MY_CONST"]) + placement = GroupPlacement(group=["_block_1"], target_file="constants.py") + result = _add_re_exports( + source, [placement], {"_block_1": entity}, {"_block_1": entity_src} + ) + assert "MY_CONST" in result # assignment-defined name re-exported + assert "Dict" not in result # import-derived name suppressed + + +def test_add_re_exports_all_private_no_change(): + # Private name not called anywhere in remaining source → no import added. + source = "import os\n\ndef _helper():\n pass\n" + entity = _make_entity("_helper", 3, 4) + placement = GroupPlacement(group=["_helper"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"_helper": entity}, {}) + assert result == source + + +def test_add_re_exports_private_referenced_in_source(): + # Private name still called in remaining source → import is added. + source = "import os\n\n_helper()\n" + entity = _make_entity("_helper", 3, 3) + placement = GroupPlacement(group=["_helper"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"_helper": entity}, {}) + assert "from .utils import _helper" in result + + +def test_add_re_exports_public_inserted_after_imports(): + source = "import os\n\ndef foo():\n pass\n" + entity = _make_entity("foo", 3, 4) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"foo": entity}, {}) + assert "from .utils import foo" in result + # Re-export line should come after "import os" + lines = result.splitlines() + import_idx = next(i for i, l in enumerate(lines) if "import os" in l) + reexport_idx = next(i for i, l in enumerate(lines) if "from .utils import foo" in l) + assert reexport_idx > import_idx + + +def test_add_re_exports_no_import_in_source(): + # No imports and no docstring → re-export inserted at beginning. + source = "\ndef foo():\n pass\n" + entity = _make_entity("foo", 2, 3) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"foo": entity}, {}) + assert "from .utils import foo" in result + + +def test_add_re_exports_no_import_with_module_docstring(): + # No imports but module docstring present → re-export inserted after docstring, + # not before it, so the docstring remains the first statement. + source = '"""Module docstring."""\n\n\ndef foo():\n pass\n' + entity = _make_entity("foo", 4, 5) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"foo": entity}, {}) + lines = result.splitlines() + docstring_idx = next( + i for i, l in enumerate(lines) if '"""Module docstring."""' in l + ) + reexport_idx = next(i for i, l in enumerate(lines) if "from .utils import foo" in l) + assert docstring_idx == 0 + assert reexport_idx > docstring_idx + + +def test_add_re_exports_from_import_line(): + # "from pathlib import Path" should be detected as an import line. + source = "from pathlib import Path\n\ndef foo():\n pass\n" + entity = _make_entity("foo", 3, 4) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"foo": entity}, {}) + lines = result.splitlines() + from_import_idx = next( + i for i, l in enumerate(lines) if "from pathlib import Path" in l + ) + reexport_idx = next(i for i, l in enumerate(lines) if "from .utils import foo" in l) + assert reexport_idx > from_import_idx + + +def test_add_re_exports_multiple_targets_sorted(): + source = "import os\n" + e1 = _make_entity("foo", 1, 2) + e2 = _make_entity("bar", 3, 4) + placements = [ + GroupPlacement(group=["foo"], target_file="b_module.py"), + GroupPlacement(group=["bar"], target_file="a_module.py"), + ] + result = _add_re_exports(source, placements, {"foo": e1, "bar": e2}, {}) + # a_module comes before b_module (sorted) + a_idx = result.index("a_module") + b_idx = result.index("b_module") + assert a_idx < b_idx + + +def test_add_re_exports_mixed_public_private(): + source = "import os\n" + entity_map = { + "pub": _make_entity("pub", 1, 2), + "_priv": _make_entity("_priv", 3, 4), + } + placement = GroupPlacement(group=["pub", "_priv"], target_file="utils.py") + result = _add_re_exports(source, [placement], entity_map, {}) + # Only "pub" in re-export, not "_priv" + assert "pub" in result + assert "_priv" not in result + + +def test_add_re_exports_test_function_not_re_exported(): + # test_ functions must never get a proxy import — pytest would discover and + # run them twice (once from the original file, once from the new file). + source = "import os\n" + entity = _make_entity("test_something", 1, 3) + placement = GroupPlacement(group=["test_something"], target_file="tests/helpers.py") + result = _add_re_exports(source, [placement], {"test_something": entity}, {}) + assert result == source + + +def test_add_re_exports_test_function_never_re_exported_even_when_referenced(): + # test_* names are never re-exported at module level even when the + # remaining source references them — _inject_inline_test_imports_original + # handles those cases inline to prevent pytest double-discovery. + source = "import os\n\ntest_something()\n" + entity = _make_entity("test_something", 1, 3) + placement = GroupPlacement(group=["test_something"], target_file="tests/helpers.py") + result = _add_re_exports(source, [placement], {"test_something": entity}, {}) + assert "from .tests.helpers import test_something" not in result + + +def test_class_has_test_methods_true(): + src = "class TestFoo:\n def test_bar(self): pass\n" + assert _class_has_test_methods(src) is True + + +def test_class_has_test_methods_false(): + src = "class Helper:\n def run(self): pass\n" + assert _class_has_test_methods(src) is False + + +def test_class_has_test_methods_syntax_error(): + assert _class_has_test_methods("def (") is False + + +def test_add_re_exports_test_class_not_re_exported(): + # A class that contains test_ methods must not be re-exported — pytest + # would discover it via the original file and the new file, running every + # test twice. + source = "import os\n" + entity = Entity(EntityKind.CLASS, "TestFoo", 1, 5, ["TestFoo"]) + entity_src = "class TestFoo:\n def test_bar(self): pass\n" + placement = GroupPlacement(group=["TestFoo"], target_file="tests/helpers.py") + result = _add_re_exports( + source, [placement], {"TestFoo": entity}, {"TestFoo": entity_src} + ) + assert result == source + + +def test_add_re_exports_test_class_never_re_exported_even_when_referenced(): + # Test-named symbols are never re-exported at module level even when + # referenced in remaining source — _inject_inline_test_imports_original + # handles them inline to prevent pytest double-discovery. + source = "import os\n\nTestFoo()\n" + entity = Entity(EntityKind.CLASS, "TestFoo", 1, 5, ["TestFoo"]) + entity_src = "class TestFoo:\n def test_bar(self): pass\n" + placement = GroupPlacement(group=["TestFoo"], target_file="tests/helpers.py") + result = _add_re_exports( + source, [placement], {"TestFoo": entity}, {"TestFoo": entity_src} + ) + assert "from .tests.helpers import TestFoo" not in result + + +def test_add_re_exports_top_level_block_private_names_referenced(): + # TOP_LEVEL block entity name (_block_1) differs from its defined names. + # Both defined names are still loaded in remaining source → re-imported. + source = "import os\n\n_DUP_SOURCE\n_DUP_RANGES\n" + entity = Entity( + EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_DUP_SOURCE", "_DUP_RANGES"] + ) + placement = GroupPlacement(group=["_block_1"], target_file="test_helpers.py") + result = _add_re_exports(source, [placement], {"_block_1": entity}, {}) + assert "from .test_helpers import _DUP_RANGES, _DUP_SOURCE" in result + + +def test_add_re_exports_entity_not_in_map_falls_back_to_entity_name(): + # Entity name in group is missing from entity_map → falls back to entity name. + source = "import os\n\nghost()\n" # 'ghost' is still referenced + placement = GroupPlacement(group=["ghost"], target_file="utils.py") + result = _add_re_exports(source, [placement], {}, {}) + assert "from .utils import ghost" in result + + +def test_add_re_exports_top_level_block_private_names_not_referenced(): + # TOP_LEVEL block entity whose defined name is private and not used → no import. + source = "import os\n" + entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) + placement = GroupPlacement(group=["_block_1"], target_file="constants.py") + result = _add_re_exports(source, [placement], {"_block_1": entity}, {}) + assert result == source + + +def test_add_re_exports_indented_local_import_not_treated_as_last_import(): + # Functions with local (indented) imports must not cause re-exports to be + # inserted inside the function body. The re-export should appear after the + # top-level "import os" line, not after the indented "from x import y". + source = ( + "import os\n" + "\n" + "def foo():\n" + " from unittest.mock import MagicMock\n" + " MagicMock()\n" + "\n" + "def bar():\n" + " pass\n" + ) + entity = _make_entity("baz", 7, 8) + placement = GroupPlacement(group=["baz"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"baz": entity}, {}) + # Re-export must appear immediately after "import os", not inside foo(). + lines = result.splitlines() + os_idx = next(i for i, l in enumerate(lines) if l == "import os") + reexport_idx = next(i for i, l in enumerate(lines) if "from .utils import baz" in l) + assert reexport_idx == os_idx + 1 + # The function body must remain intact (local import line must still be there). + assert " from unittest.mock import MagicMock" in result + + +def test_add_re_exports_syntax_error_returns_source_unchanged(): + # If the source has a SyntaxError, _add_re_exports cannot determine where + # to insert re-exports and must return the source unchanged. + source = "import os\ndef (invalid\n" + entity = _make_entity("baz", 1, 1) + placement = GroupPlacement(group=["baz"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"baz": entity}, {}) + assert result == source + + +def test_add_re_exports_abs_pkg_package_prefix(): + # abs_pkg="tests" → absolute import: "from tests.utils import foo" + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"foo": entity}, {}, abs_pkg="tests") + assert "from tests.utils import foo" in result + assert "from .utils import foo" not in result + + +def test_add_re_exports_abs_pkg_root_level(): + # abs_pkg="" → root-level absolute import: "from utils import foo" + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"foo": entity}, {}, abs_pkg="") + assert "from utils import foo" in result + assert "from .utils import foo" not in result + + +def test_add_re_exports_private_in_external_loads(): + # Private name not referenced in remaining source but present in external_loads + # → re-export proxy IS added so the external caller continues to work. + source = "import os\n" + entity = _make_entity("_helper", 1, 2) + placement = GroupPlacement(group=["_helper"], target_file="utils.py") + result = _add_re_exports( + source, [placement], {"_helper": entity}, {}, external_loads={"_helper"} + ) + assert "from .utils import _helper" in result + + +def test_add_re_exports_test_function_in_external_loads_not_re_exported(): + # test_ functions must never get a proxy even when listed in external_loads, + # because pytest would discover and run them twice. + source = "import os\n" + entity = _make_entity("test_something", 1, 2) + placement = GroupPlacement(group=["test_something"], target_file="helpers.py") + result = _add_re_exports( + source, + [placement], + {"test_something": entity}, + {}, + external_loads={"test_something"}, + ) + assert result == source + + +def test_add_re_exports_mode_always_public_always_reexported(): + # "always" mode: public names are unconditionally re-exported (current behaviour). + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports( + source, [placement], {"foo": entity}, {}, reexport_mode="always" + ) + assert "from .utils import foo" in result + + +def test_add_re_exports_mode_application_non_test_public_reexported(): + # "application" mode + non-test file: public names are re-exported. + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports( + source, + [placement], + {"foo": entity}, + {}, + reexport_mode="application", + is_test_file=False, + ) + assert "from .utils import foo" in result + + +def test_add_re_exports_mode_application_test_file_public_not_reexported(): + # "application" mode + test file: public names are NOT unconditionally re-exported. + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports( + source, + [placement], + {"foo": entity}, + {}, + reexport_mode="application", + is_test_file=True, + ) + assert result == source + + +def test_add_re_exports_mode_application_test_file_in_external_loads_reexported(): + # "application" mode + test file: public name IS re-exported when in external_loads. + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports( + source, + [placement], + {"foo": entity}, + {}, + external_loads={"foo"}, + reexport_mode="application", + is_test_file=True, + ) + assert "from .utils import foo" in result + + +def test_add_re_exports_mode_application_test_file_public_in_still_loaded_reexported(): + # "application" mode + test file: public name IS re-exported when still referenced. + source = "import os\n\nfoo()\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports( + source, + [placement], + {"foo": entity}, + {}, + reexport_mode="application", + is_test_file=True, + ) + assert "from .utils import foo" in result + + +def test_add_re_exports_mode_imported_public_not_in_external_loads_not_reexported(): + # "imported" mode: public name is NOT re-exported if absent from external_loads. + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports( + source, [placement], {"foo": entity}, {}, reexport_mode="imported" + ) + assert result == source + + +def test_add_re_exports_mode_imported_public_in_external_loads_reexported(): + # "imported" mode: public name IS re-exported when in external_loads. + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports( + source, + [placement], + {"foo": entity}, + {}, + external_loads={"foo"}, + reexport_mode="imported", + ) + assert "from .utils import foo" in result + + +def test_add_re_exports_mode_imported_public_in_still_loaded_reexported(): + # "imported" mode: public name IS re-exported when still referenced in source. + source = "import os\n\nfoo()\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports( + source, [placement], {"foo": entity}, {}, reexport_mode="imported" + ) + assert "from .utils import foo" in result + + +def test_add_re_exports_mode_imported_private_in_external_loads_reexported(): + # "imported" mode: private names still follow the same rule (external_loads). + source = "import os\n" + entity = _make_entity("_helper", 1, 2) + placement = GroupPlacement(group=["_helper"], target_file="utils.py") + result = _add_re_exports( + source, + [placement], + {"_helper": entity}, + {}, + external_loads={"_helper"}, + reexport_mode="imported", + ) + assert "from .utils import _helper" in result + + +def test_generate_private_entity_reexported_when_external_caller(tmp_path): + # Private entity is re-exported when an external file imports it. + (tmp_path / "pyproject.toml").write_text("") + pkg = tmp_path / "mypkg" + pkg.mkdir() + mod = pkg / "big.py" + mod.write_text("def _helper():\n pass\n") + caller = tmp_path / "tests" / "test_big.py" + caller.parent.mkdir() + caller.write_text("from mypkg.big import _helper\n") + + source = "def _helper():\n pass\n" + entity = _make_entity("_helper", 1, 2) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["_helper"], target_file="private.py")]) + + result = generate_file_splits(c, plan, source, str(mod)) + + assert "from .private import _helper" in result.original_source + + +def test_generate_file_splits_reexport_imported_public_not_reexported_without_caller( + tmp_path, +): + # "imported" mode: public entity not imported elsewhere → no re-export stub. + (tmp_path / "pyproject.toml").write_text("") + pkg = tmp_path / "mypkg" + pkg.mkdir() + mod = pkg / "big.py" + mod.write_text("def foo():\n pass\n") + # No external callers import foo. + + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["foo"], target_file="helpers.py")]) + + result = generate_file_splits(c, plan, source, str(mod), reexport_mode="imported") + + assert "from .helpers import foo" not in result.original_source + + +def test_generate_file_splits_reexport_mode_imported_public_reexported_with_caller( + tmp_path, +): + # "imported" mode: public entity imported elsewhere → re-export stub is added. + (tmp_path / "pyproject.toml").write_text("") + pkg = tmp_path / "mypkg" + pkg.mkdir() + mod = pkg / "big.py" + mod.write_text("def foo():\n pass\n") + caller = tmp_path / "other.py" + caller.write_text("from mypkg.big import foo\n") + + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["foo"], target_file="helpers.py")]) + + result = generate_file_splits(c, plan, source, str(mod), reexport_mode="imported") + + assert "from .helpers import foo" in result.original_source + + +def test_generate_file_splits_reexport_mode_always_public_reexported_without_caller( + tmp_path, +): + # "always" mode: public entity re-exported even when no external callers exist. + (tmp_path / "pyproject.toml").write_text("") + pkg = tmp_path / "mypkg" + pkg.mkdir() + mod = pkg / "big.py" + mod.write_text("def foo():\n pass\n") + + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["foo"], target_file="helpers.py")]) + + result = generate_file_splits(c, plan, source, str(mod), reexport_mode="always") + + assert "from .helpers import foo" in result.original_source + + +def test_add_re_exports_private_external_only_gets_noqa(): + # Private name in external_loads but NOT in remaining source → fmt: skip # noqa comment. + source = "import os\n" + entity = _make_entity("_helper", 1, 2) + placement = GroupPlacement(group=["_helper"], target_file="utils.py") + result = _add_re_exports( + source, [placement], {"_helper": entity}, {}, external_loads={"_helper"} + ) + assert "from .utils import _helper # fmt: skip # noqa: F401, E501" in result + + +def test_add_re_exports_private_in_still_loaded_no_noqa(): + # Private name referenced in remaining source but NOT in external_loads + # → re-export without noqa (it is actively used; no future-pruning risk). + source = "import os\n\n_helper()\n" + entity = _make_entity("_helper", 3, 3) + placement = GroupPlacement(group=["_helper"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"_helper": entity}, {}) + assert "from .utils import _helper\n" in result + assert "# noqa" not in result + + +def test_add_re_exports_private_in_still_loaded_and_external_loads_gets_noqa(): + # Private name referenced in remaining source AND in external_loads → noqa + # marker is added even though it is currently "used", because the non-migrated + # entity that uses it may itself be migrated in a later recursive split, at + # which point _prune_unused_imports would silently drop an un-annotated stub. + source = "import os\n\n_helper()\n" + entity = _make_entity("_helper", 3, 3) + placement = GroupPlacement(group=["_helper"], target_file="utils.py") + result = _add_re_exports( + source, [placement], {"_helper": entity}, {}, external_loads={"_helper"} + ) + assert "from .utils import _helper # fmt: skip # noqa: F401, E501" in result + + +def test_add_re_exports_public_not_in_still_loaded_gets_noqa(): + # Public name migrated but not referenced in remaining source → fmt: skip # noqa. + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"foo": entity}, {}) + assert "from .utils import foo # fmt: skip # noqa: F401, E501" in result + + +def test_add_re_exports_public_in_still_loaded_no_noqa(): + # Public name still referenced in remaining source → re-export without noqa. + source = "import os\n\nfoo()\n" + entity = _make_entity("foo", 3, 3) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"foo": entity}, {}) + assert "from .utils import foo\n" in result + assert "# noqa" not in result + + +def test_add_re_exports_multiple_noqa_each_on_own_line(): + # Two names both need noqa → one import line each so Black can't break the comment. + source = "import os\n" + entity = _make_entity("_block", 3, 4, ["_a", "_b"]) + placement = GroupPlacement(group=["_block"], target_file="utils.py") + result = _add_re_exports( + source, + [placement], + {"_block": entity}, + {}, + external_loads={"_a", "_b"}, + ) + lines = result.splitlines() + noqa_lines = [line for line in lines if "# fmt: skip # noqa: F401, E501" in line] + assert len(noqa_lines) == 2 + names = {line.split("import")[1].split("#")[0].strip() for line in noqa_lines} + assert names == {"_a", "_b"} + + +def test_add_re_exports_mixed_splits_into_two_lines(): + # One entity defines two names: one in still_loaded, one purely re-exported. + # Both are in external_loads, so both get # noqa: F401 to protect them from + # being pruned if the non-migrated entity that currently uses _used is itself + # migrated in a later recursive split. + source = "import os\n\n_used()\n" + entity = _make_entity("_block", 3, 4, ["_used", "_reexport"]) + placement = GroupPlacement(group=["_block"], target_file="utils.py") + result = _add_re_exports( + source, + [placement], + {"_block": entity}, + {}, + external_loads={"_used", "_reexport"}, + ) + lines = result.splitlines() + noqa_lines = [line for line in lines if "# fmt: skip # noqa: F401, E501" in line] + assert len(noqa_lines) == 2 + names = {line.split("import")[1].split("#")[0].strip() for line in noqa_lines} + assert names == {"_used", "_reexport"} + + +def test_add_re_exports_mixed_only_still_loaded_in_external_loads_gets_noqa(): + # When only the used name is in external_loads (not the purely re-exported one), + # verify external_loads membership drives noqa independently of still_loaded. + source = "import os\n\n_used()\n" + entity = _make_entity("_block", 3, 4, ["_used", "_reexport"]) + placement = GroupPlacement(group=["_block"], target_file="utils.py") + result = _add_re_exports( + source, + [placement], + {"_block": entity}, + {}, + external_loads={"_used"}, # only _used is externally imported + ) + lines = result.splitlines() + noqa_lines = [line for line in lines if "# fmt: skip # noqa: F401, E501" in line] + # _used is in still_loaded AND external_loads → gets noqa + assert len(noqa_lines) == 1 + assert "_used" in noqa_lines[0] + # _reexport is not in still_loaded and not in external_loads → not re-exported + assert "_reexport" not in result + + +def test_add_re_exports_is_test_file_adds_comment_before_first_noqa(): + # is_test_file=True → single explanatory comment inserted before the first + # F401 import; non-test files and test files with no noqa imports get no comment. + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports( + source, [placement], {"foo": entity}, {}, is_test_file=True + ) + lines = result.splitlines() + comment_idx = next( + ( + i + for i, l in enumerate(lines) + if "Re-exported for backwards compatibility" in l + ), + None, + ) + noqa_idx = next( + (i for i, l in enumerate(lines) if "# noqa: F401" in l), + None, + ) + assert comment_idx is not None + assert noqa_idx is not None + assert comment_idx == noqa_idx - 1 + + +def test_add_re_exports_is_test_file_false_no_comment(): + # is_test_file=False (default) → no explanatory comment added. + source = "import os\n" + entity = _make_entity("foo", 1, 2) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports(source, [placement], {"foo": entity}, {}) + assert "Re-exported for backwards compatibility" not in result + + +def test_add_re_exports_is_test_file_no_noqa_imports_no_comment(): + # is_test_file=True but all re-exports are already referenced in source + # (no noqa imports) → comment is not added. + source = "import os\n\nfoo()\n" + entity = _make_entity("foo", 3, 3) + placement = GroupPlacement(group=["foo"], target_file="utils.py") + result = _add_re_exports( + source, [placement], {"foo": entity}, {}, is_test_file=True + ) + assert "Re-exported for backwards compatibility" not in result + + +def test_add_re_exports_is_test_file_comment_added_once_for_multiple_noqa(): + # Multiple noqa imports → comment appears exactly once, before the first one. + source = "import os\n" + entity = _make_entity("_block", 1, 2, ["foo", "bar"]) + placement = GroupPlacement(group=["_block"], target_file="utils.py") + result = _add_re_exports( + source, [placement], {"_block": entity}, {}, is_test_file=True + ) + comment_count = result.count("Re-exported for backwards compatibility") + assert comment_count == 1 + + +def test_add_re_exports_is_test_file_comment_before_noqa_when_mixed(): + # is_test_file=True with a mix of used (no noqa) and pure re-export (noqa) + # imports: the comment must appear before the noqa line, not before the used line. + source = "import os\n\n_used()\n" + entity = _make_entity("_block", 3, 4, ["_used", "pub"]) + placement = GroupPlacement(group=["_block"], target_file="utils.py") + result = _add_re_exports( + source, [placement], {"_block": entity}, {}, is_test_file=True + ) + lines = result.splitlines() + comment_idx = next( + i for i, l in enumerate(lines) if "Re-exported for backwards" in l + ) + noqa_idx = next(i for i, l in enumerate(lines) if "# noqa: F401" in l) + used_idx = next(i for i, l in enumerate(lines) if "import _used" in l) + assert used_idx < comment_idx + assert comment_idx == noqa_idx - 1 + + +def test_add_re_exports_relative_from_uses_relative_prefix(): + # When relative_from is set, imports are computed via _relative_import_prefix + # rather than _target_module_name, so "service/__init__.py" → ".utils" + # (not ".service.utils"). + source = "# stayed\n" + entity = _make_entity("Foo", 1, 1) + placements = [GroupPlacement(group=["Foo"], target_file="service/utils.py")] + entity_map = {"Foo": entity} + entity_source_map = {"Foo": "class Foo: pass"} + + result = _add_re_exports( + source, + placements, + entity_map, + entity_source_map, + relative_from="service/__init__.py", + ) + + assert "from .utils import Foo" in result + # Must NOT use the fully-qualified form that would be wrong from __init__.py. + assert "from .service.utils" not in result diff --git a/tests/code_gen/test_external_imported_names.py b/tests/code_gen/test_external_imported_names.py new file mode 100644 index 0000000..9b65948 --- /dev/null +++ b/tests/code_gen/test_external_imported_names.py @@ -0,0 +1,156 @@ +from __future__ import annotations +from crispen.file_limiter.code_gen import _collect_external_imported_names + + +def test_collect_external_imported_names_relative_path(): + # Non-absolute path → empty set (no scan). + assert _collect_external_imported_names("relative/path.py") == set() + + +def test_collect_external_imported_names_nonexistent_file(tmp_path): + # Absolute but non-existent → empty set. + assert _collect_external_imported_names(str(tmp_path / "ghost.py")) == set() + + +def test_collect_external_imported_names_no_project_root(tmp_path): + # File exists but no pyproject.toml/.git above it → empty set. + f = tmp_path / "module.py" + f.write_text("x = 1\n") + # tmp_path is under /tmp which typically has no project markers. + result = _collect_external_imported_names(str(f)) + # May or may not find a root depending on environment; we just verify no crash. + assert isinstance(result, set) + + +def test_collect_external_imported_names_absolute_import(tmp_path): + (tmp_path / "pyproject.toml").write_text("") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + mod = pkg / "utils.py" + mod.write_text("def _helper():\n pass\n") + caller = tmp_path / "tests" / "test_utils.py" + caller.parent.mkdir() + caller.write_text("from mypkg.utils import _helper\n") + result = _collect_external_imported_names(str(mod)) + assert "_helper" in result + + +def test_collect_external_imported_names_relative_import(tmp_path): + (tmp_path / "pyproject.toml").write_text("") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + mod = pkg / "utils.py" + mod.write_text("def _helper():\n pass\n") + sibling = pkg / "other.py" + sibling.write_text("from .utils import _helper\n") + result = _collect_external_imported_names(str(mod)) + assert "_helper" in result + + +def test_collect_external_imported_names_self_excluded(tmp_path): + # The file being scanned is excluded from the search. + (tmp_path / "pyproject.toml").write_text("") + mod = tmp_path / "module.py" + mod.write_text("from module import _x\n") # self-referential (ignored) + result = _collect_external_imported_names(str(mod)) + assert "_x" not in result + + +def test_collect_external_imported_names_syntax_error_skipped(tmp_path): + (tmp_path / "pyproject.toml").write_text("") + mod = tmp_path / "module.py" + mod.write_text("def _helper(): pass\n") + bad = tmp_path / "bad.py" + bad.write_text("def (invalid\n") + good = tmp_path / "good.py" + good.write_text("from module import _helper\n") + result = _collect_external_imported_names(str(mod)) + assert "_helper" in result + + +def test_collect_external_imported_names_non_matching_import_ignored(tmp_path): + (tmp_path / "pyproject.toml").write_text("") + mod = tmp_path / "module.py" + mod.write_text("def _helper(): pass\n") + other = tmp_path / "other.py" + other.write_text("from different_module import _helper\n") + result = _collect_external_imported_names(str(mod)) + assert "_helper" not in result + + +def test_collect_external_imported_names_non_importfrom_nodes_skipped(tmp_path): + # Caller file contains a plain `import` statement (not ImportFrom) mixed + # with a matching `from … import`. The plain import must be skipped without + # crashing, and the matching ImportFrom still contributes to the result. + (tmp_path / "pyproject.toml").write_text("") + mod = tmp_path / "module.py" + mod.write_text("def _helper(): pass\n") + caller = tmp_path / "caller.py" + caller.write_text("import os\nfrom module import _helper\n") + result = _collect_external_imported_names(str(mod)) + assert "_helper" in result + + +def test_collect_external_imported_names_deep_relative_import(tmp_path): + # Two-level relative import: `from ..utils import _helper` + (tmp_path / "pyproject.toml").write_text("") + mod = tmp_path / "utils.py" + mod.write_text("def _helper(): pass\n") + sub = tmp_path / "pkg" / "sub" / "caller.py" + sub.parent.mkdir(parents=True) + sub.write_text("from ...utils import _helper\n") + result = _collect_external_imported_names(str(mod)) + assert "_helper" in result + + +def test_collect_external_imported_names_init_py_at_root(tmp_path): + # A bare __init__.py at the project root has no package prefix, so no + # external caller can import from it by package path — returns empty set. + (tmp_path / "pyproject.toml").write_text("") + init_py = tmp_path / "__init__.py" + init_py.write_text("class Foo: pass\n") + result = _collect_external_imported_names(str(init_py)) + assert result == set() + + +def test_collect_external_imported_names_init_py(tmp_path): + # When original_path is an __init__.py, callers import from the package + # name (e.g. "mypkg.sub"), not "mypkg.sub.__init__". + (tmp_path / "pyproject.toml").write_text("") + pkg = tmp_path / "mypkg" / "sub" + pkg.mkdir(parents=True) + init_py = pkg / "__init__.py" + init_py.write_text("class Foo: pass\n") + caller = tmp_path / "caller.py" + caller.write_text("from mypkg.sub import Foo\n") + result = _collect_external_imported_names(str(init_py)) + assert "Foo" in result + + +def test_collect_external_imported_names_init_py_relative_caller(tmp_path): + # Relative import from sibling module targeting a package __init__.py. + (tmp_path / "pyproject.toml").write_text("") + pkg = tmp_path / "mypkg" + pkg.mkdir() + sub = pkg / "sub" + sub.mkdir() + (sub / "__init__.py").write_text("def _helper(): pass\n") + sibling = pkg / "other.py" + sibling.write_text("from .sub import _helper\n") + result = _collect_external_imported_names(str(sub / "__init__.py")) + assert "_helper" in result + + +def test_collect_external_imported_names_relative_level_too_deep(tmp_path): + # Relative import that goes above the project root → skipped without crash. + (tmp_path / "pyproject.toml").write_text("") + mod = tmp_path / "utils.py" + mod.write_text("def _helper(): pass\n") + # A file at the top level trying to go up 5 packages (impossible). + top = tmp_path / "top.py" + top.write_text("from .....utils import _helper\n") + result = _collect_external_imported_names(str(mod)) + # The over-deep import is silently skipped; no crash. + assert isinstance(result, set) diff --git a/tests/code_gen/test_find_cross_file_imports.py b/tests/code_gen/test_find_cross_file_imports.py new file mode 100644 index 0000000..77ba915 --- /dev/null +++ b/tests/code_gen/test_find_cross_file_imports.py @@ -0,0 +1,386 @@ +from __future__ import annotations +from crispen.file_limiter.code_gen import ( + _abs_package_for_dir, + _find_cross_file_imports, + _find_cross_file_type_checking_imports, + _module_import_stmt, + _relative_import_prefix, +) + + +def test_find_cross_file_imports_basic(): + # fn_a references _MODEL which is defined in block_1.py + entity_source_map = {"fn_a": "def fn_a():\n return _MODEL\n"} + name_to_target_file = {"_MODEL": "block_1.py"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["fn_a"], entity_source_map, name_to_target_file, "llm_extract.py" + ) + assert from_imports == ["from .block_1 import _MODEL"] + assert module_imports == [] + assert rewrites == {} + + +def test_find_cross_file_imports_same_file_excluded(): + # _MODEL goes to the same file as fn_a → no cross-file import needed + entity_source_map = {"fn_a": "def fn_a():\n return _MODEL\n"} + name_to_target_file = {"_MODEL": "llm_extract.py"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["fn_a"], entity_source_map, name_to_target_file, "llm_extract.py" + ) + assert from_imports == [] + assert module_imports == [] + assert rewrites == {} + + +def test_find_cross_file_imports_no_match(): + # Referenced name not in name_to_target_file → no cross-file import + entity_source_map = {"fn_a": "def fn_a():\n return os.getcwd()\n"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["fn_a"], entity_source_map, {}, "utils.py" + ) + assert from_imports == [] + assert module_imports == [] + assert rewrites == {} + + +def test_find_cross_file_imports_entity_not_in_map(): + # Entity name not in entity_source_map → treated as empty source, no imports + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["ghost"], {}, {"x": "other.py"}, "utils.py" + ) + assert from_imports == [] + assert module_imports == [] + assert rewrites == {} + + +def test_relative_import_prefix_same_directory(): + # Both files at the root level → single dot. + assert _relative_import_prefix("a.py", "b.py") == ".b" + + +def test_relative_import_prefix_sibling_subdir(): + # from_file is in sub/, to_file is in helpers/ → go up one, then down. + assert _relative_import_prefix("sub/a.py", "helpers/b.py") == "..helpers.b" + + +def test_relative_import_prefix_same_subdir(): + # Both in the same subdirectory → single dot. + assert _relative_import_prefix("sub/a.py", "sub/b.py") == ".b" + + +def test_relative_import_prefix_to_nested(): + # to_file is in a subdirectory of root while from_file is at root. + assert _relative_import_prefix("a.py", "helpers/b.py") == ".helpers.b" + + +def test_relative_import_prefix_to_init_same_dir(): + # to_file is __init__.py in the same directory → "." (the package itself). + assert _relative_import_prefix("a.py", "__init__.py") == "." + + +def test_relative_import_prefix_to_init_same_subdir(): + # Both in sub/, to_file is sub/__init__.py → "." (the package itself). + assert _relative_import_prefix("sub/a.py", "sub/__init__.py") == "." + + +def test_find_cross_file_imports_cross_directory(): + # fn_a is in tests/test.py; helper is in helpers/entities.py. + # Cross-directory import needs ".." to go up from tests/ to root. + entity_source_map = {"fn_a": "def fn_a():\n return _helper()\n"} + name_to_target_file = {"_helper": "helpers/entities.py"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["fn_a"], entity_source_map, name_to_target_file, "tests/test.py" + ) + assert from_imports == ["from ..helpers.entities import _helper"] + assert module_imports == [] + assert rewrites == {} + + +def test_find_cross_file_imports_top_level_var_uses_module_import(): + # SAFE_MODE is a TOP_LEVEL variable in conversion.py; runtime.py references it. + # Should produce a module-level import (from . import conversion) in + # module_imports, not a direct name import, so that later mutations to the + # variable propagate correctly. + entity_source_map = { + "create_lua_runtime": ( + "def create_lua_runtime(safe_mode=None):\n" + " if safe_mode is None:\n" + " safe_mode = SAFE_MODE\n" + ) + } + name_to_target_file = {"SAFE_MODE": "conversion.py"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["create_lua_runtime"], + entity_source_map, + name_to_target_file, + "runtime.py", + top_level_var_names={"SAFE_MODE"}, + ) + assert from_imports == [] + assert module_imports == ["from . import conversion"] + assert rewrites == {"SAFE_MODE": "conversion.SAFE_MODE"} + + +def test_find_cross_file_imports_top_level_var_abs_pkg(): + # Same as above but with abs_pkg set (test-file context). + # Uses "import pkg.module as local" syntax to avoid test-name misclassification. + entity_source_map = {"fn_a": "def fn_a():\n return SAFE_MODE\n"} + name_to_target_file = {"SAFE_MODE": "conversion.py"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["fn_a"], + entity_source_map, + name_to_target_file, + "test_fn.py", + abs_pkg="mylib", + top_level_var_names={"SAFE_MODE"}, + ) + assert from_imports == [] + assert module_imports == ["import mylib.conversion as conversion"] + assert rewrites == {"SAFE_MODE": "conversion.SAFE_MODE"} + + +def test_find_cross_file_imports_top_level_var_abs_pkg_empty(): + # abs_pkg="" (root-level test) — no package prefix, plain "import conversion". + entity_source_map = {"fn_a": "def fn_a():\n return SAFE_MODE\n"} + name_to_target_file = {"SAFE_MODE": "conversion.py"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["fn_a"], + entity_source_map, + name_to_target_file, + "test_fn.py", + abs_pkg="", + top_level_var_names={"SAFE_MODE"}, + ) + assert from_imports == [] + assert module_imports == ["import conversion"] + assert rewrites == {"SAFE_MODE": "conversion.SAFE_MODE"} + + +def test_find_cross_file_imports_top_level_var_cross_directory(): + # TOP_LEVEL var in sub/constants.py, referenced from runtime.py at root. + entity_source_map = {"fn_a": "def fn_a():\n return TIMEOUT\n"} + name_to_target_file = {"TIMEOUT": "sub/constants.py"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["fn_a"], + entity_source_map, + name_to_target_file, + "runtime.py", + top_level_var_names={"TIMEOUT"}, + ) + assert from_imports == [] + assert module_imports == ["from .sub import constants"] + assert rewrites == {"TIMEOUT": "constants.TIMEOUT"} + + +def test_find_cross_file_imports_mixed_top_level_and_function(): + # SAFE_MODE is a TOP_LEVEL var; _helper is a function — mixed case. + entity_source_map = { + "fn_a": ("def fn_a():\n" " if SAFE_MODE:\n" " return _helper()\n") + } + name_to_target_file = {"SAFE_MODE": "conversion.py", "_helper": "helpers.py"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["fn_a"], + entity_source_map, + name_to_target_file, + "runtime.py", + top_level_var_names={"SAFE_MODE"}, + ) + assert from_imports == ["from .helpers import _helper"] + assert module_imports == ["from . import conversion"] + assert rewrites == {"SAFE_MODE": "conversion.SAFE_MODE"} + + +def test_module_import_stmt_sibling_relative(): + stmt, local = _module_import_stmt("runtime.py", "conversion.py", abs_pkg=None) + assert stmt == "from . import conversion" + assert local == "conversion" + + +def test_module_import_stmt_cross_directory_relative(): + stmt, local = _module_import_stmt("runtime.py", "sub/constants.py", abs_pkg=None) + assert stmt == "from .sub import constants" + assert local == "constants" + + +def test_module_import_stmt_parent_directory_relative(): + # svc/test_fns.py importing from test_svc.py (parent dir) + stmt, local = _module_import_stmt("svc/test_fns.py", "test_svc.py", abs_pkg=None) + assert stmt == "from .. import test_svc" + assert local == "test_svc" + + +def test_module_import_stmt_abs_pkg_with_prefix(): + # Uses "import pkg.module as local" to avoid test-name collision. + stmt, local = _module_import_stmt("test_fn.py", "conversion.py", abs_pkg="mylib") + assert stmt == "import mylib.conversion as conversion" + assert local == "conversion" + + +def test_module_import_stmt_abs_pkg_empty(): + # No package prefix → plain "import conversion". + stmt, local = _module_import_stmt("test_fn.py", "conversion.py", abs_pkg="") + assert stmt == "import conversion" + assert local == "conversion" + + +def test_module_import_stmt_abs_pkg_nested_module(): + # source_file has a nested path within the package + stmt, local = _module_import_stmt("test_fn.py", "sub/constants.py", abs_pkg="mylib") + assert stmt == "import mylib.sub.constants as constants" + assert local == "constants" + + +def test_find_cross_file_imports_abs_pkg_package_prefix(): + # abs_pkg="tests" → "from tests.block_1 import _MODEL" + entity_source_map = {"fn_a": "def fn_a():\n return _MODEL\n"} + name_to_target_file = {"_MODEL": "block_1.py"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["fn_a"], entity_source_map, name_to_target_file, "test_fn.py", abs_pkg="tests" + ) + assert from_imports == ["from tests.block_1 import _MODEL"] + assert module_imports == [] + assert rewrites == {} + + +def test_find_cross_file_imports_abs_pkg_root_level(): + # abs_pkg="" → "from block_1 import _MODEL" (no package prefix) + entity_source_map = {"fn_a": "def fn_a():\n return _MODEL\n"} + name_to_target_file = {"_MODEL": "block_1.py"} + from_imports, module_imports, rewrites = _find_cross_file_imports( + ["fn_a"], entity_source_map, name_to_target_file, "test_fn.py", abs_pkg="" + ) + assert from_imports == ["from block_1 import _MODEL"] + assert module_imports == [] + assert rewrites == {} + + +def test_find_cross_file_type_checking_imports_basic(): + # _LLMAccumulator appears only in a quoted annotation in fn_a. + # It lives in block_1.py — a TYPE_CHECKING import should be generated. + entity_source_map = { + "fn_a": 'def fn_a(x: Optional["_LLMAccumulator"]) -> None:\n pass\n' + } + name_to_target_file = {"_LLMAccumulator": "block_1.py"} + result = _find_cross_file_type_checking_imports( + ["fn_a"], entity_source_map, name_to_target_file, "placements.py" + ) + assert result == ["from .block_1 import _LLMAccumulator"] + + +def test_find_cross_file_type_checking_imports_same_file_excluded(): + # _LLMAccumulator goes to the same target file — no import needed. + entity_source_map = { + "fn_a": 'def fn_a(x: Optional["_LLMAccumulator"]) -> None:\n pass\n' + } + name_to_target_file = {"_LLMAccumulator": "placements.py"} + result = _find_cross_file_type_checking_imports( + ["fn_a"], entity_source_map, name_to_target_file, "placements.py" + ) + assert result == [] + + +def test_find_cross_file_type_checking_imports_runtime_excluded(): + # _LLMAccumulator is used at runtime (not just annotation) — excluded. + entity_source_map = {"fn_a": "def fn_a():\n return _LLMAccumulator()\n"} + name_to_target_file = {"_LLMAccumulator": "block_1.py"} + result = _find_cross_file_type_checking_imports( + ["fn_a"], entity_source_map, name_to_target_file, "placements.py" + ) + assert result == [] + + +def test_find_cross_file_type_checking_imports_no_annotations(): + # No quoted annotations at all → empty result. + entity_source_map = {"fn_a": "def fn_a():\n pass\n"} + name_to_target_file = {"_LLMAccumulator": "block_1.py"} + result = _find_cross_file_type_checking_imports( + ["fn_a"], entity_source_map, name_to_target_file, "placements.py" + ) + assert result == [] + + +def test_find_cross_file_type_checking_imports_not_in_map(): + # Referenced quoted name not in name_to_target_file → no import. + entity_source_map = {"fn_a": 'def fn_a(x: "UnknownType") -> None:\n pass\n'} + result = _find_cross_file_type_checking_imports( + ["fn_a"], entity_source_map, {}, "placements.py" + ) + assert result == [] + + +def test_find_cross_file_type_checking_imports_top_level_var_excluded(): + # A name in top_level_var_names is skipped (handled separately). + entity_source_map = { + "fn_a": 'def fn_a(x: Optional["SAFE_MODE"]) -> None:\n pass\n' + } + name_to_target_file = {"SAFE_MODE": "constants.py"} + result = _find_cross_file_type_checking_imports( + ["fn_a"], + entity_source_map, + name_to_target_file, + "placements.py", + top_level_var_names={"SAFE_MODE"}, + ) + assert result == [] + + +def test_find_cross_file_type_checking_imports_abs_pkg(): + # With abs_pkg set, use absolute import style. + entity_source_map = { + "fn_a": 'def fn_a(x: Optional["_LLMAccumulator"]) -> None:\n pass\n' + } + name_to_target_file = {"_LLMAccumulator": "block_1.py"} + result = _find_cross_file_type_checking_imports( + ["fn_a"], + entity_source_map, + name_to_target_file, + "test_fn.py", + abs_pkg="tests", + ) + assert result == ["from tests.block_1 import _LLMAccumulator"] + + +def test_find_cross_file_type_checking_imports_entity_not_in_map(): + # Entity not in entity_source_map → treated as empty, no imports. + result = _find_cross_file_type_checking_imports( + ["ghost"], {}, {"_X": "other.py"}, "placements.py" + ) + assert result == [] + + +def test_abs_package_for_dir_subdir(tmp_path): + (tmp_path / "pyproject.toml").touch() + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + test_file = tests_dir / "test_engine.py" + test_file.touch() + assert _abs_package_for_dir(str(test_file)) == "tests" + + +def test_abs_package_for_dir_root_level(tmp_path): + (tmp_path / "pyproject.toml").touch() + test_file = tmp_path / "test_engine.py" + test_file.touch() + assert _abs_package_for_dir(str(test_file)) == "" + + +def test_abs_package_for_dir_no_project_root(monkeypatch): + monkeypatch.setattr( + "crispen.file_limiter.code_gen.cross_file_deps._find_project_root", + lambda _p: None, + ) + assert _abs_package_for_dir("/some/random/path/test_engine.py") is None + + +def test_abs_package_for_dir_non_ancestor_root(tmp_path, monkeypatch): + # Defensive branch: project root is not an ancestor of the file's directory. + other_dir = tmp_path / "other" + other_dir.mkdir() + monkeypatch.setattr( + "crispen.file_limiter.code_gen.cross_file_deps._find_project_root", + lambda _p: other_dir, + ) + test_file = tmp_path / "tests" / "test_engine.py" + test_file.parent.mkdir() + test_file.touch() + assert _abs_package_for_dir(str(test_file)) is None diff --git a/tests/code_gen/test_find_needed_imports.py b/tests/code_gen/test_find_needed_imports.py new file mode 100644 index 0000000..9b19f03 --- /dev/null +++ b/tests/code_gen/test_find_needed_imports.py @@ -0,0 +1,235 @@ +from __future__ import annotations +from crispen.file_limiter.code_gen import ( + ImportInfo, + _find_needed_imports, + _find_type_checking_needed_imports, +) + + +def test_find_needed_imports_referenced_name(): + # Entity references "os"; import for "os" should be included. + entity_src_map = {"foo": "def foo():\n os.getcwd()\n"} + infos = [ImportInfo(names=["os"], source="import os", is_future=False)] + result = _find_needed_imports(["foo"], entity_src_map, infos, {"foo"}) + assert "import os" in result + + +def test_find_needed_imports_unreferenced_name(): + # Entity doesn't reference "sys"; import should be excluded. + entity_src_map = {"foo": "def foo():\n pass\n"} + infos = [ImportInfo(names=["sys"], source="import sys", is_future=False)] + result = _find_needed_imports(["foo"], entity_src_map, infos, {"foo"}) + assert result == [] + + +def test_find_needed_imports_future_always_included(): + # __future__ import is always included regardless of entity references. + entity_src_map = {"foo": "def foo():\n pass\n"} + infos = [ + ImportInfo( + names=["annotations"], + source="from __future__ import annotations", + is_future=True, + ) + ] + result = _find_needed_imports(["foo"], entity_src_map, infos, {"foo"}) + assert "from __future__ import annotations" in result + + +def test_find_needed_imports_deduplicates(): + # Two ImportInfo entries with the same source string → only one included. + entity_src_map = {"foo": "def foo():\n os.getcwd()\n"} + infos = [ + ImportInfo(names=["os"], source="import os", is_future=False), + ImportInfo(names=["os"], source="import os", is_future=False), # duplicate + ] + result = _find_needed_imports(["foo"], entity_src_map, infos, {"foo"}) + assert result.count("import os") == 1 + + +def test_find_needed_imports_entity_not_in_map(): + # Entity name not in entity_source_map → treated as empty source. + infos = [ImportInfo(names=["os"], source="import os", is_future=False)] + result = _find_needed_imports(["ghost"], {}, infos, set()) + assert result == [] + + +def test_find_needed_imports_skips_type_checking(): + # is_type_checking imports must not appear as regular imports. + entity_src_map = {"foo": 'def foo(x: "MyConfig") -> None:\n pass\n'} + infos = [ + ImportInfo( + names=["MyConfig"], + source="from .config import MyConfig", + is_future=False, + is_type_checking=True, + ) + ] + result = _find_needed_imports(["foo"], entity_src_map, infos, {"foo"}) + assert result == [] + + +def test_find_type_checking_needed_imports_quoted_only(): + # "MyType" appears only in a quoted annotation, not a runtime load. + entity_src_map = {"foo": 'def foo(x: Optional["MyType"]) -> None:\n pass\n'} + infos = [ + ImportInfo( + names=["MyType"], source="from models import MyType", is_future=False + ) + ] + result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) + assert "from models import MyType" in result + + +def test_find_type_checking_needed_imports_runtime_excluded(): + # When the name is used at runtime (not just in a quoted annotation), + # it should NOT appear in the TYPE_CHECKING-only list. + # annotation_only = quoted - runtime excludes runtime names directly. + entity_src_map = {"foo": "def foo():\n return MyType()\n"} + infos = [ + ImportInfo( + names=["MyType"], source="from models import MyType", is_future=False + ) + ] + result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) + assert result == [] + + +def test_find_type_checking_needed_imports_no_annotations(): + # No quoted annotations → result is empty. + entity_src_map = {"foo": "def foo():\n pass\n"} + infos = [ + ImportInfo( + names=["MyType"], source="from models import MyType", is_future=False + ) + ] + result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) + assert result == [] + + +def test_find_type_checking_needed_imports_future_excluded(): + # __future__ imports are never returned (they're always in regular imports). + entity_src_map = {"foo": 'def foo(x: "MyType") -> None:\n pass\n'} + infos = [ + ImportInfo( + names=["annotations"], + source="from __future__ import annotations", + is_future=True, + ), + ImportInfo( + names=["MyType"], source="from models import MyType", is_future=False + ), + ] + result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) + assert "from __future__ import annotations" not in result + assert "from models import MyType" in result + + +def test_find_type_checking_needed_imports_deduplicates(): + # Two ImportInfo entries with the same source → only one returned. + entity_src_map = {"foo": 'def foo(x: "MyType") -> None:\n pass\n'} + infos = [ + ImportInfo( + names=["MyType"], source="from models import MyType", is_future=False + ), + ImportInfo( + names=["MyType"], source="from models import MyType", is_future=False + ), + ] + result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) + assert result.count("from models import MyType") == 1 + + +def test_find_type_checking_needed_imports_import_names_no_match(): + # annotation_only has "MyType" but the ImportInfo names do not include it → + # the tc_names check returns False → import is skipped. + entity_src_map = {"foo": 'def foo(x: "MyType") -> None:\n pass\n'} + infos = [ + ImportInfo( + names=["OtherType"], source="from models import OtherType", is_future=False + ) + ] + result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) + assert result == [] + + +def test_find_type_checking_needed_imports_partial_multi_name_import(): + # From a multi-name import, only the annotation-only name should appear in + # the TYPE_CHECKING block; the other name (not referenced at all) must not. + entity_src_map = { + "foo": 'def foo(x: "MyResult") -> None:\n pass\n', + } + infos = [ + ImportInfo( + names=["MyResult", "run_thing"], + source="from mymod import MyResult, run_thing", + is_future=False, + ) + ] + result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) + assert len(result) == 1 + assert "MyResult" in result[0] + assert "run_thing" not in result[0] + + +def test_find_type_checking_needed_imports_narrowed_src_dedup(): + # When two ImportInfo entries produce the same narrowed source after + # filtering, only one copy should appear in the result (line 535 branch). + entity_src_map = {"foo": 'def foo(x: "MyResult") -> None:\n pass\n'} + infos = [ + ImportInfo( + names=["MyResult", "run_thing"], + source="from mymod import MyResult, run_thing", + is_future=False, + ), + # A second entry with the same source (e.g. two entities requested it). + ImportInfo( + names=["MyResult", "run_thing"], + source="from mymod import MyResult, run_thing", + is_future=False, + ), + ] + result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) + assert result.count("from mymod import MyResult") == 1 + + +def test_find_type_checking_needed_imports_shared_import_with_runtime_peer(): + # Regression: when an import line covers both a runtime name and an + # annotation-only name, the annotation-only name must still get a + # TYPE_CHECKING import even though the import source appears in the + # regular imports (where _prune_unused_imports will later drop it). + entity_src_map = { + "foo": ( + 'def foo(_acc: Optional["_LLMAccumulator"] = None) -> None:\n' + " call_with_tool(_PLACEMENT_TOOL)\n" + ) + } + infos = [ + ImportInfo( + names=["_LLMAccumulator", "_PLACEMENT_TOOL"], + source="from .llm_schemas import _LLMAccumulator, _PLACEMENT_TOOL", + is_future=False, + ) + ] + result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) + # _LLMAccumulator is only in a quoted annotation → must be in TC block + assert any("_LLMAccumulator" in r for r in result) + # _PLACEMENT_TOOL is a runtime reference → must NOT be in TC block + assert not any("_PLACEMENT_TOOL" in r for r in result) + + +def test_find_type_checking_needed_imports_uses_is_type_checking_infos(): + # is_type_checking=True ImportInfo entries are used for TC distribution; + # the function should return them for entities that use the name in a + # quoted annotation. + entity_src_map = {"foo": 'def foo(config: "MyConfig") -> None:\n pass\n'} + infos = [ + ImportInfo( + names=["MyConfig"], + source="from .config import MyConfig", + is_future=False, + is_type_checking=True, + ) + ] + result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) + assert "from .config import MyConfig" in result diff --git a/tests/code_gen/test_generate_file_splits.py b/tests/code_gen/test_generate_file_splits.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/tests/code_gen/test_generate_file_splits.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/code_gen/test_helpers.py b/tests/code_gen/test_helpers.py new file mode 100644 index 0000000..c00f1b8 --- /dev/null +++ b/tests/code_gen/test_helpers.py @@ -0,0 +1,1558 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.classifier import ClassifiedEntities +from crispen.import_sort import _sort_imports_pep8 +from crispen.file_limiter.code_gen import ( + _collect_name_loads, + _collect_name_stores, + _collect_quoted_annotation_names, + _extract_import_info, + _extract_module_docstring, + _extract_shared_helpers, + _find_project_root, + _import_derived_names, + _import_line_numbers, + _inject_inline_imports, + _inject_module_level_imports, + _inject_type_checking_imports, + _is_test_name, + _merge_from_imports, + _module_path_from_file, + _narrow_import_source, + _remove_entity_lines, + _rewrite_module_level_stores, + _rewrite_module_var_names, + _source_is_only_docstring, + _strip_module_docstring, + _strip_top_level_import_lines, + _target_module_name, + _test_names_in_decorators, + _topo_depth, + generate_file_splits, +) +from crispen.file_limiter.entity_parser import Entity, EntityKind +from .helpers import _classified, _make_classified, _make_entity, _plan + + +def test_collect_name_loads_basic(): + source = "x = foo + bar" + names = _collect_name_loads(source) + assert "foo" in names + assert "bar" in names + + +def test_collect_name_loads_store_not_included(): + source = "x = 1" + names = _collect_name_loads(source) + # x is a Store, not a Load + assert "x" not in names + + +def test_collect_name_loads_syntax_error(): + assert _collect_name_loads("def (invalid") == set() + + +def test_collect_name_loads_excludes_function_params(): + # 'client' is a parameter of test_foo — excluded from loads inside the body. + source = "def test_foo(client):\n client.call()\n" + names = _collect_name_loads(source) + assert "client" not in names + + +def test_collect_name_loads_includes_non_param_name(): + # 'helper' is not a parameter of test_foo — still counted as a load. + source = "def test_foo(client):\n helper(client)\n" + names = _collect_name_loads(source) + assert "helper" in names + assert "client" not in names + + +def test_collect_name_loads_excludes_nested_function_params(): + # Inner function params are excluded only within that function's own body. + source = textwrap.dedent( + """\ + def outer(x): + def inner(y): + return y + x + return inner + """ + ) + names = _collect_name_loads(source) + assert "y" not in names # param of inner — excluded inside inner body + assert "x" not in names # param of outer — excluded inside outer body + + +def test_collect_name_loads_includes_annotation_names(): + # Type annotations are in the outer scope — their names are included. + source = "def f(x: MyType) -> ReturnType:\n pass\n" + names = _collect_name_loads(source) + assert "MyType" in names + assert "ReturnType" in names + assert "x" not in names # param name itself, not counted + + +def test_collect_name_loads_includes_decorator_names(): + # Decorator expressions are in the outer scope. + source = "@pytest.fixture\ndef client():\n pass\n" + names = _collect_name_loads(source) + assert "pytest" in names + + +def test_collect_name_loads_kw_defaults_none_skipped(): + # kw_defaults may contain None for keyword-only args without defaults. + # None entries must not cause a crash and are simply skipped. + source = "def f(*, a, b=DEFAULT):\n pass\n" + names = _collect_name_loads(source) + assert "DEFAULT" in names + assert "a" not in names + assert "b" not in names + + +def test_collect_name_loads_annotated_vararg_kwarg(): + # *args: T and **kwargs: T annotations are in the outer scope. + source = "def f(*args: VarType, **kwargs: KwType):\n pass\n" + names = _collect_name_loads(source) + assert "VarType" in names + assert "KwType" in names + + +def test_collect_name_loads_excludes_local_variable_assignments(): + # A name assigned in the function body is a local variable — not an import. + # Loads of that name (e.g. attribute access) must not generate cross-file imports. + source = textwrap.dedent( + """\ + def test_foo(tmp_path): + helpers = tmp_path / "helpers.py" + helpers.write_text("X = 1", encoding="utf-8") + assert str(helpers.resolve()) == "x" + """ + ) + names = _collect_name_loads(source) + assert "helpers" not in names + assert "tmp_path" not in names # also a param — still excluded + + +def test_collect_name_loads_local_store_does_not_suppress_outer_loads(): + # A local assignment in an inner function must not suppress the outer scope's load. + source = textwrap.dedent( + """\ + def outer(): + use(helper) + def inner(): + helper = 1 + use(helper) + """ + ) + names = _collect_name_loads(source) + # outer() loads 'helper' (not locally defined there); inner() assigns it locally. + assert "helper" in names + + +def test_collect_quoted_annotation_names_basic(): + # "MyType" in a string annotation → detected. + source = 'def f(x: "MyType") -> None:\n pass\n' + names = _collect_quoted_annotation_names(source) + assert "MyType" in names + + +def test_collect_quoted_annotation_names_optional(): + # Optional["_LLMAccumulator"] — the inner string is parsed. + source = 'def f(x: Optional["_LLMAccumulator"]) -> None:\n pass\n' + names = _collect_quoted_annotation_names(source) + assert "_LLMAccumulator" in names + + +def test_collect_quoted_annotation_names_return(): + # Quoted return annotation. + source = 'def f() -> "ReturnType":\n pass\n' + names = _collect_quoted_annotation_names(source) + assert "ReturnType" in names + + +def test_collect_quoted_annotation_names_annassign(): + # Variable annotation: x: "MyClass" + source = 'x: "MyClass"\n' + names = _collect_quoted_annotation_names(source) + assert "MyClass" in names + + +def test_collect_quoted_annotation_names_unquoted_not_included(): + # Normal (unquoted) annotation names are NOT returned by this function. + source = "def f(x: MyType) -> None:\n pass\n" + names = _collect_quoted_annotation_names(source) + assert "MyType" not in names + + +def test_collect_quoted_annotation_names_syntax_error(): + # Unparseable source returns empty set (no crash). + assert _collect_quoted_annotation_names("def (invalid") == set() + + +def test_collect_quoted_annotation_names_inner_syntax_error(): + # A string annotation that isn't valid Python is silently ignored. + source = 'def f(x: "not valid python !!") -> None:\n pass\n' + names = _collect_quoted_annotation_names(source) + assert names == set() + + +def test_collect_quoted_annotation_names_vararg_kwarg(): + # *args and **kwargs with quoted annotations. + source = 'def f(*args: "VarType", **kwargs: "KwType") -> None:\n pass\n' + names = _collect_quoted_annotation_names(source) + assert "VarType" in names + assert "KwType" in names + + +def test_collect_quoted_annotation_names_annassign_with_value(): + # x: "MyClass" = SomeFactory() — annotation has quoted name AND there is a value. + # The _walk branch for AnnAssign with node.value must execute. + source = 'x: "MyClass" = object()\n' + names = _collect_quoted_annotation_names(source) + assert "MyClass" in names + + +def test_collect_name_stores_simple_assign(): + assert _collect_name_stores("X = 1\n") == {"X"} + + +def test_collect_name_stores_multiple_assigns(): + src = "X = 1\nY = 2\n" + assert _collect_name_stores(src) == {"X", "Y"} + + +def test_collect_name_stores_augassign(): + assert _collect_name_stores("X += 1\n") == {"X"} + + +def test_collect_name_stores_annotated_assign_with_value(): + assert _collect_name_stores("X: int = 42\n") == {"X"} + + +def test_collect_name_stores_annotated_assign_without_value(): + # Declaration only (no assignment) — not a store. + assert _collect_name_stores("X: int\n") == set() + + +def test_collect_name_stores_function_body_not_included(): + # Assignments inside function bodies are not module-level stores. + src = "def f():\n X = 1\n" + assert _collect_name_stores(src) == set() + + +def test_collect_name_stores_load_not_included(): + assert _collect_name_stores("y = X\n") == {"y"} + assert "X" not in _collect_name_stores("y = X\n") + + +def test_collect_name_stores_syntax_error(): + assert _collect_name_stores("def (broken:\n") == set() + + +def test_collect_name_stores_empty(): + assert _collect_name_stores("") == set() + + +def test_collect_name_stores_non_name_assign_target(): + # Tuple-unpacking targets are not plain Name nodes — must not crash. + src = "a, b = 1, 2\n" + result = _collect_name_stores(src) + assert "a" not in result # tuple target, not a plain Name store + assert "b" not in result + + +def test_collect_name_stores_non_name_augassign_target(): + # Attribute augmented assignment — target is Attribute, not Name. + src = "obj.x += 1\n" + result = _collect_name_stores(src) + assert result == set() + + +def test_inject_module_level_imports_docstring_only(): + # Source with only a docstring and no imports — insert after the docstring. + src = '"""Module doc."""\n\nx = 1\n' + result = _inject_module_level_imports(src, ["from . import converters"]) + assert '"""Module doc."""' in result + assert "from . import converters" in result + doc_pos = result.index('"""Module doc."""') + imp_pos = result.index("from . import converters") + assert doc_pos < imp_pos + + +def test_inject_module_level_imports_empty_list(): + src = "x = 1\n" + assert _inject_module_level_imports(src, []) == src + + +def test_inject_module_level_imports_after_imports(): + src = "import os\n\nx = 1\n" + result = _inject_module_level_imports(src, ["from . import converters"]) + assert result == "import os\nfrom . import converters\n\nx = 1\n" + + +def test_inject_module_level_imports_no_existing_imports(): + src = "x = 1\n" + result = _inject_module_level_imports(src, ["from . import converters"]) + # Prepended before non-import content + assert "from . import converters" in result + assert result.index("from . import converters") < result.index("x = 1") + + +def test_inject_module_level_imports_sorted(): + src = "import os\n\nx = 1\n" + result = _inject_module_level_imports( + src, ["from . import z_mod", "from . import a_mod"] + ) + lines = result.splitlines() + import_lines = [ln for ln in lines if "import" in ln] + assert import_lines.index("from . import a_mod") < import_lines.index( + "from . import z_mod" + ) + + +def test_inject_module_level_imports_syntax_error_prepends(): + src = "def (broken:\n" + result = _inject_module_level_imports(src, ["import os"]) + assert result.startswith("import os\n") + + +def test_inject_type_checking_imports_empty_list(): + src = "import os\n" + assert _inject_type_checking_imports(src, []) == src + + +def test_inject_type_checking_imports_syntax_error(): + src = "def (broken:\n" + assert _inject_type_checking_imports(src, ["from .config import Cfg"]) == src + + +def test_inject_type_checking_imports_all_already_present(): + # If every requested import is already in an existing TC block, no change. + src = ( + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n" + " from .config import Cfg\n" + "\n" + "x = 1\n" + ) + result = _inject_type_checking_imports(src, ["from .config import Cfg"]) + assert result == src + + +def test_inject_type_checking_imports_appends_to_existing_block(): + # New import should be appended inside the existing TYPE_CHECKING block. + src = ( + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n" + " from .config import Cfg\n" + "\n" + "x = 1\n" + ) + result = _inject_type_checking_imports(src, ["from .models import MyModel"]) + assert "from .models import MyModel" in result + tc_start = result.index("if TYPE_CHECKING:") + assert result.index("from .models import MyModel") > tc_start + assert "x = 1" in result + + +def test_inject_type_checking_imports_creates_block_with_typing_import(): + # No existing TC block and TYPE_CHECKING not imported → add both. + src = "from typing import List\n\ndef foo(x: 'Cfg') -> None:\n pass\n" + result = _inject_type_checking_imports(src, ["from .config import Cfg"]) + assert "from typing import TYPE_CHECKING" in result + assert "if TYPE_CHECKING:" in result + assert " from .config import Cfg" in result + + +def test_inject_type_checking_imports_creates_block_type_checking_already_imported(): + # TYPE_CHECKING already in typing import → don't add it again. + src = ( + "from typing import List, TYPE_CHECKING\n" + "\n" + "def foo(x: 'Cfg') -> None:\n" + " pass\n" + ) + result = _inject_type_checking_imports(src, ["from .config import Cfg"]) + assert result.count("TYPE_CHECKING") == 2 # one in import, one in if-block + assert "if TYPE_CHECKING:" in result + assert " from .config import Cfg" in result + + +def test_inject_type_checking_imports_block_after_last_import(): + # The new block should appear after the last import, before other code. + src = "import os\nimport sys\n\nx = 1\n" + result = _inject_type_checking_imports(src, ["from .config import Cfg"]) + lines = result.splitlines() + sys_line = next(i for i, l in enumerate(lines) if "import sys" in l) + if_line = next(i for i, l in enumerate(lines) if "if TYPE_CHECKING" in l) + x_line = next(i for i, l in enumerate(lines) if "x = 1" in l) + assert sys_line < if_line < x_line + + +def test_test_names_in_decorators_finds_name_in_decorator(): + src = ( + "@pytest.mark.parametrize('x', TestFixture.PARAMS)\ndef test_fn(x):\n pass\n" + ) + assert _test_names_in_decorators(src, {"TestFixture"}) == {"TestFixture"} + + +def test_test_names_in_decorators_name_only_in_body_not_found(): + src = "def test_fn():\n TestFixture.setup()\n" + assert _test_names_in_decorators(src, {"TestFixture"}) == set() + + +def test_test_names_in_decorators_syntax_error_returns_empty(): + assert _test_names_in_decorators("def (invalid", {"TestFixture"}) == set() + + +def test_test_names_in_decorators_class_decorator(): + src = "@TestFixture.mark\nclass TestSomething:\n pass\n" + assert _test_names_in_decorators(src, {"TestFixture"}) == {"TestFixture"} + + +def test_extract_import_info_syntax_error(): + assert _extract_import_info("def (invalid") == [] + + +def test_extract_import_info_plain_import(): + infos = _extract_import_info("import os\n") + assert len(infos) == 1 + assert "os" in infos[0].names + assert infos[0].is_future is False + + +def test_extract_import_info_import_with_asname(): + infos = _extract_import_info("import os as operating_system\n") + assert infos[0].names == ["operating_system"] + + +def test_extract_import_info_dotted_import(): + infos = _extract_import_info("import os.path\n") + assert infos[0].names == ["os"] + + +def test_extract_import_info_from_import(): + infos = _extract_import_info("from pathlib import Path\n") + assert "Path" in infos[0].names + assert infos[0].is_future is False + + +def test_extract_import_info_from_import_with_asname(): + infos = _extract_import_info("from pathlib import Path as P\n") + assert infos[0].names == ["P"] + + +def test_extract_import_info_future_import(): + infos = _extract_import_info("from __future__ import annotations\n") + assert infos[0].is_future is True + assert "annotations" in infos[0].names + + +def test_extract_import_info_skips_non_imports(): + infos = _extract_import_info("def foo():\n pass\n") + assert infos == [] + + +def test_extract_import_info_multiple(): + source = "import os\nfrom pathlib import Path\n" + infos = _extract_import_info(source) + assert len(infos) == 2 + + +def test_extract_import_info_multiline_parens_normalized(): + # Multi-line parenthesized from-import must be normalized to a single line + # so that _merge_from_imports can process it without producing malformed output. + source = "from pathlib import (\n Path,\n PurePath,\n)\n" + infos = _extract_import_info(source) + assert len(infos) == 1 + assert infos[0].source == "from pathlib import Path, PurePath" + assert "\n" not in infos[0].source + assert "Path" in infos[0].names + assert "PurePath" in infos[0].names + + +def test_extract_import_info_type_checking_from_import(): + # Imports inside `if TYPE_CHECKING:` are extracted with is_type_checking=True. + source = ( + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n" + " from .config import MyConfig\n" + ) + infos = _extract_import_info(source) + tc = [i for i in infos if i.is_type_checking] + assert len(tc) == 1 + assert "MyConfig" in tc[0].names + assert tc[0].source == "from .config import MyConfig" + assert tc[0].is_future is False + + +def test_extract_import_info_type_checking_plain_import(): + # Plain `import` inside `if TYPE_CHECKING:` is also captured. + source = "if TYPE_CHECKING:\n import sys\n" + infos = _extract_import_info(source) + tc = [i for i in infos if i.is_type_checking] + assert len(tc) == 1 + assert "sys" in tc[0].names + assert tc[0].is_type_checking is True + + +def test_extract_import_info_type_checking_not_is_future(): + # TYPE_CHECKING block imports must not be marked as is_future. + source = "if TYPE_CHECKING:\n from .foo import Bar\n" + infos = _extract_import_info(source) + tc = [i for i in infos if i.is_type_checking] + assert all(not i.is_future for i in tc) + + +def test_extract_import_info_type_checking_skips_non_import_children(): + # Non-import statements inside a TYPE_CHECKING block (rare but valid) + # must not cause errors and must be silently skipped. + source = "if TYPE_CHECKING:\n from .foo import Bar\n x = 1\n" + infos = _extract_import_info(source) + tc = [i for i in infos if i.is_type_checking] + assert len(tc) == 1 + assert "Bar" in tc[0].names + + +def test_narrow_import_source_syntax_error(): + # Invalid Python → original string returned unchanged. + bad = "from ??? import Foo" + assert _narrow_import_source(bad, {"Foo"}) == bad + + +def test_narrow_import_source_plain_import(): + # Non-ImportFrom statement (bare `import X`) → returned unchanged. + src = "import os" + assert _narrow_import_source(src, {"os"}) == src + + +def test_narrow_import_source_empty_keep(): + # keep_names matches nothing → alias_strs is empty → return original. + src = "from mymod import A, B" + assert _narrow_import_source(src, {"C"}) == src + + +def test_target_module_name_simple(): + assert _target_module_name("utils.py") == "utils" + + +def test_target_module_name_nested(): + assert _target_module_name("helpers/io.py") == "helpers.io" + + +def test_target_module_name_init(): + # __init__.py represents the package, not a "__init__" submodule. + assert _target_module_name("pkg/__init__.py") == "pkg" + + +def test_remove_entity_lines_removes_range(): + source = "line1\nline2\nline3\nline4\n" + entity = _make_entity("foo", 2, 3) + entity_map = {"foo": entity} + result = _remove_entity_lines(source, {"foo"}, entity_map, {}) + assert "line1" in result + assert "line2" not in result + assert "line3" not in result + assert "line4" in result + + +def test_remove_entity_lines_name_not_in_map(): + # Name not in entity_map → nothing removed. + source = "line1\nline2\n" + result = _remove_entity_lines(source, {"ghost"}, {}, {}) + assert result == source + + +def test_remove_entity_lines_top_level_preserves_import_lines(): + # When a TOP_LEVEL entity containing both imports and assignments is + # migrated, the import lines must be kept in the original file so that + # the remaining functions still have access to those names. + source = "import os\n_CONST = 1\n\ndef foo():\n return os.getcwd()\n" + entity_src = "import os\n_CONST = 1\n" + entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 2, ["os", "_CONST"]) + entity_map = {"_block_1": entity} + entity_source_map = {"_block_1": entity_src} + result = _remove_entity_lines(source, {"_block_1"}, entity_map, entity_source_map) + assert "import os" in result # import line preserved + assert "_CONST" not in result # assignment line removed + assert "def foo():" in result # function untouched + + +def test_remove_entity_lines_top_level_no_source_map_removes_all(): + # Empty entity_source_map → no imports can be identified, all lines removed. + source = "import os\n_CONST = 1\n\ndef foo():\n pass\n" + entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 2, ["os", "_CONST"]) + entity_map = {"_block_1": entity} + result = _remove_entity_lines(source, {"_block_1"}, entity_map, {}) + assert "import os" not in result + assert "_CONST" not in result + + +def test_import_derived_names_plain_import(): + src = "import os\nimport sys\n" + assert _import_derived_names(src) == {"os", "sys"} + + +def test_import_derived_names_from_import(): + src = "from typing import Dict, List\n" + assert _import_derived_names(src) == {"Dict", "List"} + + +def test_import_derived_names_aliased(): + src = "import libcst as cst\nfrom dataclasses import dataclass\n" + assert _import_derived_names(src) == {"cst", "dataclass"} + + +def test_import_derived_names_ignores_assignments(): + src = "_MODEL = 'x'\n_MIN = 3\n" + assert _import_derived_names(src) == set() + + +def test_import_derived_names_syntax_error(): + assert _import_derived_names("def (\n") == set() + + +def test_import_line_numbers_basic(): + src = "import os\n_CONST = 1\n" + entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 5, 6, []) + # Entity starts at line 5; "import os" is relative line 1 → absolute line 5. + result = _import_line_numbers(entity, src) + assert result == {5} + + +def test_import_line_numbers_no_imports(): + src = "_CONST = 1\n_OTHER = 2\n" + entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 2, []) + assert _import_line_numbers(entity, src) == set() + + +def test_import_line_numbers_syntax_error(): + entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, []) + assert _import_line_numbers(entity, "def (\n") == set() + + +def test_rewrite_module_level_stores_simple(): + src = "_CONST = int('99')\n" + result = _rewrite_module_level_stores(src, {"_CONST": "constants._CONST"}) + assert result == "constants._CONST = int('99')\n" + + +def test_rewrite_module_level_stores_augassign(): + src = "X += 1\n" + result = _rewrite_module_level_stores(src, {"X": "mod.X"}) + assert result == "mod.X += 1\n" + + +def test_rewrite_module_level_stores_annassign_with_value(): + src = "X: int = 42\n" + result = _rewrite_module_level_stores(src, {"X": "mod.X"}) + assert result == "mod.X: int = 42\n" + + +def test_rewrite_module_level_stores_annassign_without_value_skipped(): + # Declaration only — no value, so nothing to rewrite. + src = "X: int\n" + result = _rewrite_module_level_stores(src, {"X": "mod.X"}) + assert result == src + + +def test_rewrite_module_level_stores_function_body_not_rewritten(): + # Assignments inside function bodies must not be touched. + src = "def f():\n X = 1\n" + result = _rewrite_module_level_stores(src, {"X": "mod.X"}) + assert result == src + + +def test_rewrite_module_level_stores_empty_rewrites(): + src = "X = 1\n" + assert _rewrite_module_level_stores(src, {}) == src + + +def test_rewrite_module_level_stores_syntax_error(): + src = "def (broken:\n" + assert _rewrite_module_level_stores(src, {"X": "mod.X"}) == src + + +def test_rewrite_module_level_stores_name_not_in_rewrites(): + src = "Y = 1\n" + result = _rewrite_module_level_stores(src, {"X": "mod.X"}) + assert result == src + + +def test_rewrite_module_level_stores_augassign_non_name_target(): + # Attribute augmented assignment — target is Attribute, not Name; must be skipped. + src = "obj.x += 1\n" + result = _rewrite_module_level_stores(src, {"x": "mod.x"}) + assert result == src + + +def test_rewrite_module_var_names_basic(): + src = "def fn():\n if SAFE_MODE:\n pass\n" + result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) + assert "conversion.SAFE_MODE" in result + # bare SAFE_MODE no longer appears as a standalone Name + import ast + + tree = ast.parse(result) + bare = [ + n for n in ast.walk(tree) if isinstance(n, ast.Name) and n.id == "SAFE_MODE" + ] + assert bare == [] + + +def test_rewrite_module_var_names_skips_attribute_access(): + # obj.SAFE_MODE must NOT become obj.conversion.SAFE_MODE — the regex approach + # would corrupt this; the AST approach correctly skips it because 'SAFE_MODE' + # is the attr string of an Attribute node, not an ast.Name load. + src = "def fn():\n return obj.SAFE_MODE\n" + result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) + assert result == src + + +def test_rewrite_module_var_names_skips_strings(): + src = 'x = "SAFE_MODE"\n' + result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) + assert result == src + + +def test_rewrite_module_var_names_skips_comments(): + src = "# use SAFE_MODE here\nx = 1\n" + result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) + assert result == src + + +def test_rewrite_module_var_names_no_partial_name_match(): + # SAFE_MODE_EXTRA is a different identifier and must not be rewritten + src = "x = SAFE_MODE_EXTRA\ny = SAFE_MODE\n" + result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) + assert "SAFE_MODE_EXTRA" in result + assert "y = conversion.SAFE_MODE" in result + + +def test_rewrite_module_var_names_empty_rewrites(): + src = "def fn():\n return SAFE_MODE\n" + result = _rewrite_module_var_names(src, {}) + assert result == src + + +def test_rewrite_module_var_names_initial_syntax_error_returns_original(): + # Unparseable source at the start → return unchanged (first ast.parse fails) + src = "def fn(\n" + result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) + assert result == src + + +def test_rewrite_module_var_names_no_name_nodes_returns_original(): + # Source has no Name nodes for the given key → return unchanged + src = "x = 1\n" + result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) + assert result == src + + +def test_rewrite_module_var_names_verify_bare_name_survives_returns_original(): + # If a rewrite introduces a new bare Name that itself appears in rewrites, + # verification catches it and returns the original source. + # rewrites={"A": "mod.A", "mod": "pkg.mod"}: rewriting "A" → "mod.A" leaves + # "mod" as a bare Name load, which is in rewrites → verification fails. + src = "x = A\n" + result = _rewrite_module_var_names(src, {"A": "mod.A", "mod": "pkg.mod"}) + assert result == src + + +def test_rewrite_module_var_names_verify_syntax_error_returns_original(monkeypatch): + # If re-parsing the rewritten result raises SyntaxError (defensive guard), + # the original source is returned unchanged. + import crispen.file_limiter.code_gen as _code_gen + import ast as _ast + + call_count = [0] + real_parse = _ast.parse + + def patched_parse(src, *args, **kwargs): + call_count[0] += 1 + if call_count[0] >= 2: # fail on the verification parse + raise SyntaxError("synthetic verify failure") + return real_parse(src, *args, **kwargs) + + monkeypatch.setattr(_code_gen.ast, "parse", patched_parse) + src = "x = SAFE_MODE\n" + result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) + assert result == src + + +def test_merge_from_imports_no_overlap(): + imports = ["from .a import x", "from .b import y"] + assert _merge_from_imports(imports) == ["from .a import x", "from .b import y"] + + +def test_merge_from_imports_overlapping(): + imports = ["from .conv import A, C", "from .conv import B, C"] + result = _merge_from_imports(imports) + assert result == ["from .conv import A, B, C"] + + +def test_merge_from_imports_deduplicates_names(): + imports = ["from .m import foo, bar", "from .m import bar, baz"] + result = _merge_from_imports(imports) + assert result == ["from .m import bar, baz, foo"] + + +def test_merge_from_imports_preserves_plain_imports(): + imports = ["import os", "from .m import x", "import sys"] + result = _merge_from_imports(imports) + assert result == ["from .m import x", "import os", "import sys"] + + +def test_merge_from_imports_empty(): + assert _merge_from_imports([]) == [] + + +def test_sort_imports_pep8_basic_ordering(): + # Third-party plain import after relative from-import → should be reordered. + imports = [ + "from typing import Any", + "from .conversion import foo", + "import lupa", + ] + result = _sort_imports_pep8(imports) + assert result == [ + "from typing import Any", + "import lupa", + "from .conversion import foo", + ] + + +def test_sort_imports_pep8_future_first(): + imports = ["import os", "from __future__ import annotations", "from .x import y"] + result = _sort_imports_pep8(imports) + assert result[0] == "from __future__ import annotations" + + +def test_sort_imports_pep8_preserves_within_group_order(): + imports = ["from .b import y", "from .a import x"] + result = _sort_imports_pep8(imports) + # Both are local; original order preserved + assert result == ["from .b import y", "from .a import x"] + + +def test_sort_imports_pep8_empty(): + assert _sort_imports_pep8([]) == [] + + +def test_sort_imports_pep8_all_stdlib(): + imports = ["import os", "import sys", "from pathlib import Path"] + result = _sort_imports_pep8(imports) + assert result == imports # already ordered, stable sort keeps original order + + +def test_topo_depth_empty(): + assert _topo_depth({}) == {} + + +def test_topo_depth_dag(): + # Linear chain: a → b → c. c is the leaf (depth 0), b has depth 1, a depth 2. + # The outer loop visits a first, which recurses into b then c, memoising both. + # When the outer loop reaches b and c they are already in depths (True branch). + graph = {"a": {"b"}, "b": {"c"}, "c": set()} + assert _topo_depth(graph) == {"a": 2, "b": 1, "c": 0} + + +def test_topo_depth_cycle(): + graph = {"a": {"b"}, "b": {"a"}} + assert _topo_depth(graph) == {"a": 0, "b": 0} + + +def test_extract_shared_helpers_extracts_referenced_function(): + # _helper is non-migrated, test_fn (migrated to helpers.py) references it. + e_helper = Entity(EntityKind.FUNCTION, "_helper", 1, 2, ["_helper"]) + e_test = Entity(EntityKind.FUNCTION, "test_fn", 4, 6, ["test_fn"]) + classified, migrated_names = _make_classified([e_helper, e_test], ["test_fn"]) + entity_map = {"_helper": e_helper, "test_fn": e_test} + entity_source_map = { + "_helper": "def _helper():\n pass", + "test_fn": "def test_fn():\n return _helper()", + } + file_entity_names = {"helpers.py": ["test_fn"]} + name_to_target_file = {"_helper": "original.py", "test_fn": "helpers.py"} + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + # _helper extracted into helpers.py (prepended before test_fn) + assert file_entity_names["helpers.py"] == ["_helper", "test_fn"] + assert "_helper" in migrated_names + assert name_to_target_file["_helper"] == "helpers.py" + assert len(synthetic) == 1 + assert synthetic[0].group == ["_helper"] + assert synthetic[0].target_file == "helpers.py" + + +def test_extract_shared_helpers_skips_top_level_entities(): + # TOP_LEVEL entities are not extracted (only FUNCTION/CLASS). + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) + e_test = Entity(EntityKind.FUNCTION, "test_fn", 3, 4, ["test_fn"]) + classified, migrated_names = _make_classified([e_block, e_test], ["test_fn"]) + entity_map = {"_block_1": e_block, "test_fn": e_test} + entity_source_map = { + "_block_1": "_CONST = 42", + "test_fn": "def test_fn():\n return _CONST", + } + file_entity_names = {"helpers.py": ["test_fn"]} + name_to_target_file = {"_CONST": "original.py", "test_fn": "helpers.py"} + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + assert "_block_1" not in migrated_names + assert file_entity_names["helpers.py"] == ["test_fn"] + assert synthetic == [] + + +def test_extract_shared_helpers_extracts_only_once_for_multiple_refs(): + # _helper referenced twice in the same migrated entity → extracted once. + e_helper = Entity(EntityKind.FUNCTION, "_helper", 1, 2, ["_helper"]) + e_test = Entity(EntityKind.FUNCTION, "test_fn", 4, 6, ["test_fn"]) + classified, migrated_names = _make_classified([e_helper, e_test], ["test_fn"]) + entity_map = {"_helper": e_helper, "test_fn": e_test} + entity_source_map = { + "_helper": "def _helper():\n pass", + "test_fn": "def test_fn():\n _helper()\n _helper()", + } + file_entity_names = {"helpers.py": ["test_fn"]} + name_to_target_file = {"_helper": "original.py", "test_fn": "helpers.py"} + + _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + assert file_entity_names["helpers.py"].count("_helper") == 1 + + +def test_extract_shared_helpers_skips_name_already_pointing_to_other_target(): + # A non-migrated FUNCTION entity whose defined name already points to a + # non-original target in name_to_target_file (e.g. a migrated entity also + # defines it) should not be added to defined_to_entity. + e_helper = Entity(EntityKind.FUNCTION, "_helper", 1, 2, ["_helper"]) + e_test = Entity(EntityKind.FUNCTION, "test_fn", 4, 5, ["test_fn"]) + classified, migrated_names = _make_classified([e_helper, e_test], ["test_fn"]) + entity_map = {"_helper": e_helper, "test_fn": e_test} + entity_source_map = { + "_helper": "def _helper(): pass", + "test_fn": "def test_fn(): return _helper()", + } + file_entity_names = {"helpers.py": ["test_fn"]} + # _helper already points to helpers.py (not original) — skip it + name_to_target_file = {"_helper": "helpers.py", "test_fn": "helpers.py"} + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + assert "_helper" not in migrated_names + assert synthetic == [] + + +def test_extract_shared_helpers_no_extraction_when_no_original_dep(): + # test_fn references other_fn which is also migrated → no extraction needed. + e_other = Entity(EntityKind.FUNCTION, "other_fn", 1, 2, ["other_fn"]) + e_test = Entity(EntityKind.FUNCTION, "test_fn", 4, 5, ["test_fn"]) + classified, migrated_names = _make_classified( + [e_other, e_test], ["test_fn", "other_fn"] + ) + entity_map = {"other_fn": e_other, "test_fn": e_test} + entity_source_map = { + "other_fn": "def other_fn():\n pass", + "test_fn": "def test_fn():\n return other_fn()", + } + file_entity_names = {"helpers.py": ["test_fn", "other_fn"]} + name_to_target_file = {"other_fn": "helpers.py", "test_fn": "helpers.py"} + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + assert synthetic == [] + assert file_entity_names["helpers.py"] == ["test_fn", "other_fn"] + + +def test_extract_shared_helpers_transitive_pull_in(): + # _helper_a is directly wanted by fn_a (in f1.py). + # _helper_a's source calls _helper_b (non-migrated, in original). + # _helper_b must be transitively extracted into f1.py to prevent an + # O→f1.py cycle (f1.py imports _helper_a which calls _helper_b in original; + # original re-exports _helper_a from f1.py → cycle). + e_a = Entity(EntityKind.FUNCTION, "_helper_a", 1, 2, ["_helper_a"]) + e_b = Entity(EntityKind.FUNCTION, "_helper_b", 3, 4, ["_helper_b"]) + e_fn = Entity(EntityKind.FUNCTION, "fn_a", 6, 7, ["fn_a"]) + classified, migrated_names = _make_classified([e_a, e_b, e_fn], ["fn_a"]) + entity_map = {"_helper_a": e_a, "_helper_b": e_b, "fn_a": e_fn} + entity_source_map = { + "_helper_a": "def _helper_a():\n _helper_b()", + "_helper_b": "def _helper_b():\n pass", + "fn_a": "def fn_a():\n _helper_a()", + } + file_entity_names = {"f1.py": ["fn_a"]} + name_to_target_file = { + "_helper_a": "original.py", + "_helper_b": "original.py", + "fn_a": "f1.py", + } + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + # Both helpers extracted into f1.py. + assert "_helper_a" in file_entity_names["f1.py"] + assert "_helper_b" in file_entity_names["f1.py"] + assert "_helper_a" in migrated_names + assert "_helper_b" in migrated_names + assert name_to_target_file["_helper_a"] == "f1.py" + assert name_to_target_file["_helper_b"] == "f1.py" + assert len(synthetic) == 2 + + +def test_extract_shared_helpers_scc_prevents_new_to_new_cycle(): + # helper_a is wanted by f1.py; helper_b is wanted by f2.py. + # They mutually reference each other → one SCC → must go to the same file + # to prevent the F1→F2→F1 import cycle. + e_a = Entity(EntityKind.FUNCTION, "helper_a", 1, 2, ["helper_a"]) + e_b = Entity(EntityKind.FUNCTION, "helper_b", 3, 4, ["helper_b"]) + e_fn1 = Entity(EntityKind.FUNCTION, "fn_1", 6, 7, ["fn_1"]) + e_fn2 = Entity(EntityKind.FUNCTION, "fn_2", 9, 10, ["fn_2"]) + classified = ClassifiedEntities( + entities=[e_a, e_b, e_fn1, e_fn2], + entity_class={}, + graph={ + "helper_a": {"helper_b"}, + "helper_b": {"helper_a"}, + "fn_1": set(), + "fn_2": set(), + }, + set_1=[], + set_2_groups=[], + set_3_groups=[], + abort=False, + ) + migrated_names = {"fn_1", "fn_2"} + entity_map = {"helper_a": e_a, "helper_b": e_b, "fn_1": e_fn1, "fn_2": e_fn2} + entity_source_map = { + "helper_a": "def helper_a():\n helper_b()", + "helper_b": "def helper_b():\n helper_a()", + "fn_1": "def fn_1():\n helper_a()", + "fn_2": "def fn_2():\n helper_b()", + } + file_entity_names = {"f1.py": ["fn_1"], "f2.py": ["fn_2"]} + name_to_target_file = { + "helper_a": "original.py", + "helper_b": "original.py", + "fn_1": "f1.py", + "fn_2": "f2.py", + } + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + # Both helpers must land in the same file (f1.py is first in plan order). + assert name_to_target_file["helper_a"] == name_to_target_file["helper_b"] + chosen = name_to_target_file["helper_a"] + assert "helper_a" in file_entity_names[chosen] + assert "helper_b" in file_entity_names[chosen] + assert "helper_a" in migrated_names + assert "helper_b" in migrated_names + # One synthetic placement covering both (single SCC). + assert len(synthetic) == 1 + assert set(synthetic[0].group) == {"helper_a", "helper_b"} + + +def test_extract_shared_helpers_transitive_dep_already_wanted(): + # helper_a is directly wanted by f1.py; helper_b is directly wanted by f2.py. + # helper_a's source also references helper_b (transitive), so helper_b's + # wanting-set grows from {f2.py} to {f1.py, f2.py} — True branch of the + # transitive update condition. + e_a = Entity(EntityKind.FUNCTION, "helper_a", 1, 2, ["helper_a"]) + e_b = Entity(EntityKind.FUNCTION, "helper_b", 3, 4, ["helper_b"]) + e_fn1 = Entity(EntityKind.FUNCTION, "fn_1", 6, 7, ["fn_1"]) + e_fn2 = Entity(EntityKind.FUNCTION, "fn_2", 9, 10, ["fn_2"]) + classified, migrated_names = _make_classified( + [e_a, e_b, e_fn1, e_fn2], ["fn_1", "fn_2"] + ) + entity_map = {"helper_a": e_a, "helper_b": e_b, "fn_1": e_fn1, "fn_2": e_fn2} + entity_source_map = { + "helper_a": "def helper_a():\n helper_b()", + "helper_b": "def helper_b():\n pass", + "fn_1": "def fn_1():\n helper_a()", + "fn_2": "def fn_2():\n helper_b()", + } + file_entity_names = {"f1.py": ["fn_1"], "f2.py": ["fn_2"]} + name_to_target_file = { + "helper_a": "original.py", + "helper_b": "original.py", + "fn_1": "f1.py", + "fn_2": "f2.py", + } + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + # Both helpers are extracted (as separate SCCs since no mutual cycle in graph). + assert "helper_a" in migrated_names + assert "helper_b" in migrated_names + # Two synthetic placements — one for each singleton SCC. + assert len(synthetic) == 2 + + +def test_extract_shared_helpers_transitive_dep_no_new_targets(): + # fn_1 directly references both helper_a and helper_b. + # helper_a's source also references helper_b (transitive dep). + # When the transitive loop processes helper_a, helper_b already has the same + # wanting-set {f1.py} → new_targets is empty → False branch of update condition. + e_a = Entity(EntityKind.FUNCTION, "helper_a", 1, 2, ["helper_a"]) + e_b = Entity(EntityKind.FUNCTION, "helper_b", 3, 4, ["helper_b"]) + e_fn = Entity(EntityKind.FUNCTION, "fn_1", 6, 7, ["fn_1"]) + classified, migrated_names = _make_classified([e_a, e_b, e_fn], ["fn_1"]) + entity_map = {"helper_a": e_a, "helper_b": e_b, "fn_1": e_fn} + entity_source_map = { + "helper_a": "def helper_a():\n helper_b()", + "helper_b": "def helper_b():\n pass", + "fn_1": "def fn_1():\n helper_a()\n helper_b()", + } + file_entity_names = {"f1.py": ["fn_1"]} + name_to_target_file = { + "helper_a": "original.py", + "helper_b": "original.py", + "fn_1": "f1.py", + } + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + # Both helpers are still extracted; the transitive dep on helper_b is a no-op + # because helper_b already has {f1.py} in its wanting-set (direct want). + assert "helper_a" in migrated_names + assert "helper_b" in migrated_names + assert len(synthetic) == 2 + + +def test_extract_shared_helpers_avoids_cycle_by_choosing_downstream_file(): + # _run is wanted by both test_skip.py and test_transformers.py. + # test_skip.py already imports from test_transformers.py (_RaisingTransformer). + # Placing _run in test_skip.py would force test_transformers.py to import from + # test_skip.py → cycle. The cycle-aware logic must pick test_transformers.py + # (the downstream file) instead. + e_raise = Entity( + EntityKind.FUNCTION, "_RaisingTransformer", 1, 3, ["_RaisingTransformer"] + ) + e_run = Entity(EntityKind.FUNCTION, "_run", 4, 5, ["_run"]) + e_skip = Entity(EntityKind.FUNCTION, "fn_skip", 7, 9, ["fn_skip"]) + e_transform = Entity(EntityKind.FUNCTION, "fn_transform", 11, 13, ["fn_transform"]) + classified, migrated_names = _make_classified( + [e_raise, e_run, e_skip, e_transform], + ["fn_skip", "fn_transform", "_RaisingTransformer"], + ) + entity_map = { + "_RaisingTransformer": e_raise, + "_run": e_run, + "fn_skip": e_skip, + "fn_transform": e_transform, + } + entity_source_map = { + "_RaisingTransformer": "def _RaisingTransformer():\n pass", + "_run": "def _run(x):\n return x", + # fn_skip refs _RaisingTransformer (migrated to test_transformers.py) AND + # _run (non-migrated) → _run is wanted by test_skip.py. + "fn_skip": "def fn_skip():\n _RaisingTransformer()\n _run(1)", + # fn_transform also refs _run → _run is wanted by test_transformers.py too. + "fn_transform": "def fn_transform():\n _run(2)", + } + file_entity_names = { + "test_skip.py": ["fn_skip"], + "test_transformers.py": ["fn_transform", "_RaisingTransformer"], + } + name_to_target_file = { + "_RaisingTransformer": "test_transformers.py", + "_run": "original.py", + "fn_skip": "test_skip.py", + "fn_transform": "test_transformers.py", + } + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + # _run must go to test_transformers.py, not test_skip.py. + assert name_to_target_file["_run"] == "test_transformers.py" + assert "_run" in file_entity_names["test_transformers.py"] + assert "_run" not in file_entity_names["test_skip.py"] + assert "_run" in migrated_names + assert len(synthetic) == 1 + assert synthetic[0].group == ["_run"] + assert synthetic[0].target_file == "test_transformers.py" + + +def test_extract_shared_helpers_skips_scc_when_no_cycle_free_placement(): + # fn_1 (in f1.py) refs fn_2 (in f2.py) and fn_2 refs fn_1 → pre-existing + # cycle in file_deps. fn_1 also refs helper_h (non-migrated), which itself + # refs fn_2. The only candidate for helper_h is f1.py; placing it there + # would still result in a cycle (f1.py→f2.py→f1.py already exists). + # Since no cycle-free placement exists, the SCC is skipped entirely. + e_fn1 = Entity(EntityKind.FUNCTION, "fn_1", 1, 2, ["fn_1"]) + e_fn2 = Entity(EntityKind.FUNCTION, "fn_2", 4, 5, ["fn_2"]) + e_h = Entity(EntityKind.FUNCTION, "helper_h", 7, 8, ["helper_h"]) + classified, migrated_names = _make_classified([e_fn1, e_fn2, e_h], ["fn_1", "fn_2"]) + entity_map = {"fn_1": e_fn1, "fn_2": e_fn2, "helper_h": e_h} + entity_source_map = { + "fn_1": "def fn_1():\n fn_2()\n helper_h()", + "fn_2": "def fn_2():\n fn_1()", + "helper_h": "def helper_h():\n fn_2()", + } + file_entity_names = {"f1.py": ["fn_1"], "f2.py": ["fn_2"]} + name_to_target_file = { + "fn_1": "f1.py", + "fn_2": "f2.py", + "helper_h": "original.py", + } + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + # helper_h is skipped — no placement avoids the pre-existing cycle. + assert "helper_h" not in migrated_names + assert synthetic == [] + + +def test_extract_shared_helpers_helper_refs_migrated_entity_in_other_file(): + # helper_a (non-migrated) references fn_2 (migrated to f2.py). + # When placed in f1.py the trial and apply phases must account for the + # resulting f1.py → f2.py dependency edge. + e_fn1 = Entity(EntityKind.FUNCTION, "fn_1", 1, 2, ["fn_1"]) + e_fn2 = Entity(EntityKind.FUNCTION, "fn_2", 4, 5, ["fn_2"]) + e_helper = Entity(EntityKind.FUNCTION, "helper_a", 7, 8, ["helper_a"]) + classified, migrated_names = _make_classified( + [e_fn1, e_fn2, e_helper], ["fn_1", "fn_2"] + ) + entity_map = {"fn_1": e_fn1, "fn_2": e_fn2, "helper_a": e_helper} + entity_source_map = { + "fn_1": "def fn_1():\n helper_a()", + "fn_2": "def fn_2():\n pass", + "helper_a": "def helper_a():\n fn_2()", + } + file_entity_names = {"f1.py": ["fn_1"], "f2.py": ["fn_2"]} + name_to_target_file = { + "fn_1": "f1.py", + "fn_2": "f2.py", + "helper_a": "original.py", + } + + synthetic = _extract_shared_helpers( + file_entity_names, + entity_source_map, + entity_map, + classified, + name_to_target_file, + migrated_names, + "original.py", + ) + + # helper_a is extracted to f1.py; its dep on fn_2 (f2.py) is tracked in + # both the trial and apply dep-file branches. + assert "helper_a" in migrated_names + assert name_to_target_file["helper_a"] == "f1.py" + assert len(synthetic) == 1 + assert synthetic[0].target_file == "f1.py" + + +def test_generate_no_circular_import_when_helper_referenced_by_migrated(): + # Integration test: _run stays in original and is used by test_fn (migrated). + # Without the fix: original → helpers.py (re-export) and helpers.py → original. + # With the fix: _run is moved into helpers.py; original imports _run from helpers. + source = textwrap.dedent( + """\ + def _run(x): + return x + + def test_fn(tmp_path): + return _run(tmp_path) + """ + ) + e_run = _make_entity("_run", 1, 2) + e_test = _make_entity("test_fn", 4, 5) + c = _classified(entities=[e_run, e_test]) + plan = _plan([GroupPlacement(group=["test_fn"], target_file="helpers.py")]) + + result = generate_file_splits(c, plan, source, "original.py") + + helpers_src = result.new_files["helpers.py"] + # _run is defined in helpers.py (extracted), not imported from original + assert "def _run" in helpers_src + assert "from .original import _run" not in helpers_src + # original re-imports _run from helpers.py (since it's still used there via + # non-migrated code — but in this minimal example there's nothing left) + # At minimum, no circular self-import exists + assert "from .original import" not in helpers_src + + +def test_find_project_root_finds_pyproject_toml(tmp_path): + (tmp_path / "pyproject.toml").write_text("") + sub = tmp_path / "pkg" / "module.py" + sub.parent.mkdir() + sub.write_text("x = 1\n") + assert _find_project_root(sub) == tmp_path + + +def test_find_project_root_finds_git(tmp_path): + (tmp_path / ".git").mkdir() + sub = tmp_path / "module.py" + sub.write_text("x = 1\n") + assert _find_project_root(sub) == tmp_path + + +def test_find_project_root_called_with_directory(tmp_path): + (tmp_path / "pyproject.toml").write_text("") + assert _find_project_root(tmp_path) == tmp_path + + +def test_find_project_root_not_found(tmp_path): + # tmp_path is under /tmp which has no project markers → None. + sub = tmp_path / "module.py" + sub.write_text("x = 1\n") + result = _find_project_root(sub) + # If the test runner is inside a project that happens to include tmp_path + # (unlikely but possible with in-tree pytest), just ensure the function + # returns without crashing. The important coverage is the happy path above. + assert result is None or result.exists() + + +def test_module_path_from_file_success(tmp_path): + f = tmp_path / "pkg" / "utils.py" + f.parent.mkdir() + f.write_text("") + assert _module_path_from_file(tmp_path, f) == "pkg.utils" + + +def test_module_path_from_file_top_level(tmp_path): + f = tmp_path / "module.py" + f.write_text("") + assert _module_path_from_file(tmp_path, f) == "module" + + +def test_module_path_from_file_not_under_root(tmp_path): + other = tmp_path.parent / "other.py" + assert _module_path_from_file(tmp_path, other) is None + + +def test_strip_top_level_import_lines_removes_imports(): + src = "import os\nfrom typing import List\n\n_CONST = 1\n" + result = _strip_top_level_import_lines(src) + assert "import os" not in result + assert "from typing import List" not in result + assert "_CONST = 1" in result + + +def test_strip_top_level_import_lines_no_imports(): + src = "_CONST = 1\n" + assert _strip_top_level_import_lines(src) == src + + +def test_strip_top_level_import_lines_syntax_error(): + src = "def (\n" + assert _strip_top_level_import_lines(src) == src + + +def test_strip_top_level_import_lines_strips_type_checking_block(): + # `if TYPE_CHECKING:` blocks must be stripped so that their imports are + # not emitted verbatim in sub-files (wrong path, wrong file). + src = "if TYPE_CHECKING:\n" " from .config import MyConfig\n" "\n" "_CONST = 1\n" + result = _strip_top_level_import_lines(src) + assert "TYPE_CHECKING" not in result + assert "MyConfig" not in result + assert "_CONST = 1" in result + + +def test_extract_module_docstring_present(): + src = '"""My module."""\n\nimport os\n' + assert _extract_module_docstring(src) == '"""My module."""' + + +def test_extract_module_docstring_absent(): + src = "import os\n\ndef foo():\n pass\n" + assert _extract_module_docstring(src) is None + + +def test_extract_module_docstring_syntax_error(): + assert _extract_module_docstring("def (\n") is None + + +def test_extract_module_docstring_non_string_expr(): + # First statement is an expression but not a string constant. + src = "1 + 1\n\ndef foo():\n pass\n" + assert _extract_module_docstring(src) is None + + +def test_strip_module_docstring_removes_docstring(): + src = '"""My module."""\n\n_CONST = 1\n' + result = _strip_module_docstring(src) + assert '"""My module."""' not in result + assert "_CONST = 1" in result + + +def test_strip_module_docstring_no_docstring(): + src = "_CONST = 1\n" + assert _strip_module_docstring(src) == src + + +def test_strip_module_docstring_syntax_error(): + src = "def (\n" + assert _strip_module_docstring(src) == src + + +def test_source_is_only_docstring_true(): + assert _source_is_only_docstring('"""Just a docstring."""\n') is True + + +def test_source_is_only_docstring_with_other_content(): + assert _source_is_only_docstring('"""Doc."""\n\nimport os\n') is False + + +def test_source_is_only_docstring_no_docstring(): + assert _source_is_only_docstring("import os\n") is False + + +def test_source_is_only_docstring_syntax_error(): + assert _source_is_only_docstring("def (\n") is False + + +def test_is_test_name_test_class(): + assert _is_test_name("TestFoo") is True + + +def test_is_test_name_test_function(): + assert _is_test_name("test_bar") is True + + +def test_is_test_name_non_test(): + assert _is_test_name("helper") is False + assert _is_test_name("Foo") is False + assert _is_test_name("_test_private") is False + + +def test_inject_inline_imports_into_function(): + src = "def foo():\n return 1\n" + result = _inject_inline_imports(src, ["from .bar import Baz"]) + assert result == "def foo():\n from .bar import Baz\n return 1\n" + + +def test_inject_inline_imports_skips_docstring(): + src = 'def foo():\n """Doc."""\n return 1\n' + result = _inject_inline_imports(src, ["from .bar import Baz"]) + assert ( + result == 'def foo():\n """Doc."""\n from .bar import Baz\n return 1\n' + ) + + +def test_inject_inline_imports_into_class(): + src = "class Foo:\n x = 1\n" + result = _inject_inline_imports(src, ["from .bar import Baz"]) + assert result == "class Foo:\n from .bar import Baz\n x = 1\n" + + +def test_inject_inline_imports_toplevel_noop(): + # TOP_LEVEL entity (bare if-statement): no body scope, returns unchanged. + src = "if True:\n pass\n" + result = _inject_inline_imports(src, ["from .bar import Baz"]) + assert result == src + + +def test_inject_inline_imports_empty_list_noop(): + src = "def foo():\n pass\n" + assert _inject_inline_imports(src, []) == src + + +def test_inject_inline_imports_syntax_error_noop(): + src = "def (invalid" + assert _inject_inline_imports(src, ["from .x import Y"]) == src + + +def test_inject_inline_imports_empty_source_noop(): + # Empty source parses to empty tree.body — returns unchanged. + assert _inject_inline_imports("", ["from .x import Y"]) == "" + + +def test_inject_inline_imports_only_docstring_injects_after(): + # Function with only a docstring — inserts after docstring (at body[0] line) + # since len(body) == 1. + src = 'def foo():\n """Only doc."""\n' + result = _inject_inline_imports(src, ["from .bar import Baz"]) + assert result == 'def foo():\n from .bar import Baz\n """Only doc."""\n' diff --git a/tests/code_gen/test_named_inline.py b/tests/code_gen/test_named_inline.py new file mode 100644 index 0000000..199610e --- /dev/null +++ b/tests/code_gen/test_named_inline.py @@ -0,0 +1,260 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.code_gen import ( + _inject_inline_test_imports_original, + generate_file_splits, +) +from crispen.file_limiter.entity_parser import Entity, EntityKind +from .helpers import _classified, _plan + + +def test_inject_inline_test_imports_original_basic(): + source = textwrap.dedent( + """\ + def runner(): + TestFoo() + """ + ) + migrated = {"TestFoo": "sub/test_foo.py"} + result = _inject_inline_test_imports_original( + source, migrated, abs_pkg="pkg.tests", original_basename="test_orig.py" + ) + assert "from pkg.tests.sub.test_foo import TestFoo" in result + # Import appears inside the function body, not before the def line. + lines = result.splitlines() + def_idx = next(i for i, l in enumerate(lines) if l.startswith("def runner")) + import_idx = next(i for i, l in enumerate(lines) if "import TestFoo" in l) + assert import_idx > def_idx + + +def test_inject_inline_test_imports_original_skips_docstring(): + source = textwrap.dedent( + """\ + def runner(): + \"\"\"Run tests.\"\"\" + TestFoo() + """ + ) + migrated = {"TestFoo": "sub/test_foo.py"} + result = _inject_inline_test_imports_original( + source, migrated, abs_pkg="tests", original_basename="test_orig.py" + ) + lines = result.splitlines() + doc_idx = next(i for i, l in enumerate(lines) if '"""Run tests."""' in l) + import_idx = next(i for i, l in enumerate(lines) if "import TestFoo" in l) + assert import_idx > doc_idx + + +def test_inject_inline_test_imports_original_no_reference(): + source = "def runner():\n pass\n" + migrated = {"TestFoo": "sub/test_foo.py"} + result = _inject_inline_test_imports_original( + source, migrated, abs_pkg="tests", original_basename="test_orig.py" + ) + assert result == source + + +def test_inject_inline_test_imports_original_empty_map(): + source = "def runner():\n TestFoo()\n" + result = _inject_inline_test_imports_original( + source, {}, abs_pkg="tests", original_basename="test_orig.py" + ) + assert result == source + + +def test_inject_inline_test_imports_original_syntax_error(): + result = _inject_inline_test_imports_original( + "def (invalid", + {"TestFoo": "sub/test_foo.py"}, + abs_pkg="tests", + original_basename="test_orig.py", + ) + assert result == "def (invalid" + + +def test_inject_inline_test_imports_original_relative_import(): + source = "def runner():\n TestFoo()\n" + migrated = {"TestFoo": "sub/test_foo.py"} + result = _inject_inline_test_imports_original( + source, migrated, abs_pkg=None, original_basename="test_orig.py" + ) + assert "from .sub.test_foo import TestFoo" in result + + +def test_inject_inline_test_imports_original_unreferenced_symbol_skipped(): + # Function references `helper` (not test-named) and `other_func`, neither + # of which is in migrated_test_symbols — the false branch of `if tfile:`. + source = "def runner():\n helper()\n other_func()\n" + migrated = {"TestFoo": "sub/test_foo.py"} + result = _inject_inline_test_imports_original( + source, migrated, abs_pkg="tests", original_basename="test_orig.py" + ) + assert result == source + + +def test_generate_test_named_cross_import_inlined(): + # TestHelper migrates to helpers.py; runner stays in original and + # references TestHelper — the import must be injected inside runner's body. + source = textwrap.dedent( + """\ + class TestHelper: + def test_x(self): + pass + + def runner(): + TestHelper() + """ + ) + e_cls = Entity(EntityKind.CLASS, "TestHelper", 1, 3, ["TestHelper"]) + e_run = Entity(EntityKind.FUNCTION, "runner", 5, 6, ["runner"]) + c = _classified(entities=[e_cls, e_run]) + plan = _plan([GroupPlacement(group=["TestHelper"], target_file="helpers.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + orig = result.original_source + # No module-level re-export of TestHelper. + lines = orig.splitlines() + top_level_import_lines = [ + ln for ln in lines if ln.startswith("from") and "TestHelper" in ln + ] + assert top_level_import_lines == [] + # Import appears inside runner's body. + assert " from .helpers import TestHelper" in orig + + +def test_generate_test_named_inline_not_applied_to_toplevel_entity(): + # A TOP_LEVEL entity referencing a test-named symbol falls back to + # module-level import since it has no body scope to inject into. + source = textwrap.dedent( + """\ + class TestHelper: + def test_x(self): + pass + + _inst = TestHelper() + """ + ) + e_cls = Entity(EntityKind.CLASS, "TestHelper", 1, 3, ["TestHelper"]) + e_block = Entity(EntityKind.TOP_LEVEL, "_block_5", 5, 5, ["_inst"]) + c = _classified(entities=[e_cls, e_block]) + plan = _plan( + [GroupPlacement(group=["TestHelper", "_block_5"], target_file="helpers.py")] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + # TestHelper and _block_5 were migrated together — no cross-file issue here. + # Test just ensures no crash and the file is produced. + assert "helpers.py" in result.new_files + + +def test_generate_test_named_inlined_in_function_in_new_file(): + # TestA goes to file_a.py; func_b (which calls TestA) goes to file_b.py. + # The cross-file import of TestA into file_b.py should be injected inline + # inside func_b's body rather than at the top of file_b.py. + source = textwrap.dedent( + """\ + class TestA: + def test_x(self): + pass + + def func_b(): + TestA() + """ + ) + e_a = Entity(EntityKind.CLASS, "TestA", 1, 3, ["TestA"]) + e_b = Entity(EntityKind.FUNCTION, "func_b", 5, 6, ["func_b"]) + c = _classified(entities=[e_a, e_b]) + plan = _plan( + [ + GroupPlacement(group=["TestA"], target_file="file_a.py"), + GroupPlacement(group=["func_b"], target_file="file_b.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + file_b = result.new_files["file_b.py"] + lines = file_b.splitlines() + # No module-level import of TestA. + assert not any(ln.startswith("from") and "TestA" in ln for ln in lines) + # Inline import inside func_b. + assert " from .file_a import TestA" in file_b + + +def test_generate_toplevel_entity_in_new_file_test_import_falls_back_to_module_level(): + # A TOP_LEVEL entity in a new file that references a test-named symbol + # from another new file: no function body to inject into, falls back to + # module-level import. Two TOP_LEVEL entities referencing the same + # test name exercise the dedup path on the second. + source = textwrap.dedent( + """\ + class TestA: + def test_x(self): + pass + + _inst1 = TestA() + + def _sep(): + pass + + _inst2 = TestA() + """ + ) + e_a = Entity(EntityKind.CLASS, "TestA", 1, 3, ["TestA"]) + e_b1 = Entity(EntityKind.TOP_LEVEL, "_block_5", 5, 5, ["_inst1"]) + e_sep = Entity(EntityKind.FUNCTION, "_sep", 7, 8, ["_sep"]) + e_b2 = Entity(EntityKind.TOP_LEVEL, "_block_10", 10, 10, ["_inst2"]) + c = _classified(entities=[e_a, e_b1, e_sep, e_b2]) + plan = _plan( + [ + GroupPlacement(group=["TestA"], target_file="file_a.py"), + GroupPlacement( + group=["_block_5", "_sep", "_block_10"], target_file="file_b.py" + ), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + file_b = result.new_files["file_b.py"] + # Module-level import is acceptable for TOP_LEVEL entities (no body scope). + assert "TestA" in file_b + # Dedup: the same import appears only once despite two TOP_LEVEL entities + # both referencing TestA. + assert file_b.count("import TestA") == 1 + + +def test_generate_cross_import_dedup_across_entities(): + # helper goes to helpers.py; foo and bar both go to workers.py and both + # reference helper — the cross-file import should appear once (dedup). + source = textwrap.dedent( + """\ + def helper(): + pass + + def foo(): + helper() + + def bar(): + helper() + """ + ) + e_h = Entity(EntityKind.FUNCTION, "helper", 1, 2, ["helper"]) + e_foo = Entity(EntityKind.FUNCTION, "foo", 4, 5, ["foo"]) + e_bar = Entity(EntityKind.FUNCTION, "bar", 7, 8, ["bar"]) + c = _classified(entities=[e_h, e_foo, e_bar]) + plan = _plan( + [ + GroupPlacement(group=["helper"], target_file="helpers.py"), + GroupPlacement(group=["foo", "bar"], target_file="workers.py"), + ] + ) + + result = generate_file_splits(c, plan, source, "big.py") + + workers = result.new_files["workers.py"] + # "from .helpers import helper" should appear exactly once. + assert workers.count("import helper") == 1 diff --git a/tests/code_gen/test_prune_unused_imports.py b/tests/code_gen/test_prune_unused_imports.py new file mode 100644 index 0000000..1ca7471 --- /dev/null +++ b/tests/code_gen/test_prune_unused_imports.py @@ -0,0 +1,271 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.code_gen import ( + _prune_inline_redundant_imports, + _prune_unused_imports, + generate_file_splits, +) +from .helpers import _classified, _make_entity, _plan + + +def test_prune_unused_imports_syntax_error(): + # Unparseable source → returned unchanged. + source = "def (invalid syntax" + assert _prune_unused_imports(source) == source + + +def test_prune_unused_imports_no_replacements_needed(): + # All imports are fully used → fast-path returns source unchanged. + source = "import os\n\ndef f():\n os.getcwd()\n" + assert _prune_unused_imports(source) == source + + +def test_prune_unused_imports_preserves_future_import(): + # __future__ imports are always kept, even when the name isn't referenced. + source = "from __future__ import annotations\n\ndef f():\n pass\n" + result = _prune_unused_imports(source) + assert "from __future__ import annotations" in result + + +def test_prune_unused_imports_preserves_star_import(): + # Star imports cannot be pruned — kept as-is. + source = "from os.path import *\n\ndef f():\n pass\n" + result = _prune_unused_imports(source) + assert "from os.path import *" in result + + +def test_prune_unused_imports_removes_fully_unused_plain_import(): + # import whose name is never referenced is dropped entirely. + source = "import sys\n\ndef f():\n pass\n" + result = _prune_unused_imports(source) + assert "import sys" not in result + + +def test_prune_unused_imports_removes_fully_unused_from_import(): + # from-import whose names are never referenced is dropped entirely. + source = "from typing import Dict\n\ndef f():\n return 1\n" + result = _prune_unused_imports(source) + assert "from typing import" not in result + + +def test_prune_unused_imports_narrows_partial_from_import(): + # Only List is used — import narrowed to just List. + source = "from typing import Dict, List\n\ndef f(x: List):\n return x\n" + result = _prune_unused_imports(source) + assert "from typing import List" in result + assert "Dict" not in result + + +def test_prune_unused_imports_narrows_plain_import(): + # import x, y where only y is used → narrowed to import y. + source = "import os, sys\n\ndef f():\n sys.exit()\n" + result = _prune_unused_imports(source) + assert "import sys" in result + assert "os" not in result + + +def test_prune_unused_imports_multiline_import_collapsed(): + # Multi-line parenthesised import is collapsed to a single line. + source = textwrap.dedent( + """\ + from typing import ( + Dict, + List, + ) + + def f(x: List): + return x + """ + ) + result = _prune_unused_imports(source) + assert "from typing import List" in result + assert "Dict" not in result + assert "(\n" not in result + + +def test_prune_unused_imports_relative_import_narrowed(): + # Relative from-import is reconstructed with dots preserved. + source = "from .utils import foo, bar\n\ndef f():\n return foo()\n" + result = _prune_unused_imports(source) + assert "from .utils import foo" in result + assert "bar" not in result + + +def test_prune_unused_imports_preserves_noqa_f401(): + # Imports marked "# noqa: F401" are intentional re-export stubs and must + # never be pruned, even when the name is unused in the file body. + source = ( + "from .utils import _helper # fmt: skip # noqa: F401, E501\n" + "\n" + "def f():\n" + " pass\n" + ) + result = _prune_unused_imports(source) + assert "from .utils import _helper" in result + + +def test_prune_unused_imports_prunes_unused_without_noqa(): + # Without noqa, unused imports are still removed. + source = "from .utils import _helper\n\ndef f():\n pass\n" + result = _prune_unused_imports(source) + assert "from .utils import _helper" not in result + + +def test_prune_inline_syntax_error(): + # Unparseable source → returned unchanged. + source = "def (invalid syntax" + assert _prune_inline_redundant_imports(source) == source + + +def test_prune_inline_no_top_level_imports(): + # No module-level imports → nothing can be redundant, return unchanged. + source = "def f():\n from os import path\n path.join('a', 'b')\n" + assert _prune_inline_redundant_imports(source) == source + + +def test_prune_inline_no_inner_imports(): + # Only top-level imports, no function-body imports → unchanged. + source = "import os\n\ndef f():\n return os.getcwd()\n" + assert _prune_inline_redundant_imports(source) == source + + +def test_prune_inline_no_redundancy(): + # Inner import brings in a different name than the top-level import. + source = "import os\n\ndef f():\n from sys import argv\n return argv\n" + assert _prune_inline_redundant_imports(source) == source + + +def test_prune_inline_removes_fully_redundant_from_import(): + # Top-level import covers all names in the inner from-import → remove it. + source = textwrap.dedent( + """\ + from unittest.mock import patch + from mymod import Foo + + def test_thing(): + from mymod import Foo + assert Foo() + """ + ) + result = _prune_inline_redundant_imports(source) + assert result.count("from mymod import Foo") == 1 + assert "assert Foo()" in result + + +def test_prune_inline_narrows_partially_redundant_from_import(): + # Only one of two inner names is already at top level → narrow the inner import. + source = textwrap.dedent( + """\ + from mymod import Foo + + def test_thing(): + from mymod import Foo, Bar + assert Foo() and Bar() + """ + ) + result = _prune_inline_redundant_imports(source) + lines = result.splitlines() + inner = [ln for ln in lines if "from mymod import" in ln and ln.startswith(" ")] + assert len(inner) == 1 + assert "Bar" in inner[0] + assert "Foo" not in inner[0] + + +def test_prune_inline_removes_fully_redundant_plain_import(): + # Inner ``import x`` where x is already available at top level → removed. + source = textwrap.dedent( + """\ + import os + + def f(): + import os + return os.getcwd() + """ + ) + result = _prune_inline_redundant_imports(source) + assert result.count("import os") == 1 + + +def test_prune_inline_narrows_partially_redundant_plain_import(): + # ``import os, sys`` inside function where os is already top-level → narrows to sys. + source = textwrap.dedent( + """\ + import os + + def f(): + import os, sys + return sys.argv + """ + ) + result = _prune_inline_redundant_imports(source) + inner = [ + ln + for ln in result.splitlines() + if ln.strip().startswith("import") and ln.startswith(" ") + ] + assert len(inner) == 1 + assert "sys" in inner[0] + assert "os" not in inner[0] + + +def test_prune_inline_preserves_indentation(): + # The narrowed replacement line must preserve the original indentation. + source = textwrap.dedent( + """\ + from mymod import Foo + + def test_thing(): + if True: + from mymod import Foo, Bar + assert Bar() + """ + ) + result = _prune_inline_redundant_imports(source) + inner = [ + ln + for ln in result.splitlines() + if "from mymod import" in ln and ln.startswith(" ") + ] + assert len(inner) == 1 + assert inner[0].startswith(" from mymod import Bar") + + +def test_prune_inline_preserves_type_checking_block(): + # Imports inside 'if TYPE_CHECKING:' must never be stripped even when the + # same name is already imported at module level — removing them would leave + # an empty (and syntactically invalid) if-block. + source = textwrap.dedent( + """\ + from typing import TYPE_CHECKING + from mymod import Foo + + if TYPE_CHECKING: + from mymod import Foo + """ + ) + result = _prune_inline_redundant_imports(source) + assert result == source + + +def test_generate_file_splits_removes_inline_redundant_imports(): + # When a split new file has both a top-level import and an inline re-import + # of the same name, the inline one should be removed. + source = textwrap.dedent( + """\ + from mymod import Helper + + def test_uses_helper(): + from mymod import Helper + assert Helper() + """ + ) + entity = _make_entity("test_uses_helper", 3, 5) + c = _classified(entities=[entity]) + plan = _plan( + [GroupPlacement(group=["test_uses_helper"], target_file="test_split.py")] + ) + result = generate_file_splits(c, plan, source, "big.py") + new_src = result.new_files["test_split.py"] + # The inline re-import should be removed; the module-level one covers it. + assert new_src.count("from mymod import Helper") == 1 diff --git a/tests/code_gen/test_pytest_conftest.py b/tests/code_gen/test_pytest_conftest.py new file mode 100644 index 0000000..aaadf9a --- /dev/null +++ b/tests/code_gen/test_pytest_conftest.py @@ -0,0 +1,599 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.code_gen import _merge_conftest_sources, generate_file_splits +from crispen.file_limiter.entity_parser import Entity, EntityKind +from .helpers import _classified, _plan + + +def test_generate_pytest_conftest_disabled_no_conftest(): + # Default (pytest_conftest=False): fixture goes to assigned file, re-exported. + src = "@pytest.fixture\ndef client():\n pass\n" + entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["client"], target_file="fixtures.py")]) + + result = generate_file_splits(c, plan, src, "test_big.py") + + assert "fixtures.py" in result.new_files + assert "conftest.py" not in result.new_files + assert "client" in result.new_files["fixtures.py"] + + +def test_generate_pytest_conftest_subdir_routes_to_subdir_conftest(): + # With pytest_conftest=True AND subdir_name set, fixtures go to + # /conftest.py (not the parent conftest.py). This prevents + # multiple test files in the same directory from conflicting when they + # each have a fixture of the same name. + src = "@pytest.fixture\ndef client():\n pass\n" + entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["client"], target_file="expr/fixtures.py")]) + + result = generate_file_splits( + c, plan, src, "test_big.py", subdir_name="expr", pytest_conftest=True + ) + + assert "expr/conftest.py" in result.new_files + assert "def client():" in result.new_files["expr/conftest.py"] + assert "conftest.py" not in result.new_files # parent conftest untouched + assert "import client" not in result.original_source + + +def test_generate_pytest_conftest_subdir_fixture_referenced_in_remaining_goes_to_parent(): # noqa: E501 + # When a fixture is migrated from a subdir split but its name still appears + # in entities that remain in the original file, route it to the parent + # conftest.py (not the subdir conftest) so those tests can find it. + src = textwrap.dedent( + """\ + @pytest.fixture + def client(): + pass + + def test_big(client): + pass + """ + ) + e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + e_test = Entity(EntityKind.FUNCTION, "test_big", 5, 6, ["test_big"]) + # Only the fixture is migrated; the test stays in the original. + c = _classified(entities=[e_client, e_test]) + plan = _plan([GroupPlacement(group=["client"], target_file="expr/fixtures.py")]) + + result = generate_file_splits( + c, plan, src, "test_big.py", subdir_name="expr", pytest_conftest=True + ) + + # Fixture goes to parent conftest.py, not the subdir one. + assert "conftest.py" in result.new_files + assert "def client():" in result.new_files["conftest.py"] + assert "expr/conftest.py" not in result.new_files + # No import of client back into the original. + assert "import client" not in result.original_source + + +def test_generate_pytest_conftest_subdir_fixture_overrides_parent_conftest(tmp_path): + # When the fixture is referenced in remaining source AND the parent conftest + # already has a fixture with the same name (the module was overriding it), + # the fixture is *copied* (not moved) to the subdir conftest so migrated + # tests get the override; the entity also stays in the original file so + # the original test discovers it from its own module. + parent_conftest = tmp_path / "conftest.py" + parent_conftest.write_text( + "@pytest.fixture\ndef client():\n return 'base'\n", encoding="utf-8" + ) + src = textwrap.dedent( + """\ + @pytest.fixture + def client(): + return 'override' + + def test_big(client): + pass + """ + ) + e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + e_test = Entity(EntityKind.FUNCTION, "test_big", 5, 6, ["test_big"]) + c = _classified(entities=[e_client, e_test]) + plan = _plan([GroupPlacement(group=["client"], target_file="expr/fixtures.py")]) + original_path = str(tmp_path / "test_big.py") + + result = generate_file_splits( + c, plan, src, original_path, subdir_name="expr", pytest_conftest=True + ) + + # Fixture goes to subdir conftest for migrated tests. + assert "expr/conftest.py" in result.new_files + assert "def client():" in result.new_files["expr/conftest.py"] + assert "return 'override'" in result.new_files["expr/conftest.py"] + # Parent conftest is NOT modified (would drop the override via merge). + assert "conftest.py" not in result.new_files + # Fixture stays in original file so the original test finds the override. + assert "def client():" in result.original_source + assert "return 'override'" in result.original_source + # No re-export import injected. + assert "import client" not in result.original_source + + +def test_generate_pytest_conftest_subdir_parent_conftest_imports_only(tmp_path): + # When parent conftest exists but contains only imports (no function defs), + # no conflict is detected and the fixture routes to parent conftest normally. + parent_conftest = tmp_path / "conftest.py" + parent_conftest.write_text("import pytest\n", encoding="utf-8") + src = textwrap.dedent( + """\ + @pytest.fixture + def client(): + pass + + def test_big(client): + pass + """ + ) + e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + e_test = Entity(EntityKind.FUNCTION, "test_big", 5, 6, ["test_big"]) + c = _classified(entities=[e_client, e_test]) + plan = _plan([GroupPlacement(group=["client"], target_file="expr/fixtures.py")]) + original_path = str(tmp_path / "test_big.py") + + result = generate_file_splits( + c, plan, src, original_path, subdir_name="expr", pytest_conftest=True + ) + + # No conflict in parent conftest → fixture routes to parent conftest. + assert "conftest.py" in result.new_files + assert "def client():" in result.new_files["conftest.py"] + assert "expr/conftest.py" not in result.new_files + + +def test_generate_pytest_conftest_subdir_parent_conftest_syntax_error(tmp_path): + # When parent conftest has a syntax error, the OSError/SyntaxError handler + # silently ignores it (no names loaded), so no conflict is detected and the + # fixture routes to parent conftest normally. + parent_conftest = tmp_path / "conftest.py" + parent_conftest.write_text("def (broken syntax", encoding="utf-8") + src = textwrap.dedent( + """\ + @pytest.fixture + def client(): + pass + + def test_big(client): + pass + """ + ) + e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + e_test = Entity(EntityKind.FUNCTION, "test_big", 5, 6, ["test_big"]) + c = _classified(entities=[e_client, e_test]) + plan = _plan([GroupPlacement(group=["client"], target_file="expr/fixtures.py")]) + original_path = str(tmp_path / "test_big.py") + + result = generate_file_splits( + c, plan, src, original_path, subdir_name="expr", pytest_conftest=True + ) + + # Unreadable parent conftest → no conflict detected → parent conftest. + assert "conftest.py" in result.new_files + assert "def client():" in result.new_files["conftest.py"] + assert "expr/conftest.py" not in result.new_files + + +def test_generate_pytest_conftest_fixture_goes_to_conftest(): + # With pytest_conftest=True, fixture entity lands in conftest.py, not the + # LLM-assigned file, and no re-export import appears in the original. + src = "@pytest.fixture\ndef client():\n pass\n" + entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["client"], target_file="fixtures.py")]) + + result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) + + assert "conftest.py" in result.new_files + assert "def client():" in result.new_files["conftest.py"] + # No import of client back into the original (no F401/F811). + assert "import client" not in result.original_source + # The LLM-assigned file is dropped (all entities redirected). + assert "fixtures.py" not in result.new_files + + +def test_generate_pytest_conftest_mixed_group_splits(): + # Fixture goes to conftest.py; non-fixture stays in the assigned file. + src = textwrap.dedent( + """\ + @pytest.fixture + def client(): + pass + + def helper(): + pass + """ + ) + e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + e_helper = Entity(EntityKind.FUNCTION, "helper", 5, 6, ["helper"]) + c = _classified(entities=[e_client, e_helper]) + plan = _plan([GroupPlacement(group=["client", "helper"], target_file="support.py")]) + + result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) + + assert "conftest.py" in result.new_files + assert "def client():" in result.new_files["conftest.py"] + assert "support.py" in result.new_files + assert "def helper():" in result.new_files["support.py"] + assert "import client" not in result.original_source + + +def test_generate_pytest_conftest_no_fixtures_no_conftest(): + # pytest_conftest=True but no fixture entities → no conftest.py created. + src = "def helper():\n pass\n" + entity = Entity(EntityKind.FUNCTION, "helper", 1, 2, ["helper"]) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["helper"], target_file="support.py")]) + + result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) + + assert "conftest.py" not in result.new_files + assert "support.py" in result.new_files + + +def test_generate_pytest_conftest_prepends_existing(tmp_path): + # When conftest.py already exists on disk, its content is prepended. + existing = tmp_path / "conftest.py" + existing.write_text( + "# existing fixture\ndef prior():\n pass\n", encoding="utf-8" + ) + + src = "@pytest.fixture\ndef client():\n pass\n" + entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["client"], target_file="fixtures.py")]) + original_path = str(tmp_path / "test_big.py") + + result = generate_file_splits(c, plan, src, original_path, pytest_conftest=True) + + conftest_src = result.new_files["conftest.py"] + assert "# existing fixture" in conftest_src + assert "def prior():" in conftest_src + assert "def client():" in conftest_src + # Existing content should come first. + assert conftest_src.index("prior") < conftest_src.index("client") + + +def test_generate_pytest_conftest_name_conflict_keeps_in_target(tmp_path): + # When conftest.py already defines a function with the same name as the + # fixture being routed, the fixture stays in its LLM-assigned target file + # instead of being dropped by _merge_conftest_sources. This preserves the + # entity in the split output so that _verify_preservation passes. + existing = tmp_path / "conftest.py" + existing.write_text( + "@pytest.fixture\nasync def client():\n return 'old'\n", encoding="utf-8" + ) + + src = "@pytest.fixture\nasync def client():\n return 'new'\n" + entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["client"], target_file="fixtures.py")]) + original_path = str(tmp_path / "test_big.py") + + result = generate_file_splits(c, plan, src, original_path, pytest_conftest=True) + + # Fixture must appear in the output — in the LLM-assigned file, not conftest. + assert "fixtures.py" in result.new_files + assert "def client():" in result.new_files["fixtures.py"] + # conftest.py should not be created/modified (no new fixtures were routed there). + assert "conftest.py" not in result.new_files + + +def test_generate_pytest_conftest_name_conflict_mixed_group(tmp_path): + # When a placement group contains both a conftest-conflict fixture AND a + # regular function, the fixture is excluded from re-exports but the regular + # function is still re-exported. This covers the branch that rebuilds the + # GroupPlacement with only the non-conflict names. + existing = tmp_path / "conftest.py" + existing.write_text( + "@pytest.fixture\ndef client():\n return 'old'\n", encoding="utf-8" + ) + + src = ( + "@pytest.fixture\ndef client():\n return 'new'\n\n" + "def helper():\n pass\n" + ) + e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + e_helper = Entity(EntityKind.FUNCTION, "helper", 5, 6, ["helper"]) + c = _classified(entities=[e_client, e_helper]) + plan = _plan([GroupPlacement(group=["client", "helper"], target_file="helpers.py")]) + original_path = str(tmp_path / "test_big.py") + + result = generate_file_splits(c, plan, src, original_path, pytest_conftest=True) + + # Both entities migrate to helpers.py. + assert "helpers.py" in result.new_files + assert "def client():" in result.new_files["helpers.py"] + assert "def helper():" in result.new_files["helpers.py"] + # helper is re-exported (public non-fixture); client is not (conftest conflict). + assert "helper" in result.original_source + assert "client" not in result.original_source + + +def test_generate_pytest_conftest_unreadable_conftest_falls_through(tmp_path): + # When conftest.py exists but has a syntax error, the OSError/SyntaxError + # handler silently ignores it and routes the fixture to conftest normally. + existing = tmp_path / "conftest.py" + existing.write_text("def (broken syntax", encoding="utf-8") + + src = "@pytest.fixture\ndef client():\n pass\n" + entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) + c = _classified(entities=[entity]) + plan = _plan([GroupPlacement(group=["client"], target_file="fixtures.py")]) + original_path = str(tmp_path / "test_big.py") + + result = generate_file_splits(c, plan, src, original_path, pytest_conftest=True) + + # With unreadable conftest, routing proceeds normally → fixture goes to conftest. + assert "conftest.py" in result.new_files + assert "def client():" in result.new_files["conftest.py"] + + +def test_generate_stays_fixture_emptied_when_tests_migrated(): + # When a fixture "stays" in the original test file but all tests migrate + # out, the original becomes fixture-only → route fixture to conftest.py + # and empty the original so the engine deletes it. + src = textwrap.dedent( + """\ + import pytest + + @pytest.fixture + def client(): + pass + + def test_foo(client): + pass + """ + ) + e_fixture = Entity(EntityKind.FUNCTION, "client", 3, 5, ["client"]) + e_test = Entity(EntityKind.FUNCTION, "test_foo", 7, 8, ["test_foo"]) + c = _classified(entities=[e_fixture, e_test]) + # Only test_foo is migrated; client "stays" in original. + plan = _plan( + [GroupPlacement(group=["test_foo"], target_file="expression/test_foo.py")] + ) + + result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) + + # Original should be empty (engine will delete it). + assert result.original_source == "" + # Fixture should be routed to conftest.py. + assert "conftest.py" in result.new_files + assert "def client():" in result.new_files["conftest.py"] + + +def test_generate_stays_fixture_merged_with_existing_conftest(tmp_path): + # If conftest.py already exists on disk (e.g. same fixture already there), + # the merge deduplicates so the fixture is not repeated. + existing = tmp_path / "conftest.py" + existing.write_text( + "import pytest\n\n\n@pytest.fixture\ndef client():\n pass\n", + encoding="utf-8", + ) + + src = textwrap.dedent( + """\ + import pytest + + @pytest.fixture + def client(): + pass + + def test_foo(client): + pass + """ + ) + e_fixture = Entity(EntityKind.FUNCTION, "client", 3, 5, ["client"]) + e_test = Entity(EntityKind.FUNCTION, "test_foo", 7, 8, ["test_foo"]) + c = _classified(entities=[e_fixture, e_test]) + plan = _plan( + [GroupPlacement(group=["test_foo"], target_file="expression/test_foo.py")] + ) + original_path = str(tmp_path / "test_expression.py") + + result = generate_file_splits(c, plan, src, original_path, pytest_conftest=True) + + assert result.original_source == "" + # Fixture should appear exactly once in conftest.py (deduplicated). + assert result.new_files["conftest.py"].count("def client():") == 1 + + +def test_generate_stays_fixture_not_emptied_when_tests_remain(): + # If test functions still remain in the original, the fixture-only cleanup + # does NOT trigger — the file should keep both fixture and test. + src = textwrap.dedent( + """\ + import pytest + + @pytest.fixture + def client(): + pass + + def test_foo(client): + pass + + def test_bar(client): + pass + """ + ) + e_fixture = Entity(EntityKind.FUNCTION, "client", 3, 5, ["client"]) + e_test_foo = Entity(EntityKind.FUNCTION, "test_foo", 7, 8, ["test_foo"]) + e_test_bar = Entity(EntityKind.FUNCTION, "test_bar", 10, 11, ["test_bar"]) + c = _classified(entities=[e_fixture, e_test_foo, e_test_bar]) + # Only test_foo migrates; test_bar stays → original still has a test. + plan = _plan( + [GroupPlacement(group=["test_foo"], target_file="expression/test_foo.py")] + ) + + result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) + + # Original should still contain the remaining test and fixture. + assert "def test_bar" in result.original_source + assert result.original_source != "" + + +def test_generate_stays_fixture_not_emptied_when_conftest_disabled(): + # When pytest_conftest=False, stranded fixtures are left in the original. + src = textwrap.dedent( + """\ + import pytest + + @pytest.fixture + def client(): + pass + + def test_foo(client): + pass + """ + ) + e_fixture = Entity(EntityKind.FUNCTION, "client", 3, 5, ["client"]) + e_test = Entity(EntityKind.FUNCTION, "test_foo", 7, 8, ["test_foo"]) + c = _classified(entities=[e_fixture, e_test]) + plan = _plan( + [GroupPlacement(group=["test_foo"], target_file="expression/test_foo.py")] + ) + + result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=False) + + # Original keeps the fixture (conftest routing disabled). + assert "def client():" in result.original_source + assert "conftest.py" not in result.new_files + + +def test_generate_stays_fixture_merged_with_already_written_conftest(): + # If conftest.py was already written by this same split run (e.g. another + # entity was already routed there), merge into it rather than reading disk. + src = textwrap.dedent( + """\ + import pytest + + @pytest.fixture + def client(): + pass + + @pytest.fixture + def db(): + pass + + def test_foo(client): + pass + """ + ) + e_client = Entity(EntityKind.FUNCTION, "client", 3, 5, ["client"]) + e_db = Entity(EntityKind.FUNCTION, "db", 7, 9, ["db"]) + e_test = Entity(EntityKind.FUNCTION, "test_foo", 11, 12, ["test_foo"]) + c = _classified(entities=[e_client, e_db, e_test]) + # db migrates (and goes to conftest.py via pytest routing); test_foo migrates; + # client stays but is then stranded. + plan = _plan( + [ + GroupPlacement(group=["db"], target_file="fixtures.py"), + GroupPlacement(group=["test_foo"], target_file="expression/test_foo.py"), + ] + ) + + result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) + + assert result.original_source == "" + conftest_src = result.new_files["conftest.py"] + # Both migrated db and stranded client fixtures should be in conftest. + assert "def db():" in conftest_src + assert "def client():" in conftest_src + + +def test_merge_conftest_sources_deduplicates_imports(): + # Imports that already exist are not repeated. + existing = "import pytest\n\n\n@pytest.fixture\ndef prior():\n pass\n" + new = "import pytest\n\n\n@pytest.fixture\ndef client():\n pass\n" + result = _merge_conftest_sources(existing, new) + assert result.count("import pytest") == 1 + + +def test_merge_conftest_sources_deduplicates_functions(): + # A function already in existing is not appended again. + existing = "@pytest.fixture\ndef client():\n return 1\n" + new = "@pytest.fixture\ndef client():\n return 2\n" + result = _merge_conftest_sources(existing, new) + assert result.count("def client():") == 1 + assert "return 1" in result + assert "return 2" not in result + + +def test_merge_conftest_sources_appends_new_fixture(): + # A new fixture not in existing is appended. + existing = "@pytest.fixture\ndef prior():\n pass\n" + new = "@pytest.fixture\ndef client():\n pass\n" + result = _merge_conftest_sources(existing, new) + assert "def prior():" in result + assert "def client():" in result + assert result.index("prior") < result.index("client") + + +def test_merge_conftest_sources_no_changes_returns_existing(): + # When nothing new to add, return existing unchanged. + existing = "import pytest\n\n\n@pytest.fixture\ndef client():\n pass\n" + new = "import pytest\n\n\n@pytest.fixture\ndef client():\n pass\n" + result = _merge_conftest_sources(existing, new) + assert result == existing + + +def test_merge_conftest_sources_inserts_new_imports_before_functions(): + # New imports are inserted after existing imports but before functions — no E402. + existing = "import pytest\n\n\n@pytest.fixture\ndef prior():\n pass\n" + new = "import asyncio\n\n\n@pytest.fixture\ndef client():\n pass\n" + result = _merge_conftest_sources(existing, new) + assert "import asyncio" in result + assert "def client():" in result + # Imports must come before the first function definition. + assert result.index("import asyncio") < result.index("def prior():") + + +def test_merge_conftest_sources_syntax_error_fallback(): + # Falls back to simple concatenation when existing cannot be parsed. + existing = "def (broken" + new = "import pytest\n" + result = _merge_conftest_sources(existing, new) + assert "def (broken" in result + assert "import pytest" in result + + +def test_merge_conftest_sources_preserves_comments(): + # Comments in the existing conftest are preserved. + existing = "# shared fixtures\nimport pytest\n\n\ndef prior():\n pass\n" + new = "@pytest.fixture\ndef client():\n pass\n" + result = _merge_conftest_sources(existing, new) + assert "# shared fixtures" in result + assert "def client():" in result + + +def test_merge_conftest_sources_from_import_dedup(): + # from-style imports are also deduplicated via the _import_key F: path. + existing = "from conftest import setup\n\n\ndef prior():\n pass\n" + new = "from conftest import setup\n\n\ndef client():\n pass\n" + result = _merge_conftest_sources(existing, new) + assert result.count("from conftest import setup") == 1 + assert "def client():" in result + + +def test_merge_conftest_sources_only_new_imports_no_defs(): + # When only new imports are added but no new functions, ends with newline. + existing = "import pytest\n\n\ndef prior():\n pass\n" + new = "import asyncio\n" + result = _merge_conftest_sources(existing, new) + assert "import asyncio" in result + assert result.endswith("\n") + # No duplicate function definition appended. + assert result.count("def prior():") == 1 + + +def test_merge_conftest_sources_non_import_non_def_in_new(): + # Bare statements (assignments, expressions) in new_content are silently ignored. + existing = "def prior():\n pass\n" + new = "X = 42\n" + result = _merge_conftest_sources(existing, new) + # Nothing to import or define → returns existing unchanged. + assert result == existing diff --git a/tests/code_gen/test_pytest_fixture_utils.py b/tests/code_gen/test_pytest_fixture_utils.py new file mode 100644 index 0000000..bd9989c --- /dev/null +++ b/tests/code_gen/test_pytest_fixture_utils.py @@ -0,0 +1,219 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.code_gen import ( + _file_has_only_fixtures, + _is_pytest_fixture, + _split_cross_imports_by_test, +) + + +def test_is_pytest_fixture_syntax_error(): + assert _is_pytest_fixture("def (") is False + + +def test_is_pytest_fixture_empty_body(): + # Empty source → empty tree body → not a fixture. + assert _is_pytest_fixture("") is False + + +def test_is_pytest_fixture_class_node(): + # Class definition is not a FunctionDef → returns False. + assert _is_pytest_fixture("class Foo:\n pass\n") is False + + +def test_is_pytest_fixture_no_decorator(): + assert _is_pytest_fixture("def client():\n pass\n") is False + + +def test_is_pytest_fixture_bare_name(): + # @fixture (plain name, no call) + src = "@fixture\ndef client():\n pass\n" + assert _is_pytest_fixture(src) is True + + +def test_is_pytest_fixture_bare_name_called(): + # @fixture() (called with no args) + src = "@fixture()\ndef client():\n pass\n" + assert _is_pytest_fixture(src) is True + + +def test_is_pytest_fixture_attribute(): + # @pytest.fixture (attribute access, no call) + src = "@pytest.fixture\ndef client():\n pass\n" + assert _is_pytest_fixture(src) is True + + +def test_is_pytest_fixture_attribute_called(): + # @pytest.fixture(scope="session") + src = '@pytest.fixture(scope="session")\ndef client():\n pass\n' + assert _is_pytest_fixture(src) is True + + +def test_is_pytest_fixture_non_matching_decorator(): + # @other_decorator — Name but id != "fixture"; not an Attribute. + src = "@other_decorator\ndef client():\n pass\n" + assert _is_pytest_fixture(src) is False + + +def test_split_cross_imports_by_test_pure_non_test(): + non_test, test_named = _split_cross_imports_by_test(["from .foo import helper"]) + assert non_test == ["from .foo import helper"] + assert test_named == [] + + +def test_split_cross_imports_by_test_pure_test(): + non_test, test_named = _split_cross_imports_by_test( + ["from .foo import TestFoo, test_bar"] + ) + assert non_test == [] + assert test_named == ["from .foo import TestFoo, test_bar"] + + +def test_split_cross_imports_by_test_mixed(): + non_test, test_named = _split_cross_imports_by_test( + ["from .foo import TestFoo, helper, test_bar"] + ) + assert non_test == ["from .foo import helper"] + assert test_named == ["from .foo import TestFoo, test_bar"] + + +def test_split_cross_imports_by_test_plain_import_passthrough(): + # Plain "import x" lines (no "from") pass through to non_test unchanged. + non_test, test_named = _split_cross_imports_by_test(["import os"]) + assert non_test == ["import os"] + assert test_named == [] + + +def test_file_has_only_fixtures_syntax_error(): + assert _file_has_only_fixtures("def (") is False + + +def test_file_has_only_fixtures_empty(): + assert _file_has_only_fixtures("") is False + + +def test_file_has_only_fixtures_no_fixture(): + # Regular function only — not a fixture. + assert _file_has_only_fixtures("def helper():\n pass\n") is False + + +def test_file_has_only_fixtures_with_test_function(): + # Has both a fixture and a test function → not fixture-only. + src = textwrap.dedent( + """\ + @pytest.fixture + def client(): + pass + + def test_foo(client): + pass + """ + ) + assert _file_has_only_fixtures(src) is False + + +def test_file_has_only_fixtures_with_test_class(): + # Has both a fixture and a Test class → not fixture-only. + src = textwrap.dedent( + """\ + @pytest.fixture + def client(): + pass + + class TestFoo: + pass + """ + ) + assert _file_has_only_fixtures(src) is False + + +def test_file_has_only_fixtures_with_non_fixture_function(): + # Has a fixture and a plain helper function → not fixture-only. + src = textwrap.dedent( + """\ + @pytest.fixture + def client(): + pass + + def helper(): + pass + """ + ) + assert _file_has_only_fixtures(src) is False + + +def test_file_has_only_fixtures_with_class(): + # Has a fixture and a regular class → not fixture-only. + src = textwrap.dedent( + """\ + @pytest.fixture + def client(): + pass + + class Config: + pass + """ + ) + assert _file_has_only_fixtures(src) is False + + +def test_file_has_only_fixtures_single_fixture(): + # Just a fixture and an import → fixture-only. + src = textwrap.dedent( + """\ + import pytest + + @pytest.fixture + def client(): + pass + """ + ) + assert _file_has_only_fixtures(src) is True + + +def test_file_has_only_fixtures_multiple_fixtures(): + # Multiple fixtures with no tests → fixture-only. + src = textwrap.dedent( + """\ + import pytest + + @pytest.fixture + def client(): + pass + + @pytest.fixture + def db(): + pass + """ + ) + assert _file_has_only_fixtures(src) is True + + +def test_file_has_only_fixtures_async_fixture(): + # Async fixture → fixture-only. + src = textwrap.dedent( + """\ + import pytest + + @pytest.fixture + async def client(): + pass + """ + ) + assert _file_has_only_fixtures(src) is True + + +def test_file_has_only_fixtures_with_docstring(): + # Module docstring + fixture → fixture-only (docstring is allowed). + src = textwrap.dedent( + """\ + \"\"\"Module docstring.\"\"\" + + import pytest + + @pytest.fixture + def client(): + pass + """ + ) + assert _file_has_only_fixtures(src) is True diff --git a/tests/code_gen/test_strip_normalize.py b/tests/code_gen/test_strip_normalize.py new file mode 100644 index 0000000..f061436 --- /dev/null +++ b/tests/code_gen/test_strip_normalize.py @@ -0,0 +1,261 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.code_gen import ( + _multiline_string_ranges, + _normalize_blank_lines, + _strip_orphaned_indented_comments, + _strip_orphaned_section_headers, + _sub_skip_strings, +) + + +def test_strip_orphaned_3line_header_at_eof(): + """3-line block with no code after it is removed.""" + div = "# ---\n" + source = "def foo():\n pass\n\n\n" + div + "# Old Section\n" + div + result = _strip_orphaned_section_headers(source) + assert "# Old Section" not in result + assert "def foo():" in result + + +def test_strip_orphaned_single_line_header_at_eof(): + """Single-line header with no code after it is removed.""" + source = "def foo():\n pass\n\n# --- Removed ---\n" + result = _strip_orphaned_section_headers(source) + assert "# --- Removed ---" not in result + assert "def foo():" in result + + +def test_strip_not_orphaned_3line_header(): + """3-line block followed by substantive code is kept.""" + div = "# ---\n" + source = div + "# Helpers\n" + div + "\n\ndef helper():\n pass\n" + result = _strip_orphaned_section_headers(source) + assert "# Helpers" in result + assert "def helper():" in result + + +def test_strip_not_orphaned_single_line_header(): + """Single-line header followed by substantive code is kept.""" + source = "# --- Tools ---\n\ndef tool():\n pass\n" + result = _strip_orphaned_section_headers(source) + assert "# --- Tools ---" in result + + +def test_strip_orphaned_header_followed_only_by_another_header(): + """Header followed only by another header (and then nothing) — both orphaned.""" + source = "def foo():\n" " pass\n" "\n" "# --- First ---\n" "# --- Second ---\n" + result = _strip_orphaned_section_headers(source) + assert "# --- First ---" not in result + assert "# --- Second ---" not in result + assert "def foo():" in result + + +def test_strip_partial_orphan(): + """Only the header with no code after it is removed; the other stays.""" + source = ( + "# --- Active ---\n" "\n" "def foo():\n" " pass\n" "\n" "# --- Empty ---\n" + ) + result = _strip_orphaned_section_headers(source) + assert "# --- Active ---" in result + assert "# --- Empty ---" not in result + + +def test_strip_no_headers_returns_unchanged(): + """Source with no section headers is returned unchanged.""" + source = "def foo():\n pass\n" + assert _strip_orphaned_section_headers(source) == source + + +def test_strip_all_headers_have_content(): + """When every header has content below it, source is returned unchanged.""" + source = ( + "# --- A ---\n" + "\n" + "def a():\n" + " pass\n" + "\n" + "# --- B ---\n" + "\n" + "def b():\n" + " pass\n" + ) + result = _strip_orphaned_section_headers(source) + assert "# --- A ---" in result + assert "# --- B ---" in result + + +def test_strip_equals_single_line_header_orphaned(): + """=== style orphaned header is also removed.""" + source = "def foo():\n pass\n\n# === OLD SECTION ===\n" + result = _strip_orphaned_section_headers(source) + assert "# === OLD SECTION ===" not in result + + +def test_normalize_blank_lines_strips_leading_blanks(): + """Leading blank lines are removed (prevents E303 at top of file).""" + source = "\n\n\ndef foo():\n pass\n" + result = _normalize_blank_lines(source) + assert result.startswith("def foo():") + + +def test_normalize_blank_lines_collapses_excess_top_level(): + """4+ consecutive newlines between top-level defs collapse to 3.""" + source = "def foo():\n pass\n\n\n\n\ndef bar():\n pass\n" + result = _normalize_blank_lines(source) + assert "\n\n\n\n" not in result + assert "def foo():" in result + assert "def bar():" in result + + +def test_normalize_blank_lines_collapses_body_blanks(): + """2+ blank lines inside an indented body collapse to 1 (prevents E303 in body).""" + source = "def foo():\n x = 1\n\n\n y = 2\n" + result = _normalize_blank_lines(source) + assert "\n\n\n y" not in result + assert "\n\n y" in result + + +def test_normalize_blank_lines_empty_source(): + """Whitespace-only source returns empty string.""" + assert _normalize_blank_lines("\n\n\n") == "" + + +def test_normalize_blank_lines_trailing_newline(): + """Result always ends with exactly one newline.""" + source = "x = 1\n\n\n" + result = _normalize_blank_lines(source) + assert result.endswith("\n") + assert not result.endswith("\n\n") + + +def test_normalize_blank_lines_preserves_multiline_string_body_blanks(): + """Blank lines inside a multi-line string literal are never collapsed. + + Regression: _EXCESS_BLANK_BODY_RE matched \\n{3,}(?=[ \\t]) inside + triple-quoted strings, collapsing 2 blank lines before an indented line + to 1 (e.g. stored source-code fixtures in tests). + """ + # The triple-quoted string contains 2 blank lines before an indented `def`. + # That produces the sequence \\n\\n\\n def inside the raw source, + # which _EXCESS_BLANK_BODY_RE would collapse to \\n\\n def. + source = textwrap.dedent( + """\ + import textwrap + def foo(): + src = textwrap.dedent( + \"\"\"\\ + @dataclass + class _SplitTask: + pass + + + def _find_free_vars(): + x = 1 + \"\"\" + ) + """ + ) + result = _normalize_blank_lines(source) + # Two blank lines before the indented `def` inside the string must survive. + # After outer textwrap.dedent the string content has 8-space indentation. + assert "\n\n\n def _find_free_vars" in result + + +def test_normalize_blank_lines_still_collapses_excess_outside_strings(): + """Blank-line collapsing still fires for code outside string literals.""" + source = "def foo():\n x = 1\n\n\n y = 2\n" + result = _normalize_blank_lines(source) + assert "\n\n\n y" not in result + assert "\n\n y" in result + + +def test_multiline_string_ranges_triple_quoted(): + """Detects a triple-quoted string spanning multiple lines.""" + source = 'x = """\nhello\n"""\n' + ranges = _multiline_string_ranges(source) + assert len(ranges) == 1 + start, end = ranges[0] + assert source[start:end] == '"""\nhello\n"""' + + +def test_multiline_string_ranges_single_line_string_ignored(): + """Single-line strings (no literal newline) are not returned.""" + source = 'x = "hello\\n"\n' + ranges = _multiline_string_ranges(source) + assert ranges == [] + + +def test_multiline_string_ranges_no_strings(): + """Returns empty list when there are no string literals.""" + source = "x = 1 + 2\n" + ranges = _multiline_string_ranges(source) + assert ranges == [] + + +def test_multiline_string_ranges_invalid_source(): + """Falls back to empty list on tokenization error.""" + # Unterminated string triggers TokenError. + source = 'x = """\nhello\n' + ranges = _multiline_string_ranges(source) + assert ranges == [] + + +def test_sub_skip_strings_does_not_touch_string_content(): + """Pattern match inside a multi-line string is not substituted.""" + import re + + pattern = re.compile(r"\n{3,}(?=[ \t])") + source = 'def f():\n s = """\n a\n\n\n b\n """\n' + result = _sub_skip_strings(pattern, "\n\n", source) + # The sequence inside the string must survive unchanged. + assert "\n\n\n b" in result + + +def test_sub_skip_strings_applies_outside_strings(): + """Pattern match outside string literals is substituted normally.""" + import re + + pattern = re.compile(r"\n{3,}(?=[ \t])") + source = "def f():\n x = 1\n\n\n y = 2\n" + result = _sub_skip_strings(pattern, "\n\n", source) + assert "\n\n\n y" not in result + assert "\n\n y" in result + + +def test_sub_skip_strings_no_strings_falls_through(): + """When there are no multi-line strings the plain .sub() path is taken.""" + import re + + pattern = re.compile(r"x") + source = "x = 1\n" + result = _sub_skip_strings(pattern, "y", source) + assert result == "y = 1\n" + + +def test_strip_orphaned_indented_comments_removes_orphan(): + """Indented comment at module level (outside any AST node) is removed.""" + source = "\n\n # This comment was left behind after function removal\n" + result = _strip_orphaned_indented_comments(source) + assert "# This comment was left behind" not in result + + +def test_strip_orphaned_indented_comments_keeps_inside_function(): + """Indented comment inside a function body is preserved.""" + source = "def foo():\n # normal comment\n pass\n" + result = _strip_orphaned_indented_comments(source) + assert "# normal comment" in result + + +def test_strip_orphaned_indented_comments_keeps_module_level_comment(): + """Non-indented module-level comment is preserved.""" + source = "# module comment\ndef foo():\n pass\n" + result = _strip_orphaned_indented_comments(source) + assert "# module comment" in result + + +def test_strip_orphaned_indented_comments_syntax_error(): + """SyntaxError in source returns source unchanged.""" + source = " # orphaned\ndef f(: pass\n" + result = _strip_orphaned_indented_comments(source) + assert result == source diff --git a/tests/code_gen/test_subdir_utils.py b/tests/code_gen/test_subdir_utils.py new file mode 100644 index 0000000..168d363 --- /dev/null +++ b/tests/code_gen/test_subdir_utils.py @@ -0,0 +1,412 @@ +from __future__ import annotations +import textwrap +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.code_gen import _bump_relative_imports, generate_file_splits +from crispen.file_limiter.entity_parser import Entity, EntityKind +from .helpers import _classified, _make_entity, _plan + + +def test_generate_file_splits_subdir_name_uses_init_as_original_basename(): + # When subdir_name="service", the dependency graph treats "service/__init__.py" + # as the original file node. Because main (public) is re-exported from + # __init__, _extract_shared_helpers pulls helper into service/main.py to + # break the __init__ → main → __init__ cycle. The split must not abort. + source = "def helper():\n return 1\n\ndef main():\n return helper()\n" + e_helper = _make_entity("helper", 1, 2) + e_main = _make_entity("main", 4, 5) + c = _classified(entities=[e_helper, e_main]) + # Only main is migrated; helper stays in "original" (→ service/__init__.py). + plan = _plan([GroupPlacement(group=["main"], target_file="service/main.py")]) + + result = generate_file_splits(c, plan, source, "service.py", subdir_name="service") + + assert not result.abort + # helper is extracted into service/main.py to break the re-export cycle. + main_src = result.new_files["service/main.py"] + assert "def helper" in main_src + assert "def main" in main_src + # Re-exports use the short relative prefix ".main", not ".service.main". + assert "from .main import" in result.original_source + assert "from .service.main" not in result.original_source + + +def test_generate_file_splits_subdir_name_re_exports_use_relative_prefix(): + # With subdir_name set (non-test), re-exports in the "original" source + # (which becomes __init__.py) use ".utils" not ".service.utils". + # target_file already has the "service/" prefix (added by runner.py). + source = "def foo():\n pass\n" + e_foo = _make_entity("foo", 1, 2) + c = _classified(entities=[e_foo]) + plan = _plan([GroupPlacement(group=["foo"], target_file="service/utils.py")]) + + result = generate_file_splits(c, plan, source, "service.py", subdir_name="service") + + assert not result.abort + assert "from .utils import foo" in result.original_source + assert "from .service.utils" not in result.original_source + + +def test_generate_file_splits_subdir_name_cross_file_uses_relative(): + # In subdir mode, cross-file imports between new files use relative imports + # even when the original file is a test (abs_pkg would normally apply). + # NOTE: runner.py prefixes target_file with subdir_name before this call, + # so target_files already include "svc/" here. + source = "def helper():\n return 1\n\ndef test_fn():\n return helper()\n" + e_helper = _make_entity("helper", 1, 2) + e_test = _make_entity("test_fn", 4, 5) + c = _classified(entities=[e_helper, e_test]) + plan = _plan( + [ + GroupPlacement(group=["helper"], target_file="svc/helpers.py"), + GroupPlacement(group=["test_fn"], target_file="svc/test_fns.py"), + ] + ) + + # Use a path that looks like a test file so abs_pkg would normally be set. + result = generate_file_splits( + c, plan, source, "tests/test_svc.py", subdir_name="svc" + ) + + assert not result.abort + # Cross-file import from test_fns.py to helpers.py should be relative. + test_src = result.new_files["svc/test_fns.py"] + assert "from .helpers import helper" in test_src + + +def test_generate_file_splits_test_subdir_nonmigrated_imports_from_original(): + # Non-migrated TOP_LEVEL variables (e.g. module-level constants) stay in + # the original test file. A new subfile that references a constant that is + # never reassigned should use a plain ``from`` import (idiomatic Python); + # module-alias access is only needed when the constant is mutated at runtime. + source = "_CONFIG = 'val'\n\ndef test_fn():\n return _CONFIG\n" + # Use TOP_LEVEL kind so _extract_shared_helpers does not pull _CONFIG into + # the new file (it only extracts FUNCTION/CLASS entities). + e_config = Entity(EntityKind.TOP_LEVEL, "_CONFIG", 1, 1, ["_CONFIG"]) + e_test = _make_entity("test_fn", 3, 4) + c = _classified(entities=[e_config, e_test]) + plan = _plan([GroupPlacement(group=["test_fn"], target_file="svc/test_fns.py")]) + + result = generate_file_splits( + c, plan, source, "tests/test_svc.py", subdir_name="svc" + ) + + assert not result.abort + test_src = result.new_files["svc/test_fns.py"] + # _CONFIG is never reassigned → plain from-import (no module alias). + assert "from ..test_svc import _CONFIG" in test_src + assert "from .. import test_svc" not in test_src + assert "test_svc._CONFIG" not in test_src + + +def test_generate_file_splits_has_main_uses_filename_as_original_basename(): + # When has_main=True, original_basename is the flat filename ("service.py"), + # not "service_lib/__init__.py". Re-exports in the original file reference + # the subdir modules directly (e.g. "from service_lib.utils import foo"). + source = "def foo():\n pass\n\nif __name__ == '__main__':\n foo()\n" + e_foo = _make_entity("foo", 1, 2) + c = _classified(entities=[e_foo]) + plan = _plan([GroupPlacement(group=["foo"], target_file="service_lib/utils.py")]) + + result = generate_file_splits( + c, plan, source, "service.py", subdir_name="service_lib", has_main=True + ) + + assert not result.abort + # Re-export in original file uses the subdir module path. + assert "service_lib" in result.original_source + # No __init__.py is created by code_gen (the runner handles that decision). + assert "service_lib/__init__.py" not in result.new_files + + +def test_bump_relative_imports_single_dot(): + assert _bump_relative_imports("from .foo import bar") == "from ..foo import bar" + + +def test_bump_relative_imports_two_dots(): + assert _bump_relative_imports("from .. import baz") == "from ... import baz" + + +def test_bump_relative_imports_leaves_absolute(): + src = "import os\nfrom typing import List" + assert _bump_relative_imports(src) == src + + +def test_bump_relative_imports_multiline(): + src = "from .a import x\nimport sys\nfrom ..b import y\n" + result = _bump_relative_imports(src) + assert "from ..a import x" in result + assert "from ...b import y" in result + assert "import sys" in result + + +def test_bump_relative_imports_n_two(): + assert _bump_relative_imports("from .. import foo", n=2) == "from .... import foo" + + +def test_bump_relative_imports_n_zero(): + src = "from .foo import bar" + assert _bump_relative_imports(src, n=0) == src + + +def test_generate_file_splits_subdir_bumps_needed_imports(): + # In subdir-split mode, relative imports from the original file that appear + # in new sub-files must be incremented by one level so they still resolve + # correctly from inside the subdirectory package. + source = "from .sibling import CONST\n\ndef foo():\n return CONST\n" + e_foo = _make_entity("foo", 3, 4) + c = _classified(entities=[e_foo]) + plan = _plan([GroupPlacement(group=["foo"], target_file="service/utils.py")]) + + result = generate_file_splits(c, plan, source, "service.py", subdir_name="service") + + assert not result.abort + utils_src = result.new_files["service/utils.py"] + assert "from ..sibling import CONST" in utils_src + assert "from .sibling import CONST" not in utils_src + + +def test_generate_file_splits_subdir_bumps_init_imports(): + # In subdir-split mode, relative imports in the non-migrated original source + # (which becomes subdir/__init__.py) must also be bumped by one level so + # they still point at the correct modules from inside the package. + source2 = ( + "from .. import llm_client\n" + "from .base import Base\n\n" + "def stayed():\n return llm_client, Base\n\n" + "def migrated():\n pass\n" + ) + e_stayed2 = _make_entity("stayed", 4, 5) + e_migrated2 = _make_entity("migrated", 7, 8) + c = _classified(entities=[e_stayed2, e_migrated2]) + plan = _plan([GroupPlacement(group=["migrated"], target_file="pkg/helpers.py")]) + + result = generate_file_splits(c, plan, source2, "pkg.py", subdir_name="pkg") + + assert not result.abort + init_src = result.original_source + assert "from ... import llm_client" in init_src + assert "from ..base import Base" in init_src + assert "from .. import llm_client" not in init_src + assert "from .base import Base" not in init_src + + +def test_generate_file_splits_subdir_bumps_two_levels_deep(): + # When the LLM places a new file two directories deep (e.g. + # "pkg/pkg/core.py"), relative imports must be bumped by 2 dots, not 1. + # This matches the real-world scenario where subdir_name="pkg" but the + # advisor proposes "pkg/pkg/core.py" as a target. + source = "from .. import llm_client\n\ndef func():\n return llm_client\n" + e_func = _make_entity("func", 3, 4) + c = _classified(entities=[e_func]) + plan = _plan([GroupPlacement(group=["func"], target_file="pkg/pkg/core.py")]) + + result = generate_file_splits(c, plan, source, "pkg.py", subdir_name="pkg") + + assert not result.abort + core_src = result.new_files["pkg/pkg/core.py"] + # 2 levels deep → original ".." becomes "...." (4 dots) + assert "from .... import llm_client" in core_src + assert "from .. import llm_client" not in core_src + assert "from ... import llm_client" not in core_src + + +def test_generate_file_splits_subdir_injects_tc_import_for_nonmigrated_entity(): + # When a _block_N TOP_LEVEL entity that holds the `if TYPE_CHECKING:` block + # is migrated to a sub-file, any non-migrated entity that references the + # guarded name in a quoted annotation must receive the TYPE_CHECKING import + # in the updated original (__init__.py). + # + # The original file has three entities: + # _block_1 — the TYPE_CHECKING block (migrated to sub.py) + # helper — migrated to sub.py + # entry — stays in __init__.py, references "MyConfig" in annotation + source = ( + "from typing import TYPE_CHECKING\n" + "if TYPE_CHECKING:\n" + " from .config import MyConfig\n" + "\n" + "def helper():\n" + " pass\n" + "\n" + "def entry(cfg: 'MyConfig') -> None:\n" + " helper()\n" + ) + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, []) + e_helper = _make_entity("helper", 5, 6) + e_entry = _make_entity("entry", 8, 9) + c = _classified(entities=[e_block, e_helper, e_entry]) + plan = _plan( + [GroupPlacement(group=["_block_1", "helper"], target_file="pkg/sub.py")] + ) + + result = generate_file_splits(c, plan, source, "pkg.py", subdir_name="pkg") + + assert not result.abort + init_src = result.original_source + # The TYPE_CHECKING import must be injected and bumped for the new depth. + assert "if TYPE_CHECKING:" in init_src + assert "from ..config import MyConfig" in init_src + + +def test_generate_subdir_module_docstring_goes_to_init(): + # In subdir-split mode the module docstring belongs in __init__.py, not + # in the split-off child module. Migrate the preamble entity (_block_1) + # along with foo so the docstring is removed from the original source, + # triggering the restore-to-__init__ logic. + source = textwrap.dedent( + """\ + \"\"\"Top-level module doc.\"\"\" + + import os + + def foo(): + return os.sep + + def bar(): + return foo() + """ + ) + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["os"]) + e_foo = _make_entity("foo", 5, 6) + e_bar = _make_entity("bar", 8, 9) + c = _classified(entities=[e_block, e_foo, e_bar]) + plan = _plan( + [GroupPlacement(group=["_block_1", "foo"], target_file="pkg/helpers.py")] + ) + + result = generate_file_splits(c, plan, source, "pkg.py", subdir_name="pkg") + + assert not result.abort + init_src = result.original_source + helpers_src = result.new_files["pkg/helpers.py"] + # Docstring belongs in __init__.py. + assert '"""Top-level module doc."""' in init_src + # Docstring must NOT appear in the child module. + assert '"""Top-level module doc."""' not in helpers_src + + +def test_generate_subdir_docstring_already_in_init_not_duplicated(): + # If the TOP_LEVEL entity stays in the original (not migrated), the + # docstring remains in the updated source and must not be prepended again. + source = textwrap.dedent( + """\ + \"\"\"Top-level module doc.\"\"\" + + _CONST = 1 + + def stayed(): + return _CONST + + def migrated(): + pass + """ + ) + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["_CONST"]) + e_stayed = _make_entity("stayed", 5, 6) + e_migrated = _make_entity("migrated", 8, 9) + c = _classified(entities=[e_block, e_stayed, e_migrated]) + plan = _plan([GroupPlacement(group=["migrated"], target_file="pkg/helpers.py")]) + + result = generate_file_splits(c, plan, source, "pkg.py", subdir_name="pkg") + + assert not result.abort + init_src = result.original_source + assert init_src.count('"""Top-level module doc."""') == 1 + + +def test_generate_subdir_module_docstring_goes_to_test_init(): + # For test-file subdir splits the module docstring goes into + # subdir/__init__.py, not into the re-export stub file. + source = textwrap.dedent( + """\ + \"\"\"Tests for the runner module.\"\"\" + + import os + + def test_foo(): + return os.sep + + def test_bar(): + return test_foo() + """ + ) + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["os"]) + e_foo = _make_entity("test_foo", 5, 6) + e_bar = _make_entity("test_bar", 8, 9) + c = _classified(entities=[e_block, e_foo, e_bar]) + plan = _plan( + [GroupPlacement(group=["_block_1", "test_foo"], target_file="svc/test_foo.py")] + ) + + result = generate_file_splits( + c, plan, source, "tests/test_svc.py", subdir_name="svc" + ) + + assert not result.abort + init_src = result.new_files["svc/__init__.py"] + child_src = result.new_files["svc/test_foo.py"] + updated_src = result.original_source + # Docstring belongs in __init__.py. + assert '"""Tests for the runner module."""' in init_src + # Docstring must NOT appear in the child test file or the stub file. + assert '"""Tests for the runner module."""' not in child_src + assert '"""Tests for the runner module."""' not in updated_src + + +def test_generate_subdir_test_docstring_only_remaining_clears_original(): + # Regression: when a test-file subdir split migrates all entities and the + # only thing left in the original is the module docstring (a TOP_LEVEL + # entity that is not migrated by _remove_entity_lines), the docstring must + # be routed to __init__.py and the original file must be cleared for + # deletion by the engine. + source = textwrap.dedent( + """\ + \"\"\"Tests for the widget module. + Covers edge cases. + \"\"\" + + def test_alpha(): + pass + + def test_beta(): + pass + """ + ) + # The module docstring is a TOP_LEVEL entity spanning lines 1-3. + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, []) + e_alpha = _make_entity("test_alpha", 5, 6) + e_beta = _make_entity("test_beta", 8, 9) + c = _classified(entities=[e_block, e_alpha, e_beta]) + # Only the test functions are migrated; the TOP_LEVEL entity stays. + plan = _plan( + [ + GroupPlacement(group=["test_alpha"], target_file="widget/test_alpha.py"), + GroupPlacement(group=["test_beta"], target_file="widget/test_beta.py"), + ] + ) + + result = generate_file_splits( + c, plan, source, "tests/test_widget.py", subdir_name="widget" + ) + + assert not result.abort + # Docstring must end up in __init__.py. + init_src = result.new_files["widget/__init__.py"] + assert '"""Tests for the widget module.' in init_src + # Original source must be empty so the engine deletes it. + assert result.original_source == "" + + +def test_generate_subdir_docstring_not_stripped_from_non_subdir_split(): + # Outside subdir-split mode, a TOP_LEVEL entity's docstring is preserved + # in the new file (only imports are stripped, not docstrings). + source = '"""Module doc."""\n\nimport os\n\ndef foo():\n return os.sep\n' + e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["os"]) + e_foo = _make_entity("foo", 5, 6) + c = _classified(entities=[e_block, e_foo]) + plan = _plan([GroupPlacement(group=["_block_1", "foo"], target_file="utils.py")]) + + result = generate_file_splits(c, plan, source, "big.py") + + new_src = result.new_files["utils.py"] + assert '"""Module doc."""' in new_src diff --git a/tests/duplicate_extractor/__init__.py b/tests/duplicate_extractor/__init__.py new file mode 100644 index 0000000..3dfec62 --- /dev/null +++ b/tests/duplicate_extractor/__init__.py @@ -0,0 +1 @@ +"""Tests for duplicate_extractor: 100% branch coverage.""" diff --git a/tests/duplicate_extractor/test_collectors.py b/tests/duplicate_extractor/test_collectors.py new file mode 100644 index 0000000..33eee0a --- /dev/null +++ b/tests/duplicate_extractor/test_collectors.py @@ -0,0 +1,360 @@ +import textwrap +from libcst.metadata import MetadataWrapper +from crispen.refactors.duplicate_extractor import ( + _FunctionCollector, + _FunctionInfo, + _SeqInfo, + _SequenceCollector, + _build_function_body_fps, + _filter_maximal_groups, + _find_duplicate_groups, + _has_def, + _has_internal_overlap, + _normalize_source, +) +import libcst as cst +from .test_node_utils import _make_seq + + +def test_find_duplicate_groups_empty(): + assert _find_duplicate_groups([], [(1, 5)]) == [] + + +def test_find_duplicate_groups_singleton(): + seq = _make_seq(1, 3) + seq.fingerprint = "fp1" + seqs = [seq] + # Only one seq with this fingerprint — not a duplicate + assert _find_duplicate_groups(seqs, [(1, 3)]) == [] + + +def test_find_duplicate_groups_no_diff_overlap(): + s1 = _SeqInfo([], 1, 3, "", "", "fp1") + s2 = _SeqInfo([], 10, 12, "", "", "fp1") + # Neither overlaps diff range (20, 30) + assert _find_duplicate_groups([s1, s2], [(20, 30)]) == [] + + +def test_find_duplicate_groups_valid(): + s1 = _SeqInfo([], 1, 3, "", "", "fp1") + s2 = _SeqInfo([], 10, 12, "", "", "fp1") + groups = _find_duplicate_groups([s1, s2], [(1, 3)]) + assert len(groups) == 1 + assert set(id(s) for s in groups[0]) == {id(s1), id(s2)} + + +def test_has_internal_overlap_no_overlap(): + s1 = _SeqInfo([], 1, 3, "", "", "fp1") + s2 = _SeqInfo([], 10, 12, "", "", "fp1") + assert not _has_internal_overlap([s1, s2]) + + +def test_has_internal_overlap_adjacent_no_overlap(): + # end_line of s1 == start_line - 1 of s2: not overlapping + s1 = _SeqInfo([], 1, 5, "", "", "fp1") + s2 = _SeqInfo([], 6, 10, "", "", "fp1") + assert not _has_internal_overlap([s1, s2]) + + +def test_has_internal_overlap_touching(): + # end_line of s1 == start_line of s2: overlap (shared boundary line) + s1 = _SeqInfo([], 1, 5, "", "", "fp1") + s2 = _SeqInfo([], 5, 9, "", "", "fp1") + assert _has_internal_overlap([s1, s2]) + + +def test_has_internal_overlap_proper_overlap(): + s1 = _SeqInfo([], 27, 30, "", "", "fp1") + s2 = _SeqInfo([], 29, 32, "", "", "fp1") + assert _has_internal_overlap([s1, s2]) + + +def test_has_internal_overlap_unsorted_order(): + # Sequences given in reverse order — function must sort before checking. + s1 = _SeqInfo([], 29, 32, "", "", "fp1") + s2 = _SeqInfo([], 27, 30, "", "", "fp1") + assert _has_internal_overlap([s1, s2]) + + +def test_find_duplicate_groups_skips_internally_overlapping(): + # Simulate the op_range pattern: two pairs [A,B] and [B,C] that share a + # statement. The group has internal overlap and must be filtered out. + s1 = _SeqInfo([], 27, 30, "", "", "fp1") + s2 = _SeqInfo([], 29, 32, "", "", "fp1") + # Diff covers both sequences. + groups = _find_duplicate_groups([s1, s2], [(27, 32)]) + assert groups == [] + + +def test_find_duplicate_groups_caps_at_max_groups(): + sequences = [] + for i in range(6): + fp = f"fp{i}" + # Place each group in a disjoint band of 20 lines so _filter_maximal_groups + # keeps all 6 (none overlap), and the max_groups=3 cap is what limits output. + sequences.append(_SeqInfo([], i * 20 + 1, i * 20 + 3, "", "", fp)) + sequences.append(_SeqInfo([], i * 20 + 10, i * 20 + 12, "", "", fp)) + # Diff range covers all sequences so the diff-overlap filter passes for all. + groups = _find_duplicate_groups(sequences, [(1, 130)], max_groups=3) + assert len(groups) == 3 + + +def test_filter_maximal_groups_empty(): + assert _filter_maximal_groups([]) == [] + + +def test_filter_maximal_groups_single_group(): + s1 = _SeqInfo([], 1, 10, "", "", "fp1") + s2 = _SeqInfo([], 20, 29, "", "", "fp1") + group = [s1, s2] + result = _filter_maximal_groups([group]) + assert result == [group] + + +def test_filter_maximal_groups_removes_subsumed(): + # Large group spans lines 1-10; small group spans 1-5 (subset). + # Only the large group should be kept. + s_large_a = _SeqInfo([], 1, 10, "", "", "fp_large") + s_large_b = _SeqInfo([], 20, 29, "", "", "fp_large") + large_group = [s_large_a, s_large_b] + + s_small_a = _SeqInfo([], 1, 5, "", "", "fp_small") + s_small_b = _SeqInfo([], 20, 24, "", "", "fp_small") + small_group = [s_small_a, s_small_b] + + result = _filter_maximal_groups([small_group, large_group]) + assert len(result) == 1 + assert result[0] is large_group + + +def test_filter_maximal_groups_keeps_non_overlapping(): + # Two groups with completely disjoint line ranges — both should be kept. + s1a = _SeqInfo([], 1, 5, "", "", "fp1") + s1b = _SeqInfo([], 30, 34, "", "", "fp1") + group1 = [s1a, s1b] + + s2a = _SeqInfo([], 10, 14, "", "", "fp2") + s2b = _SeqInfo([], 40, 44, "", "", "fp2") + group2 = [s2a, s2b] + + result = _filter_maximal_groups([group1, group2]) + assert len(result) == 2 + + +def _make_func_info(name: str, body_source: str = " pass\n") -> _FunctionInfo: + return _FunctionInfo( + name=name, + source=f"def {name}():\n{body_source}", + scope="", + body_source=body_source, + body_stmt_count=1, + params=[], + ) + + +def test_build_fps_includes_called(): + body = " x = 1\n y = 2\n z = 3\n" + func = _make_func_info("foo", body) + fps = _build_function_body_fps([func], {"foo"}) + fp = _normalize_source(body) + assert fp in fps + assert fps[fp].name == "foo" + + +def test_build_fps_excludes_uncalled(): + func = _make_func_info("bar") + fps = _build_function_body_fps([func], {"foo"}) + assert fps == {} + + +def test_build_fps_empty_functions(): + fps = _build_function_body_fps([], {"foo"}) + assert fps == {} + + +def _collect_sequences(source: str, max_seq_len: int = 8): + tree = cst.parse_module(source) + lines = source.splitlines(keepends=True) + collector = _SequenceCollector(lines, max_seq_len=max_seq_len) + MetadataWrapper(tree).visit(collector) + return collector.sequences + + +def test_collector_finds_sequences(): + source = textwrap.dedent( + """\ + def foo(): + a = 1 + b = 2 + c = 3 + """ + ) + seqs = _collect_sequences(source) + assert len(seqs) > 0 + + +def test_collector_skips_light_sequences(): + # Only 2 statements — below weight threshold of 3 + source = textwrap.dedent( + """\ + def foo(): + a = 1 + b = 2 + """ + ) + seqs = _collect_sequences(source) + assert all(seq.start_line != seq.end_line or len(seq.stmts) >= 2 for seq in seqs) + # All 2-stmt windows skipped because weight < 3 + assert len([s for s in seqs if len(s.stmts) == 2]) == 0 + + +def test_collector_skips_defs(): + source = textwrap.dedent( + """\ + def foo(): + pass + def bar(): + pass + def baz(): + pass + """ + ) + seqs = _collect_sequences(source) + # Module-level sequences of defs should be skipped + for seq in seqs: + assert not _has_def(seq.stmts) + + +def test_collector_scope_tracking(): + source = textwrap.dedent( + """\ + def my_func(): + a = 1 + b = 2 + c = 3 + """ + ) + seqs = _collect_sequences(source) + func_seqs = [s for s in seqs if s.scope == "my_func"] + assert len(func_seqs) > 0 + + +def test_sequence_collector_custom_max_seq_len(): + # max_seq_len=2 means windows are at most 2 statements. + # With 4 statements each of weight 1, all 2-stmt windows have weight 2 < + # MIN_WEIGHT=3. So no sequences pass the weight filter → sequences == []. + source = textwrap.dedent( + """\ + def foo(): + a = 1 + b = 2 + c = 3 + d = 4 + """ + ) + seqs = _collect_sequences(source, max_seq_len=2) + # No 3-stmt (or larger) windows generated; all ≤2-stmt windows fail weight check. + assert all(len(s.stmts) <= 2 for s in seqs) + assert seqs == [] + + +def _collect_functions(source: str): + tree = cst.parse_module(source) + lines = source.splitlines(keepends=True) + collector = _FunctionCollector(lines) + MetadataWrapper(tree).visit(collector) + return collector.functions + + +def test_function_collector_module_level(): + source = "def foo():\n pass\n" + funcs = _collect_functions(source) + assert len(funcs) == 1 + assert funcs[0].name == "foo" + assert funcs[0].scope == "" + assert funcs[0].body_stmt_count == 1 + assert funcs[0].params == [] + + +def test_function_collector_class_level(): + source = "class C:\n def method(self):\n pass\n" + funcs = _collect_functions(source) + assert len(funcs) == 1 + assert funcs[0].name == "method" + assert funcs[0].scope == "C" + assert funcs[0].body_stmt_count == 1 + assert funcs[0].params == ["self"] + + +def test_function_collector_skips_nested(): + source = "def outer():\n def inner():\n pass\n" + funcs = _collect_functions(source) + assert len(funcs) == 1 + assert funcs[0].name == "outer" + assert funcs[0].body_stmt_count == 1 + assert funcs[0].params == [] + + +def test_function_collector_collects_body_source(): + source = "def foo():\n x = 1\n y = 2\n" + funcs = _collect_functions(source) + assert len(funcs) == 1 + assert "x = 1" in funcs[0].body_source + + +def test_function_collector_collects_stmt_count(): + source = "def foo():\n pass\n" + funcs = _collect_functions(source) + assert funcs[0].body_stmt_count == 1 + + +def test_function_collector_collects_params(): + source = "def f(x, y):\n pass\n" + funcs = _collect_functions(source) + assert funcs[0].params == ["x", "y"] + + +def test_function_collector_no_params(): + source = "def f():\n pass\n" + funcs = _collect_functions(source) + assert funcs[0].params == [] + + +def test_sequence_collector_class_scope(): + """_SequenceCollector sets class_scope for sequences inside class methods.""" + + source = textwrap.dedent( + """\ + x = 1 + y = 2 + z = 3 + + class MyClass: + def method(self): + a = 1 + b = 2 + c = 3 + """ + ) + lines = source.splitlines(keepends=True) + tree = cst.parse_module(source) + collector = _SequenceCollector(lines, max_seq_len=8) + MetadataWrapper(tree).visit(collector) + + module_seqs = [s for s in collector.sequences if s.class_scope is None] + class_seqs = [s for s in collector.sequences if s.class_scope == "MyClass"] + assert module_seqs, "expected module-level sequences with class_scope=None" + assert class_seqs, "expected class-method sequences with class_scope='MyClass'" + + +def test_sequence_collector_min_weight_filters_light_sequences(): + # A single assignment has weight 1. With min_weight=2 it should be excluded. + source = "def f():\n a = 1\n b = 2\n" + source_lines = source.splitlines(keepends=True) + tree = cst.parse_module(source) + + collector = _SequenceCollector(source_lines, max_seq_len=2, min_weight=2) + MetadataWrapper(tree).visit(collector) + # Single-statement sequences (weight=1) should be filtered out + single_stmt_seqs = [s for s in collector.sequences if len(s.stmts) == 1] + assert single_stmt_seqs == [] diff --git a/tests/duplicate_extractor/test_edit_operations.py b/tests/duplicate_extractor/test_edit_operations.py new file mode 100644 index 0000000..7a5bf44 --- /dev/null +++ b/tests/duplicate_extractor/test_edit_operations.py @@ -0,0 +1,535 @@ +from unittest.mock import MagicMock, patch +import textwrap +from crispen.refactors.duplicate_extractor import ( + DuplicateExtractor, + _SeqInfo, + _apply_edits, + _build_helper_insertion, + _find_insertion_point, + _normalize_replacement_indentation, + _skip_class_docstring, +) +from .test_extractor_core import ( + _make_extract_response, + _make_verify_response, + _make_veto_response, +) + + +def _make_seq_with_source(source: str) -> _SeqInfo: + return _SeqInfo( + stmts=[], start_line=1, end_line=1, scope="f", source=source, fingerprint="" + ) + + +def test_normalize_indentation_already_correct(): + # Replacement already matches the block's indentation — unchanged. + seq = _make_seq_with_source(" x = compute()\n y = finalize(x)\n") + replacement = " result = helper()\n" + assert ( + _normalize_replacement_indentation(seq, replacement) + == " result = helper()\n" + ) + + +def test_normalize_indentation_col0_to_indented(): + # Replacement at column 0 is re-indented to match the original block. + seq = _make_seq_with_source(" x = compute()\n y = finalize(x)\n") + replacement = "result = helper()\n" + assert ( + _normalize_replacement_indentation(seq, replacement) + == " result = helper()\n" + ) + + +def test_normalize_indentation_multiline(): + # Multi-line replacement at column 0 gets uniformly re-indented. + seq = _make_seq_with_source(" x = a()\n y = b(x)\n") + replacement = "x = helper()\nif x is None:\n x = default()\n" + expected = ( + " x = helper()\n if x is None:\n x = default()\n" + ) + assert _normalize_replacement_indentation(seq, replacement) == expected + + +def test_normalize_indentation_module_level_block(): + # Module-level block (no indent) — replacement is just dedented. + seq = _make_seq_with_source("x = compute()\ny = finalize(x)\n") + replacement = "result = helper()\n" + assert _normalize_replacement_indentation(seq, replacement) == "result = helper()\n" + + +def test_normalize_indentation_empty_source(): + # Empty source — no indentation can be inferred; replacement returned as-is. + seq = _make_seq_with_source("") + replacement = "result = helper()\n" + assert _normalize_replacement_indentation(seq, replacement) == replacement + + +def test_apply_edits_no_edits(): + source = "a = 1\nb = 2\n" + assert _apply_edits(source, []) == source + + +def test_apply_edits_replacement(): + source = "a = 1\nb = 2\nc = 3\n" + # Replace line index 1 (b = 2) with new content + result = _apply_edits(source, [(1, 2, "x = 99\n")]) + assert result == "a = 1\nx = 99\nc = 3\n" + + +def test_apply_edits_insertion(): + source = "a = 1\nb = 2\n" + # Insert before line index 1 (b = 2) + result = _apply_edits(source, [(1, 1, "INSERTED\n")]) + assert result == "a = 1\nINSERTED\nb = 2\n" + + +def test_apply_edits_overlapping_skipped(): + source = "a = 1\nb = 2\nc = 3\n" + edits = [ + (0, 2, "FIRST\n"), + (1, 3, "SECOND\n"), # overlaps with first + ] + result = _apply_edits(source, edits) + # Higher-start edit (SECOND) wins; FIRST overlaps and is skipped + assert "SECOND" in result + assert "FIRST" not in result + + +def test_apply_edits_no_trailing_newline_source(): + source = "a = 1" # no trailing newline + result = _apply_edits(source, [(0, 1, "b = 2\n")]) + assert result == "b = 2\n" + + +def test_apply_edits_no_trailing_newline_text(): + source = "a = 1\nb = 2\n" + # Replacement text without trailing newline + result = _apply_edits(source, [(0, 1, "x = 99")]) + assert result == "x = 99\nb = 2\n" + + +def test_find_insertion_point_module_with_imports(): + source = "import os\nfrom sys import argv\n\ndef foo():\n pass\n" + # Should insert after the last import (index 1), so return 2 + assert _find_insertion_point(source, "") == 2 + + +def test_find_insertion_point_module_no_imports(): + source = "a = 1\n" + # No imports: last_import stays -1, returns 0 + assert _find_insertion_point(source, "") == 0 + + +def test_find_insertion_point_function_found(): + source = "import os\n\ndef target():\n pass\n" + # def target is at line index 2 + assert _find_insertion_point(source, "target") == 2 + + +def test_find_insertion_point_function_not_found(): + source = "a = 1\n" + # Falls back to 0 + assert _find_insertion_point(source, "missing_func") == 0 + + +def test_find_insertion_point_class_method_inserts_before_class(): + # def bar is indented inside class Foo; helper must go before the class, + # not inside it (which would end the class and turn _analyze into a nested func). + source = "import os\n\nclass Foo:\n\n def bar(self):\n pass\n" + # source_lines: ["import os", "", "class Foo:", "", + # " def bar(self):", " pass"] + # "def bar" found at i=4 (indent=4). Walk back: + # j=3 → blank → skip; j=2 → "class Foo:" indent=0 < 4 → return 2 + assert _find_insertion_point(source, "bar") == 2 + + +def test_find_insertion_point_nested_function_no_class(): + # def inner is indented inside def outer (no enclosing class). + # method_indent > 0, loop finds a non-class def at lower indent → break. + # Falls through to decorator walk, which returns i (the line of def inner). + source = "def outer():\n def inner():\n pass\n" + # "def inner" found at i=1 (indent=4). Walk back: + # j=0 → "def outer():" indent=0 < 4, not a class → break. + # Falls through to return 1. + assert _find_insertion_point(source, "inner") == 1 + + +def test_find_insertion_point_nested_func_ignores_unrelated_class(): + # Regression: a nested function inside a module-level function must not + # be confused with a class method just because an unrelated class appears + # earlier in the file. Before the fix the backward walk would skip past + # the outer function (non-class, lower indent) and incorrectly match the + # unrelated class, causing the helper to be inserted between the class's + # decorator and its class statement. + import textwrap as _textwrap + + source = _textwrap.dedent( + """\ + @dataclass + class _SplitTask: + pass + + + def _find_free_vars(): + x = 1 + def _collect_loads(): + pass + """ + ) + # source_lines (0-based): + # 0: "@dataclass\n" + # 1: "class _SplitTask:\n" + # 2: " pass\n" + # 3: "\n" + # 4: "\n" + # 5: "def _find_free_vars():\n" + # 6: " x = 1\n" + # 7: " def _collect_loads():\n" + # 8: " pass\n" + # "def _collect_loads" found at i=7 (indent=4). Walk back: + # j=6: " x = 1" indent=4, not < 4 → continue + # j=5: "def _find_free_vars():" indent=0 < 4, NOT class → break + # Falls through to decorator walk: j=6 (" x = 1"), not a decorator + # → break → return j+1 = 7. + # The old (unfixed) code would have continued past j=5 and returned 1, + # placing the helper between @dataclass and class _SplitTask:. + result = _find_insertion_point(source, "_collect_loads") + assert result != 1, "must not insert inside @dataclass/_SplitTask boundary" + assert result == 7 + + +def test_find_insertion_point_indented_func_at_file_start(): + # Edge case: the target def has method_indent > 0 but is at line 0 so the + # backward-search loop range is empty. Falls through to decorator walk + # which also exits immediately (j=-1), returning 0. + source = " def inner():\n pass\n" + # "def inner" found at i=0 (indent=4). range(-1, -1, -1) is empty → loop + # body never runs → fall through to decorator walk → j = -1 → return 0. + assert _find_insertion_point(source, "inner") == 0 + + +def test_find_insertion_point_async_def(): + # Regression: helpers extracted from async functions were inserted at line 0 + # (before imports) because the pattern only matched 'def', not 'async def'. + source = ( + "import pytest\n" # 0 + "\n" # 1 + "async def target(client):\n" # 2 + " pass\n" # 3 + ) + assert _find_insertion_point(source, "target") == 2 + + +def test_find_insertion_point_async_def_with_decorator(): + # async def with a preceding decorator: helper should land before the decorator. + source = ( + "import pytest\n" # 0 + "\n" # 1 + "@pytest.mark.asyncio\n" # 2 + "async def target(client):\n" # 3 + " pass\n" # 4 + ) + assert _find_insertion_point(source, "target") == 2 + + +def test_find_insertion_point_skips_over_decorators(): + # Helper must be inserted before the decorator block, not between the + # decorators and the def they decorate. + source = ( + "import os\n" # 0 + "\n" # 1 + "@decorator\n" # 2 + "def target():\n" # 3 + " pass\n" # 4 + ) + # Without the fix this would return 3 (the def line); with the fix it + # should return 2 (the @decorator line). + assert _find_insertion_point(source, "target") == 2 + + +def test_find_insertion_point_skips_over_multiline_decorator(): + # Multi-line decorator: @patch(\n "..."\n) above the def. + source = ( + "import os\n" # 0 + "\n" # 1 + "@patch(\n" # 2 + ' "some.module"\n' # 3 + ")\n" # 4 + "def target():\n" # 5 + " pass\n" # 6 + ) + # Should return 2 (before the @patch line), not 5 (the def line). + assert _find_insertion_point(source, "target") == 2 + + +def test_skip_class_docstring_no_docstring(): + source = "class Foo:\n def method(self):\n pass\n" + lines = source.splitlines() + # after_class_line=1 (line " def method..."), no docstring → unchanged + assert _skip_class_docstring(lines, 1) == 1 + + +def test_skip_class_docstring_triple_double_quote_single_line(): + source = 'class Foo:\n """A docstring."""\n def method(self):\n pass\n' + lines = source.splitlines() + # after_class_line=1 is the docstring line; should return 2 + assert _skip_class_docstring(lines, 1) == 2 + + +def test_skip_class_docstring_triple_single_quote_single_line(): + source = "class Foo:\n '''A docstring.'''\n def method(self):\n pass\n" + lines = source.splitlines() + assert _skip_class_docstring(lines, 1) == 2 + + +def test_skip_class_docstring_triple_quote_multiline(): + source = ( + "class Foo:\n" + ' """First line.\n' + " Second line.\n" + ' """\n' + " def method(self):\n" + " pass\n" + ) + lines = source.splitlines() + # Closing """ is on line 3 (0-based); should return 4 + assert _skip_class_docstring(lines, 1) == 4 + + +def test_skip_class_docstring_with_leading_blank_line(): + source = 'class Foo:\n\n """Docstring."""\n def method(self):\n pass\n' + lines = source.splitlines() + # Line 1 is blank, line 2 is the docstring; should return 3 + assert _skip_class_docstring(lines, 1) == 3 + + +def test_skip_class_docstring_empty_class(): + source = "class Foo:\n pass\n" + lines = source.splitlines() + assert _skip_class_docstring(lines, 1) == 1 + + +def test_skip_class_docstring_only_blank_lines(): + # after_class_line points past end of file after skipping blanks + lines = ["class Foo:", " "] + assert _skip_class_docstring(lines, 1) == 1 + + +def test_skip_class_docstring_malformed_multiline_no_close(): + # Triple-quoted docstring that never closes (malformed) — returns end-of-lines + lines = ["class Foo:", ' """This never closes', " still going"] + result = _skip_class_docstring(lines, 1) + assert result == 3 # past end of lines, best-effort + + +def test_skip_class_docstring_single_quote(): + # Single-quoted one-liner docstring + lines = ["class Foo:", ' "A brief note."', " def method(self): pass"] + assert _skip_class_docstring(lines, 1) == 2 + + +def test_build_helper_insertion_blank_before_insert_pos(): + # Blank line at index 1 is before insert_pos=2 (before_blanks=1, after_blanks=0). + # insert_at=2 (pure insertion), leading=max(0,2-1)=1 so text starts with "\n". + source = "import os\n\ndef foo():\n pass\n" + lines = source.splitlines(keepends=True) + helper = "def _helper():\n pass\n" + start, end, text = _build_helper_insertion(lines, 2, helper, "module_level") + assert start == 2 + assert end == 2 # pure insertion + assert text.startswith("\n") + assert not text.startswith("\n\n") # only 1 leading blank needed + assert text.endswith("\n\n") + assert "def _helper():" in text + + +def test_build_helper_insertion_no_surrounding_blanks(): + # No blanks to absorb → pure insertion with 2 blank lines each side. + source = "import os\ndef foo():\n pass\n" + lines = source.splitlines(keepends=True) + helper = "def _helper():\n pass\n" + start, end, text = _build_helper_insertion(lines, 1, helper, "module_level") + assert start == 1 + assert end == 1 # pure insertion + assert text.startswith("\n\n") + assert text.endswith("\n\n") + + +def test_build_helper_insertion_staticmethod_uses_one_blank(): + # Staticmethod placement: 1 blank line before and after. + source = "class Foo:\n def bar(self):\n pass\n" + lines = source.splitlines(keepends=True) + helper = " @staticmethod\n def _h():\n pass\n" + start, end, text = _build_helper_insertion(lines, 1, helper, "staticmethod:Foo") + assert start == 1 + assert end == 1 # no blanks to absorb + assert text.startswith("\n") + assert not text.startswith("\n\n") + assert text.endswith("\n\n") # clean + 1 trailing blank = \n + \n + + +def test_build_helper_insertion_blank_at_insert_pos(): + # insert_pos=1 is the blank line itself (after_blanks=1, before_blanks=0). + # insert_at=1+1=2 (pure insertion after the blank), leading=max(0,2-1)=1. + source = "import os\n\ndef foo():\n pass\n" + lines = source.splitlines(keepends=True) + helper = "def _helper():\n pass\n" + start, end, text = _build_helper_insertion(lines, 1, helper, "module_level") + assert start == 2 + assert end == 2 # pure insertion + assert text.startswith("\n") + assert not text.startswith("\n\n") # only 1 leading blank needed + assert text.endswith("\n\n") + + +def test_build_helper_insertion_strips_extra_newlines_from_helper(): + # If the LLM returns a helper with leading/trailing blank lines, they are stripped. + source = "import os\ndef foo():\n pass\n" + lines = source.splitlines(keepends=True) + helper = "\n\ndef _helper():\n pass\n\n\n" + start, end, text = _build_helper_insertion(lines, 1, helper, "module_level") + assert text.startswith("\n\n") + assert text.endswith("\n\n") + assert "\n\n\n\ndef _helper" not in text # no extra leading blanks inside text + + +def test_build_helper_insertion_two_at_same_scope(): + # Two helpers inserted before the same def via _apply_edits: both must appear. + source = "import os\n\n\ndef foo():\n pass\n" + lines = source.splitlines(keepends=True) + edits = [ + _build_helper_insertion(lines, 3, "def _h1():\n pass\n", "module_level"), + _build_helper_insertion(lines, 3, "def _h2():\n pass\n", "module_level"), + ] + result = _apply_edits(source, edits) + assert "def _h1():" in result + assert "def _h2():" in result + assert "def foo():" in result + + +def test_successful_extraction_has_two_blank_lines(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + # Each function has 4 statements. The first statement is STRUCTURALLY different + # between them (if-block vs assignment), so the normalizer produces different + # fingerprints for the full 4-stmt body. Only the trailing 3-stmt block + # (compute/transform/finalize) is duplicated, so the proxy-wrapper guard + # does not trigger (3 stmts < body_stmt_count 4). + source = textwrap.dedent( + """\ + import os + + def foo(): + if debug: + validate(data) + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(): + result = validate(data) + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor([(12, 14)], source=source) + + assert de._new_source is not None + # Exactly 2 blank lines before and after the inserted helper. + assert "\n\n\ndef _helper" in de._new_source + assert "\n\n\n\ndef _helper" not in de._new_source + assert "def _helper(data):\n pass\n\n\ndef foo" in de._new_source + + +def test_helper_placed_before_class_not_inside(monkeypatch): + """Helper extracted from class methods must be placed BEFORE the class. + + When duplicate blocks live inside class methods, inserting a module-level + helper before the method (inside the class body) ends the class definition + prematurely and turns the remaining methods into nested functions. The fix + in _find_insertion_point walks backwards to the enclosing class. + """ + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + import os + + class MyClass: + def method_a(self, x): + if self.debug: + pass + a = compute(x) + b = transform(a) + c = finalize(b) + return c + + def method_b(self, x): + result = None + a = compute(x) + b = transform(a) + c = finalize(b) + return c + """ + ) + helper = "def _do_work(x):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_do_work", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " return _do_work(x)\n", + " return _do_work(x)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor([(1, 100)], source=source) + + assert de._new_source is not None + compile(de._new_source, "", "exec") + # Helper must appear BEFORE the class definition, not inside it. + helper_pos = de._new_source.find("def _do_work") + class_pos = de._new_source.find("class MyClass") + assert ( + helper_pos < class_pos + ), "helper was placed after/inside class instead of before it" + # The class structure must be intact: MyClass still has both methods. + import ast as _ast + + tree = _ast.parse(de._new_source) + classes = [n for n in _ast.walk(tree) if isinstance(n, _ast.ClassDef)] + assert len(classes) == 1 + assert classes[0].name == "MyClass" + methods = [n.name for n in classes[0].body if isinstance(n, _ast.FunctionDef)] + assert "method_a" in methods + assert "method_b" in methods diff --git a/tests/duplicate_extractor/test_extraction_validation.py b/tests/duplicate_extractor/test_extraction_validation.py new file mode 100644 index 0000000..af952a5 --- /dev/null +++ b/tests/duplicate_extractor/test_extraction_validation.py @@ -0,0 +1,288 @@ +from crispen.refactors.duplicate_extractor import ( + _collect_attribute_names, + _collect_called_attr_names, + _collect_called_names, + _has_call_to, + _has_funcdef, + _has_mutable_literal_is_check, + _is_pure_literal, + _verify_extraction, +) + + +def test_verify_extraction_valid(): + helper = "def helper(x):\n return x + 1\n" + replacements = ["result = helper(a)\n"] + assert _verify_extraction(helper, replacements) is True + + +def test_verify_extraction_invalid_helper(): + helper = "def helper(x:\n pass\n" # unclosed paren → syntax error after dedent + replacements = ["result = helper(a)\n"] + assert _verify_extraction(helper, replacements) is False + + +def test_verify_extraction_invalid_replacement(): + helper = "def helper(x):\n return x\n" + # Dedented replacement still has a syntax error + replacements = ["result = helper(a\n"] # unclosed paren + assert _verify_extraction(helper, replacements) is False + + +def test_verify_extraction_no_helper_source(): + # Exercises the helper_source is None branch (skips helper compile check). + assert _verify_extraction(None, ["result = f()\n"]) is True + + +def test_verify_extraction_fails_on_param_overwrite(): + # Helper where the parameter is immediately overwritten before being read. + helper = "def setup(mock_obj):\n mock_obj = object()\n return mock_obj\n" + assert _verify_extraction(helper, ["x = setup(y)\n"]) is False + + +def test_verify_extraction_allows_return_in_replacement(): + # Replacements inside function bodies legally contain 'return'; the dummy- + # function wrapper must allow this without triggering a false rejection. + helper = "def helper(x):\n return x\n" + replacements = [" return helper(a)\n"] + assert _verify_extraction(helper, replacements) is True + + +def test_verify_extraction_allows_multiline_return_replacement(): + # Multi-line replacement ending with a return statement. + helper = "def helper(source):\n return helper(source)\n" + replacements = [ + " tree = helper(source)\n if tree is None:\n return set()\n" + ] + assert _verify_extraction(helper, replacements) is True + + +def test_verify_extraction_allows_continue_in_replacement(): + # 'continue' is valid inside a loop body; the dummy wrapper now includes a + # for loop so this is not rejected as a SyntaxError. + helper = "def helper():\n pass\n" + replacements = [" if done:\n continue\n"] + assert _verify_extraction(helper, replacements) is True + + +def test_verify_extraction_allows_break_in_replacement(): + # Same as above but for 'break'. + helper = "def helper():\n pass\n" + replacements = [" if done:\n break\n"] + assert _verify_extraction(helper, replacements) is True + + +def test_verify_extraction_allows_await_in_replacement(): + # Replacements inside async functions legally contain 'await'; the async + # dummy-function wrapper must allow this without triggering a false rejection. + helper = "async def helper(x):\n return await x\n" + replacements = [" result = await helper(coro)\n"] + assert _verify_extraction(helper, replacements) is True + + +def test_verify_extraction_allows_async_helper(): + # async def helpers are valid Python and must compile successfully. + helper = "async def helper(client, x):\n return await client.get(x)\n" + replacements = [" val = await helper(client, url)\n"] + assert _verify_extraction(helper, replacements) is True + + +def test_verify_extraction_rejects_invalid_await_replacement(): + # Replacement with `await` that also has a real syntax error must still fail. + helper = "async def helper(x):\n return await x\n" + replacements = [" result = await helper(coro\n"] # unclosed paren + assert _verify_extraction(helper, replacements) is False + + +def test_has_mutable_literal_is_check_set_constructor(): + assert _has_mutable_literal_is_check("if x is set(): pass") is True + + +def test_has_mutable_literal_is_check_list_constructor(): + assert _has_mutable_literal_is_check("if x is list(): pass") is True + + +def test_has_mutable_literal_is_check_dict_constructor(): + assert _has_mutable_literal_is_check("if x is dict(): pass") is True + + +def test_has_mutable_literal_is_check_list_literal(): + assert _has_mutable_literal_is_check("if x is []: pass") is True + + +def test_has_mutable_literal_is_check_dict_literal(): + assert _has_mutable_literal_is_check("if x is {}: pass") is True + + +def test_has_mutable_literal_is_check_isnot(): + assert _has_mutable_literal_is_check("if x is not set(): pass") is True + + +def test_has_mutable_literal_is_check_none_is_fine(): + assert _has_mutable_literal_is_check("if x is None: pass") is False + + +def test_has_mutable_literal_is_check_isinstance_is_fine(): + assert _has_mutable_literal_is_check("if isinstance(x, set): pass") is False + + +def test_has_mutable_literal_is_check_equality_is_fine(): + # == comparison with set() is valid; only identity (`is`) is wrong + assert _has_mutable_literal_is_check("if x == set(): pass") is False + + +def test_has_mutable_literal_is_check_syntax_error(): + assert _has_mutable_literal_is_check("def f(x:") is False + + +def test_verify_extraction_rejects_mutable_is_in_helper(): + helper = "def h(x):\n if x is set(): return True\n return False\n" + assert _verify_extraction(helper, ["h(a)\n"]) is False + + +def test_verify_extraction_rejects_mutable_is_in_replacement(): + helper = "def h(x):\n return x\n" + assert _verify_extraction(helper, ["if r is set(): pass\n"]) is False + + +def test_verify_extraction_rejects_indented_mutable_is_in_replacement(): + # Indented replacements (function-body code) are wrapped before checking, + # so `is set()` is caught even when ast.parse would fail on raw indented text. + helper = "def h(x):\n return x\n" + assert _verify_extraction(helper, [" if r is set(): pass\n"]) is False + + +def test_collect_attribute_names_basic(): + assert _collect_attribute_names("x.foo()\ny.bar") == {"foo", "bar"} + + +def test_collect_attribute_names_nested(): + assert "baz" in _collect_attribute_names("a.b.baz()") + + +def test_collect_attribute_names_syntax_error(): + assert _collect_attribute_names("def f(x:") == set() + + +def test_collect_attribute_names_no_attrs(): + assert _collect_attribute_names("x = 1 + 2") == set() + + +def test_collect_called_attr_names_method_call(): + # obj.foo() → "foo" is a called attribute + assert _collect_called_attr_names("obj.foo()") == {"foo"} + + +def test_collect_called_attr_names_ignores_plain_access(): + # obj.bar (not called) → not included + assert "bar" not in _collect_called_attr_names("x = obj.bar") + + +def test_collect_called_attr_names_ignores_type_annotation(): + # ast.AST used as a type annotation is NOT a method call → not flagged + assert "AST" not in _collect_called_attr_names( + "def f(x) -> Optional[ast.AST]: pass" + ) + + +def test_collect_called_attr_names_syntax_error(): + assert _collect_called_attr_names("def f(x:") == set() + + +def test_collect_called_attr_names_no_calls(): + assert _collect_called_attr_names("x = 1 + 2") == set() + + +def test_has_call_to_direct_call(): + assert _has_call_to("foo", "foo()\n") is True + + +def test_has_call_to_attribute_call(): + assert _has_call_to("foo", "obj.foo()\n") is True + + +def test_has_call_to_missing(): + assert _has_call_to("foo", "bar()\n") is False + + +def test_has_call_to_syntax_error(): + assert _has_call_to("foo", "def f(x:") is False + + +def test_has_funcdef_present(): + assert _has_funcdef("_helper", "def _helper(x):\n pass\n") is True + + +def test_has_funcdef_async(): + assert _has_funcdef("_helper", "async def _helper(x):\n pass\n") is True + + +def test_has_funcdef_missing(): + assert _has_funcdef("_helper", "def other(x):\n pass\n") is False + + +def test_has_funcdef_syntax_error(): + assert _has_funcdef("_helper", "def f(x:") is False + + +def test_is_pure_literal_constant(): + import ast + + assert _is_pure_literal(ast.parse("0", mode="eval").body) + assert _is_pure_literal(ast.parse('"s"', mode="eval").body) + assert _is_pure_literal(ast.parse("None", mode="eval").body) + assert _is_pure_literal(ast.parse("True", mode="eval").body) + + +def test_is_pure_literal_containers(): + import ast + + assert _is_pure_literal(ast.parse("[]", mode="eval").body) + assert _is_pure_literal(ast.parse("(1, 2)", mode="eval").body) + assert _is_pure_literal(ast.parse("{1: 2}", mode="eval").body) + assert _is_pure_literal(ast.parse("{1, 2}", mode="eval").body) + + +def test_is_pure_literal_call_is_false(): + import ast + + assert not _is_pure_literal(ast.parse("func()", mode="eval").body) + + +def test_is_pure_literal_name_is_false(): + import ast + + assert not _is_pure_literal(ast.parse("x", mode="eval").body) + + +def test_is_pure_literal_nested_call_is_false(): + import ast + + assert not _is_pure_literal(ast.parse("[func()]", mode="eval").body) + + +def test_collect_called_names_direct(): + names = _collect_called_names("foo()\n") + assert "foo" in names + + +def test_collect_called_names_method(): + names = _collect_called_names("obj.bar()\n") + assert "bar" in names + + +def test_collect_called_names_empty(): + names = _collect_called_names("x = 1\n") + assert names == set() + + +def test_collect_called_names_syntax_error(): + names = _collect_called_names("def f(: pass") + assert names == set() + + +def test_collect_called_names_other_callable(): + # func is a subscript (neither Name nor Attribute): funcs[0]() + # Covers the elif-False branch in _collect_called_names. + names = _collect_called_names("funcs[0]()\n") + assert "funcs" not in names # subscript call adds nothing diff --git a/tests/duplicate_extractor/test_extractor_combined.py b/tests/duplicate_extractor/test_extractor_combined.py new file mode 100644 index 0000000..c482933 --- /dev/null +++ b/tests/duplicate_extractor/test_extractor_combined.py @@ -0,0 +1,299 @@ +from unittest.mock import MagicMock, patch +import textwrap +from crispen.refactors.duplicate_extractor import DuplicateExtractor +from .test_extractor_core import ( + _DUP_RANGES, + _DUP_SOURCE, + _make_extract_response, + _make_verify_response, + _make_veto_response, +) + + +def _make_no_call_extractor(monkeypatch, verbose=True): + """Helper: LLM returns call replacements that don't call the helper function.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(data):\n pass\n", + # Call replacements don't reference _helper at all. + "call_site_replacements": [ + " pass\n", + " pass\n", + ], + } + ), + ] + return DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + verbose=verbose, + extraction_retries=0, + llm_verify_retries=0, + ) + + +def test_no_call_check_skips_group_verbose(monkeypatch, capsys): + de = _make_no_call_extractor(monkeypatch, verbose=True) + assert de._new_source is None + assert "not called in candidate output" in capsys.readouterr().err + + +def test_no_call_check_skips_group_verbose_false(monkeypatch): + de = _make_no_call_extractor(monkeypatch, verbose=False) + assert de._new_source is None + + +def _make_uncalled_in_combined_extractor(monkeypatch, verbose=True): + """Simulate: per-group call check passes, but combined output lacks the call. + + Achieved by patching _has_call_to: returns True for the per-group check + (first call) and False for the final combined check (second call). + """ + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic") as mock_anthropic, + patch( + "crispen.refactors.duplicate_extractor.extractor._has_call_to", + side_effect=[True, False], + ), + ): + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(data):\n pass\n", + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + return DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, verbose=verbose) + + +def test_uncalled_in_combined_drops_group_verbose(monkeypatch, capsys): + de = _make_uncalled_in_combined_extractor(monkeypatch, verbose=True) + assert de._new_source is None + assert "DROPPED" in capsys.readouterr().err + + +def test_uncalled_in_combined_drops_group_verbose_false(monkeypatch): + de = _make_uncalled_in_combined_extractor(monkeypatch, verbose=False) + assert de._new_source is None + + +def _make_undefined_in_combined_extractor(monkeypatch, verbose=True): + """Simulate: per-group checks all pass but helper definition is absent from + the combined output (insertion edit blocked by overlap detector). + + Achieved by patching _has_funcdef: returns True for the per-group pyflakes + check (not called there directly, but we patch the combined check) and + False for the final combined check. + """ + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic") as mock_anthropic, + patch( + "crispen.refactors.duplicate_extractor.extractor._has_funcdef", + side_effect=[False], + ), + ): + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(data):\n pass\n", + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + return DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, verbose=verbose) + + +def test_undefined_helper_in_combined_drops_group_verbose(monkeypatch, capsys): + de = _make_undefined_in_combined_extractor(monkeypatch, verbose=True) + assert de._new_source is None + assert "not defined in combined output" in capsys.readouterr().err + + +def test_undefined_helper_in_combined_drops_group_verbose_false(monkeypatch): + de = _make_undefined_in_combined_extractor(monkeypatch, verbose=False) + assert de._new_source is None + + +# Source with two structurally distinct duplicate pairs so _find_duplicate_groups +# returns two separate groups. The groups differ in argument count so that +# _ASTNormalizer produces different fingerprints for each group: +# group 1 (foo/bar): 3-stmt bodies using 2-argument calls → fingerprint A +# group 2 (baz/qux): 3-stmt bodies using 3-argument calls → fingerprint B +_TWO_PAIR_SOURCE = textwrap.dedent( + """\ + import os + + def foo(): + if debug: + pass + x = compute(data, config) + y = transform(x, scale) + z = finalize(y, mode) + + def bar(): + result = None + x = compute(data, config) + y = transform(x, scale) + z = finalize(y, mode) + + def baz(): + if debug: + pass + a = process(item, key, idx) + b = convert(a, fmt, enc) + c = export(b, path, opts) + + def qux(): + result = None + a = process(item, key, idx) + b = convert(a, fmt, enc) + c = export(b, path, opts) + """ +) +_TWO_PAIR_RANGES = [(4, 30)] # overlaps all duplicate sequences + + +def test_undefined_helper_in_combined_two_groups_one_dropped(monkeypatch): + """Two groups: first group's helper missing from combined, second kept. + + _has_funcdef returns [False, True]: first group dropped, second kept. + This exercises the all_edits.extend(g_edits) loop after the drop (line 2304). + """ + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic") as mock_anthropic, + patch( + "crispen.refactors.duplicate_extractor.extractor._has_funcdef", + side_effect=[False, True], + ), + ): + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper1", + "placement": "module_level", + "helper_source": "def _helper1():\n pass\n", + "call_site_replacements": [ + " _helper1()\n", + " _helper1()\n", + ], + } + ), + _make_verify_response(True, []), + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper2", + "placement": "module_level", + "helper_source": "def _helper2():\n pass\n", + "call_site_replacements": [ + " _helper2()\n", + " _helper2()\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor( + _TWO_PAIR_RANGES, source=_TWO_PAIR_SOURCE, verbose=False + ) + # First group dropped (undefined), second group kept → new source written + assert de._new_source is not None + + +def _make_two_group_drop_extractor(monkeypatch, verbose=True): + """Two extraction groups; the combined check drops one, exercising line 1533. + + _has_call_to is patched with side_effect=[True, True, True, False]: + - calls 1-2: per-group checks for each group → both pass + - call 3: combined check for first group → kept + - call 4: combined check for second group → dropped + After the drop, extraction_groups still has one entry, so the inner + ``for _, g_edits, _ in extraction_groups`` loop runs once (line 1533). + """ + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic") as mock_anthropic, + patch( + "crispen.refactors.duplicate_extractor.extractor._has_call_to", + side_effect=[True, True, True, False], + ), + ): + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + # Six LLM calls: veto+extract+verify for each of the two groups. + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper1", + "placement": "module_level", + "helper_source": "def _helper1():\n pass\n", + "call_site_replacements": [ + " _helper1()\n", + " _helper1()\n", + ], + } + ), + _make_verify_response(True, []), + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper2", + "placement": "module_level", + "helper_source": "def _helper2():\n pass\n", + "call_site_replacements": [ + " _helper2()\n", + " _helper2()\n", + ], + } + ), + _make_verify_response(True, []), + ] + return DuplicateExtractor( + _TWO_PAIR_RANGES, source=_TWO_PAIR_SOURCE, verbose=verbose + ) + + +def test_two_groups_one_dropped_combined_check(monkeypatch, capsys): + """One of two groups is dropped by the combined call check; the other is kept.""" + de = _make_two_group_drop_extractor(monkeypatch, verbose=True) + assert de._new_source is not None + assert "DROPPED" in capsys.readouterr().err diff --git a/tests/duplicate_extractor/test_extractor_core.py b/tests/duplicate_extractor/test_extractor_core.py new file mode 100644 index 0000000..907ba79 --- /dev/null +++ b/tests/duplicate_extractor/test_extractor_core.py @@ -0,0 +1,540 @@ +from unittest.mock import MagicMock, patch +import textwrap +from crispen.errors import CrispenAPIError +from crispen.refactors.duplicate_extractor import ( + DuplicateExtractor, + _SeqInfo, + _find_escaping_vars, +) +import pytest + + +def _make_esc_seq(start: int, end: int) -> _SeqInfo: + """Create a _SeqInfo for escaping-vars tests.""" + return _SeqInfo( + stmts=[], + start_line=start, + end_line=end, + scope="foo", + source="", + fingerprint="", + ) + + +def test_find_escaping_vars_no_assignments(): + # Block has no assignments → skip (branch A), returns empty set. + source_lines = [ + "def foo():\n", + " compute()\n", + " transform()\n", + " use_result()\n", + ] + seq = _make_esc_seq(2, 3) + assert _find_escaping_vars([seq], source_lines) == set() + + +def test_find_escaping_vars_nothing_after_block(): + # Block is the last thing in scope → after_lines empty (branch D), returns set(). + source_lines = [ + "def foo():\n", + " x = compute()\n", + " y = transform(x)\n", + " z = finalize(y)\n", + ] + seq = _make_esc_seq(2, 4) + assert _find_escaping_vars([seq], source_lines) == set() + + +def test_find_escaping_vars_escapes(): + # Block assigns z; z is used after the block → {"z"}. + # Also covers: blank line (branch B) and lower-indent stop (branch C). + source_lines = [ + "def foo():\n", + " x = compute()\n", + " y = transform(x)\n", + " z = finalize(y)\n", # block ends line 4 + "\n", # blank → branch B + " assert z == 42\n", # same indent, uses z + "\n", + "def bar():\n", # indent 0 < 4 → branch C (stop) + " pass\n", + ] + seq = _make_esc_seq(2, 4) + assert _find_escaping_vars([seq], source_lines) == {"z"} + + +def test_find_escaping_vars_no_escape(): + # Block assigns x/y/z; none referenced after the block → set(). + source_lines = [ + "def foo():\n", + " x = compute()\n", + " y = transform(x)\n", + " z = finalize(y)\n", + " print('done')\n", # uses 'print', not x/y/z + ] + seq = _make_esc_seq(2, 4) + assert _find_escaping_vars([seq], source_lines) == set() + + +def test_find_escaping_vars_syntax_error_after(): + # After source is invalid Python → SyntaxError branch: continue, returns set(). + source_lines = [ + "def foo():\n", + " x = compute()\n", + " y = transform(x)\n", + " z = finalize(y)\n", + " def bar(x\n", # unclosed paren at same indent + ] + seq = _make_esc_seq(2, 4) + assert _find_escaping_vars([seq], source_lines) == set() + + +def test_find_escaping_vars_module_level_stops_at_def(): + # Module-level block (indent 0): a non-def/class line is included, + # then a def line stops the scan (break via re.match). + source_lines = [ + "x = compute()\n", + "y = transform(x)\n", + "z = finalize(y)\n", # block ends line 3 + "CONSTANT = 42\n", # module-level non-def → appended (False branch of re.match) + "def foo(z):\n", # module-level def → stop + " return z\n", + ] + seq = _make_esc_seq(1, 3) + # CONSTANT is in after_lines; not in assigned → set(). + # z inside def foo(z) is not scanned (stopped before that def). + assert _find_escaping_vars([seq], source_lines) == set() + + +def _make_verify_response(is_correct: bool, issues: list) -> MagicMock: + block = MagicMock() + block.type = "tool_use" + block.name = "verify_extraction" + block.input = {"is_correct": is_correct, "issues": issues} + resp = MagicMock() + resp.content = [block] + return resp + + +def _make_extract_response(data: dict) -> MagicMock: + block = MagicMock() + block.type = "tool_use" + block.name = "extract_helper" + block.input = data + resp = MagicMock() + resp.content = [block] + return resp + + +def _make_veto_response(is_valid: bool, reason: str = "test") -> MagicMock: + block = MagicMock() + block.type = "tool_use" + block.name = "evaluate_duplicate" + block.input = {"is_valid_duplicate": is_valid, "reason": reason} + resp = MagicMock() + resp.content = [block] + return resp + + +def test_no_source_no_analysis(): + de = DuplicateExtractor([(1, 5)]) + assert de._new_source is None + assert de.get_rewritten_source() is None + + +def test_no_duplicates_no_llm_calls(monkeypatch): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + source = textwrap.dedent( + """\ + def foo(): + x = a + b + y = x * 2 + + def bar(): + if condition: + result = value + else: + result = other + """ + ) + # Structurally different blocks → no duplicate group → no API calls needed + de = DuplicateExtractor([(6, 9)], source=source) + assert de._new_source is None + + +_DUP_SOURCE = textwrap.dedent( + """\ + def foo(): + if debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ +) +_DUP_RANGES = [(10, 12)] # overlaps bar's duplicate block (x/y/z lines) + +# Source where foo's duplicate block assigns z, and foo uses z after the block. +# _has_escaping_vars should detect this and skip the extraction. +_ESC_SOURCE = textwrap.dedent( + """\ + def foo(): + x = compute(data) + y = transform(x) + z = finalize(y) + assert z == expected + + def bar(): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ +) +_ESC_RANGES = [(9, 11)] # overlaps bar's duplicate block (x/y/z lines) + + +def test_missing_api_key_raises(monkeypatch): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + with pytest.raises(CrispenAPIError, match="ANTHROPIC_API_KEY"): + DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) + + +def test_api_error_in_veto_raises(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = Exception("rate limit") + + with pytest.raises(CrispenAPIError): + DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) + + +def test_api_error_in_extract_raises(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + # First call (veto) succeeds, second call (extract) fails + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + Exception("rate limit"), + ] + + with pytest.raises(CrispenAPIError): + DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) + + +def test_parse_error_in_analyze(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic.Anthropic"): + # Invalid Python: _analyze should return silently + de = DuplicateExtractor([(1, 1)], source="def f(: pass") + assert de._new_source is None + + +def test_veto_rejects_no_changes(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.return_value = _make_veto_response(False) + + de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) + + assert de._new_source is None + assert de.changes_made == [] + + +def test_wrong_replacement_count_skipped(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "helper", + "placement": "module_level", + "helper_source": "def helper():\n pass\n", + "call_site_replacements": ["helper()\n"], # should be 2 + } + ), + ] + + de = DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + extraction_retries=0, + llm_verify_retries=0, + ) + + assert de._new_source is None + + +def test_wrong_replacement_count_skipped_verbose_false(monkeypatch): + # verbose=False covers the False branch of the new if-self.verbose guard. + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "helper", + "placement": "module_level", + "helper_source": "def helper():\n pass\n", + "call_site_replacements": ["helper()\n"], # should be 2 + } + ), + ] + + de = DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + verbose=False, + extraction_retries=0, + llm_verify_retries=0, + ) + + assert de._new_source is None + + +def test_escaping_vars_passed_to_extract(monkeypatch): + # foo's block assigns z; foo uses z after the block. + # _find_escaping_vars returns {"z"}, which is passed to _llm_extract. + # The extraction prompt must contain the note instructing the LLM to return z. + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + helper_src = ( + "def _helper(data):\n" + " x = compute(data)\n" + " y = transform(x)\n" + " z = finalize(y)\n" + " return z\n" + ) + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper_src, + "call_site_replacements": [ + " z = _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor(_ESC_RANGES, source=_ESC_SOURCE) + + # The extraction prompt must include the escaping-variable note. + extract_call = mock_client.messages.create.call_args_list[1] + extract_prompt = extract_call.kwargs["messages"][0]["content"] + assert "immediately follows the block" in extract_prompt + assert de._new_source is not None + + +def _make_invalid_assembled_extractor(monkeypatch, verbose=True): + """Helper: DuplicateExtractor where _apply_edits returns invalid Python.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic") as mock_anthropic, + patch( + "crispen.refactors.duplicate_extractor.extractor._apply_edits", + return_value="def f(:\n pass\n", # invalid Python + ), + ): + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(x):\n pass\n", + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + ] + return DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + verbose=verbose, + extraction_retries=0, + llm_verify_retries=0, + ) + + +def test_invalid_assembled_source_skipped(monkeypatch): + # Individual components pass _verify_extraction but the per-group assembled + # edit is invalid Python — the group is skipped without poisoning others. + de = _make_invalid_assembled_extractor(monkeypatch) + assert de._new_source is None + assert de.changes_made == [] + + +def test_invalid_assembled_source_skipped_verbose_false(monkeypatch): + # verbose=False: per-group compile-failure log suppressed (covers False branch). + de = _make_invalid_assembled_extractor(monkeypatch, verbose=False) + assert de._new_source is None + + +def test_successful_extraction_module_level(monkeypatch, tmp_path): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + import os + + def foo(): + if debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + + de = DuplicateExtractor([(12, 14)], source=source) + + assert de._new_source is not None + assert "_helper" in de._new_source + assert len(de.changes_made) == 1 + assert "'_helper'" in de.changes_made[0] + assert de.get_rewritten_source() == de._new_source + + +def test_duplicate_extractor_helper_docstrings_false_strips_docstring( + monkeypatch, capsys +): + """When helper_docstrings=False, the LLM-returned docstring is stripped.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_shared", + "placement": "module_level", + "helper_source": ( + "def _shared(data):\n" + ' """LLM added a docstring."""\n' + " pass\n" + ), + "call_site_replacements": [ + " _shared(data)\n", + " _shared(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor( + _DUP_RANGES, source=_DUP_SOURCE, verbose=False, helper_docstrings=False + ) + + assert de._new_source is not None + assert '"""LLM added a docstring."""' not in de._new_source + + +def test_duplicate_extractor_helper_docstrings_true_keeps_docstring( + monkeypatch, capsys +): + """When helper_docstrings=True, the LLM-returned docstring is preserved.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_shared", + "placement": "module_level", + "helper_source": ( + "def _shared(data):\n" + ' """Keep this docstring."""\n' + " pass\n" + ), + "call_site_replacements": [ + " _shared(data)\n", + " _shared(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor( + _DUP_RANGES, source=_DUP_SOURCE, verbose=False, helper_docstrings=True + ) + + assert de._new_source is not None + assert '"""Keep this docstring."""' in de._new_source + + +def test_duplicate_extractor_custom_model_used(monkeypatch): + """Custom model string is passed to the Anthropic API.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.return_value = _make_veto_response(False, "no") + DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, model="claude-opus-4-6") + # Verify the custom model was passed + call_kwargs = mock_client.messages.create.call_args_list[0][1] + assert call_kwargs["model"] == "claude-opus-4-6" diff --git a/tests/duplicate_extractor/test_extractor_guards.py b/tests/duplicate_extractor/test_extractor_guards.py new file mode 100644 index 0000000..89e4fac --- /dev/null +++ b/tests/duplicate_extractor/test_extractor_guards.py @@ -0,0 +1,528 @@ +from unittest.mock import MagicMock, patch +import textwrap +from crispen.refactors.duplicate_extractor import ( + DuplicateExtractor, + _FunctionInfo, + _SeqInfo, + _has_param_overwritten_before_read, + _missing_free_vars, + _pyflakes_new_undefined_names, + _would_create_proxy_wrappers, +) +from .test_extractor_core import ( + _DUP_RANGES, + _DUP_SOURCE, + _make_extract_response, + _make_verify_response, + _make_veto_response, +) + + +def test_has_param_overwritten_before_read_false_when_param_is_read(): + # Parameter is read before (or without) being reassigned — should return False. + helper = "def fn(x):\n return x + 1\n" + assert _has_param_overwritten_before_read(helper) is False + + +def test_has_param_overwritten_before_read_true_when_immediately_overwritten(): + # Parameter is assigned on the first statement without being read — True. + helper = "def setup(client):\n client = object()\n return client\n" + assert _has_param_overwritten_before_read(helper) is True + + +def test_has_param_overwritten_before_read_false_for_conditional_default(): + # The ``if x is None: x = default`` pattern reads before writing — False. + helper = "def fn(x=None):\n if x is None:\n x = []\n return x\n" + assert _has_param_overwritten_before_read(helper) is False + + +def test_has_param_overwritten_before_read_vararg_and_kwarg(): + # Covers the vararg/kwarg branches — neither is overwritten here. + helper = "def fn(*args, **kwargs):\n return args, kwargs\n" + assert _has_param_overwritten_before_read(helper) is False + + +def test_pyflakes_new_undefined_names_returns_empty_when_no_new_issues(): + # Names undefined in both original and candidate → no NEW issues. + original = "def foo():\n return bar()\n" + candidate = "def _h():\n pass\n\ndef foo():\n return bar()\n" + assert _pyflakes_new_undefined_names(original, candidate) == set() + + +def test_pyflakes_new_undefined_names_detects_introduced_name(): + # candidate introduces a reference to an unassigned name not in original. + original = "def foo():\n x = 1\n return x\n" + # candidate removes the assignment, leaving x undefined at the call site + candidate = "def _h():\n x = 1\n\ndef foo():\n _h(x)\n return x\n" + assert "x" in _pyflakes_new_undefined_names(original, candidate) + + +def test_missing_free_vars_catches_missing_name(): + # The exact bug pattern: `new_source` is a local variable read in the + # original block, but the LLM turned it into `transformer.new_source` + # (an attribute access). Neither the call site nor the helper body contain + # a bare `new_source` Name node. + source = ( + "def run(transformer, file_msgs, filepath):\n" + " new_source = get_source()\n" + " current_source = new_source\n" + ) + block_src = " current_source = new_source\n" + call_src = " current_source = _h(transformer, filepath, file_msgs)\n" + helper_src = ( + "def _h(transformer, filepath, file_msgs):\n" + " return transformer.new_source\n" + ) + assert "new_source" in _missing_free_vars(block_src, [call_src], helper_src, source) + + +def test_missing_free_vars_no_missing_when_passed_as_arg(): + # Free var is passed as an argument to the helper → not missing. + source = ( + "def run():\n new_source = get_source()\n current_source = new_source\n" + ) + block_src = " current_source = new_source\n" + call_src = " current_source = _h(new_source)\n" + helper_src = "def _h(new_source):\n return new_source\n" + assert _missing_free_vars(block_src, [call_src], helper_src, source) == set() + + +def test_missing_free_vars_ignores_block_locals(): + # `x` is assigned AND read within the block — it is a local, not a free + # variable. It should not be flagged even if it's absent from the helper. + source = "def run():\n x = 1\n result = x + 1\n" + block_src = " x = 1\n result = x + 1\n" + call_src = " result = _h()\n" + helper_src = "def _h():\n x = 1\n return x + 1\n" + assert _missing_free_vars(block_src, [call_src], helper_src, source) == set() + + +def test_missing_free_vars_ignores_module_level_names(): + # `compute`, `transform`, `finalize` are module-level function names that + # are never assigned anywhere — the helper can reference them directly. + source = ( + "def foo():\n" + " x = compute(data)\n" + " y = transform(x)\n" + " z = finalize(y)\n" + ) + block_src = " x = compute(data)\n y = transform(x)\n z = finalize(y)\n" + call_src = " _helper(data)\n" + helper_src = "def _helper(data):\n pass\n" + assert _missing_free_vars(block_src, [call_src], helper_src, source) == set() + + +def test_missing_free_vars_syntax_error_in_block_returns_empty(): + assert ( + _missing_free_vars("not valid python!!!", ["x = 1\n"], "def f(): pass\n", "") + == set() + ) + + +def test_missing_free_vars_syntax_error_in_replacement_returns_empty(): + source = "def run():\n a = 1\n" + assert ( + _missing_free_vars("x = a\n", ["not valid!!!\n"], "def f(): pass\n", source) + == set() + ) + + +def test_missing_free_vars_syntax_error_in_source_returns_empty(): + assert ( + _missing_free_vars("x = a\n", ["y = a\n"], "def f(a): pass\n", "not valid!!!") + == set() + ) + + +def test_missing_free_vars_empty_block_returns_empty(): + # A block with no reads has no free vars → nothing can be missing. + source = "def run():\n x = 1\n" + block_src = " x = 1\n" + call_src = " _h()\n" + helper_src = "def _h():\n x = 1\n" + assert _missing_free_vars(block_src, [call_src], helper_src, source) == set() + + +def test_missing_free_vars_function_parameter_is_caught(): + # A function parameter that's free in the block must appear in the + # replacement — parameters are local to the function and cannot be + # accessed by a helper without being passed as an argument. + source = "def run(verbose):\n msg = verbose\n" + block_src = " msg = verbose\n" + call_src = " msg = _h()\n" + helper_src = "def _h():\n pass\n" + assert "verbose" in _missing_free_vars(block_src, [call_src], helper_src, source) + + +def _make_pyflakes_check_extractor(monkeypatch, verbose=True): + """Helper: extraction that passes compile() but pyflakes finds a new undefined + name.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic") as mock_anthropic, + patch( + "crispen.refactors.duplicate_extractor.extractor._pyflakes_new_undefined_names", + return_value={"mock_client"}, + ), + ): + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(x):\n pass\n", + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + ] + return DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + verbose=verbose, + extraction_retries=0, + llm_verify_retries=0, + ) + + +def test_pyflakes_check_skips_group_verbose(monkeypatch, capsys): + # Pyflakes finds a new undefined name → group is skipped (verbose path). + de = _make_pyflakes_check_extractor(monkeypatch, verbose=True) + assert de._new_source is None + assert ( + "undefined name(s) introduced by edit: mock_client" in capsys.readouterr().err + ) + + +def test_pyflakes_check_skips_group_verbose_false(monkeypatch): + # verbose=False: pyflakes failure is silent. + de = _make_pyflakes_check_extractor(monkeypatch, verbose=False) + assert de._new_source is None + + +def _make_missing_free_vars_extractor(monkeypatch, verbose=True): + """Helper: extraction that passes all earlier guards but _missing_free_vars + detects a free variable absent from the replacement.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic") as mock_anthropic, + patch( + "crispen.refactors.duplicate_extractor.extractor._missing_free_vars", + return_value={"new_source"}, + ), + ): + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(x):\n pass\n", + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + ] + return DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + verbose=verbose, + extraction_retries=0, + llm_verify_retries=0, + ) + + +def test_missing_free_vars_check_skips_group_verbose(monkeypatch, capsys): + # _missing_free_vars returns a non-empty set → group is rejected (verbose). + de = _make_missing_free_vars_extractor(monkeypatch, verbose=True) + assert de._new_source is None + assert ( + "free variable(s) from original block missing in replacement: new_source" + in capsys.readouterr().err + ) + + +def test_missing_free_vars_check_skips_group_verbose_false(monkeypatch): + # verbose=False: _missing_free_vars failure is silent. + de = _make_missing_free_vars_extractor(monkeypatch, verbose=False) + assert de._new_source is None + + +def test_verify_fails_skipped(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "helper", + "placement": "module_level", + "helper_source": "def helper(x:\n pass\n", # unclosed paren + "call_site_replacements": [ + "helper(data)\n", + "helper(data)\n", + ], + } + ), + ] + + de = DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + extraction_retries=0, + llm_verify_retries=0, + ) + + assert de._new_source is None + + +def test_verify_fails_skipped_verbose_false(monkeypatch): + # verbose=False covers the False branch of the new if-self.verbose guard. + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "helper", + "placement": "module_level", + "helper_source": "def helper(x:\n pass\n", # unclosed paren + "call_site_replacements": [ + "helper(data)\n", + "helper(data)\n", + ], + } + ), + ] + + de = DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + verbose=False, + extraction_retries=0, + llm_verify_retries=0, + ) + + assert de._new_source is None + + +def _make_new_attr_extractor(monkeypatch, verbose=True): + """Helper: LLM returns a helper that calls a method not in the original source.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "helper", + "placement": "module_level", + # helper calls .invented_method() — not present in _DUP_SOURCE + "helper_source": ( + "def helper(data):\n" " data.invented_method()\n" + ), + "call_site_replacements": [ + "helper(data)\n", + "helper(data)\n", + ], + } + ), + ] + return DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + verbose=verbose, + extraction_retries=0, + llm_verify_retries=0, + ) + + +def test_new_attribute_check_skips_group_verbose(monkeypatch, capsys): + de = _make_new_attr_extractor(monkeypatch, verbose=True) + assert de._new_source is None + assert "new attribute access" in capsys.readouterr().err + + +def test_new_attribute_check_skips_group_verbose_false(monkeypatch): + de = _make_new_attr_extractor(monkeypatch, verbose=False) + assert de._new_source is None + + +def test_llm_name_without_underscore_is_prefixed(monkeypatch): + """LLM returns a name without a leading '_'; extractor prepends one.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "helper", # no underscore + "placement": "module_level", + "helper_source": "def helper(data):\n pass\n", + "call_site_replacements": [ + " helper(data)\n", + " helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor( + _DUP_RANGES, source=_DUP_SOURCE, extraction_retries=0, llm_verify_retries=0 + ) + + assert de._new_source is not None + assert "def _helper(" in de._new_source + assert "def helper(" not in de._new_source + assert "_helper(data)" in de._new_source + + +def _make_proxy_seq(stmts_count: int, scope: str, class_scope=None) -> _SeqInfo: + """Build a _SeqInfo with a synthetic stmts list of the given length.""" + return _SeqInfo( + stmts=[None] * stmts_count, # type: ignore[list-item] + start_line=1, + end_line=stmts_count, + scope=scope, + source="", + fingerprint="", + class_scope=class_scope, + ) + + +def _make_proxy_func( + name: str, body_stmt_count: int, scope: str = "" +) -> _FunctionInfo: + return _FunctionInfo( + name=name, + source=f"def {name}(): pass\n", + scope=scope, + body_source=" pass\n", + body_stmt_count=body_stmt_count, + params=[], + ) + + +def test_would_create_proxy_wrappers_false_single_full_body(): + """Single-member group where the seq covers the entire function body. + + All members are proxies, so extraction is still worthwhile → False. + """ + seq = _make_proxy_seq(3, scope="foo") + func = _make_proxy_func("foo", body_stmt_count=3, scope="") + assert _would_create_proxy_wrappers([seq], [func]) is False + + +def test_would_create_proxy_wrappers_false_all_full_bodies(): + """All group members cover entire function bodies → False. + + When every member becomes a proxy the group is all-or-nothing: extracting + a shared helper is still worthwhile, so the guard should not block it. + """ + seq1 = _make_proxy_seq(3, scope="process", class_scope="ClassA") + seq2 = _make_proxy_seq(3, scope="process", class_scope="ClassB") + func1 = _make_proxy_func("process", body_stmt_count=3, scope="ClassA") + func2 = _make_proxy_func("process", body_stmt_count=3, scope="ClassB") + assert _would_create_proxy_wrappers([seq1, seq2], [func1, func2]) is False + + +def test_would_create_proxy_wrappers_false_partial_body(): + """A seq that covers only part of a function body → False.""" + seq = _make_proxy_seq(2, scope="foo") + func = _make_proxy_func("foo", body_stmt_count=4, scope="") + assert _would_create_proxy_wrappers([seq], [func]) is False + + +def test_would_create_proxy_wrappers_false_module_scope(): + """A seq at module scope (not inside a function) is never a proxy → False.""" + seq = _make_proxy_seq(3, scope="") + func = _make_proxy_func("foo", body_stmt_count=3, scope="") + assert _would_create_proxy_wrappers([seq], [func]) is False + + +def test_would_create_proxy_wrappers_false_no_matching_func(): + """No function with matching name → False.""" + seq = _make_proxy_seq(3, scope="foo") + func = _make_proxy_func("bar", body_stmt_count=3, scope="") + assert _would_create_proxy_wrappers([seq], [func]) is False + + +def test_would_create_proxy_wrappers_false_scope_mismatch(): + """Seq in class method but func is module-level with same name → False.""" + seq = _make_proxy_seq(3, scope="foo", class_scope="MyClass") + func = _make_proxy_func("foo", body_stmt_count=3, scope="") + assert _would_create_proxy_wrappers([seq], [func]) is False + + +def test_would_create_proxy_wrappers_group_with_one_proxy(): + """A group with multiple seqs, one of which covers an entire body → True.""" + seq_partial = _make_proxy_seq(2, scope="foo") + seq_full = _make_proxy_seq(3, scope="bar") + func_foo = _make_proxy_func("foo", body_stmt_count=5, scope="") + func_bar = _make_proxy_func("bar", body_stmt_count=3, scope="") + assert ( + _would_create_proxy_wrappers([seq_partial, seq_full], [func_foo, func_bar]) + is True + ) + + +_PROXY_SOURCE = textwrap.dedent( + """\ + def foo(): + setup = prepare(data) + x = compute(data) + y = transform(x) + z = finalize(y) + return setup, z + + def bar(): + x = compute(data) + y = transform(x) + z = finalize(y) + """ +) +# overlaps foo: foo has 5 stmts but duplicate block is only 3 of them (not a proxy); +# bar has 3 stmts = its entire body (would become a proxy) → mixed → guard fires. +_PROXY_RANGES = [(1, 11)] + + +def test_proxy_wrapper_guard_skips_group_verbose(monkeypatch, capsys): + """Groups that would leave a function as a trivial proxy are skipped (verbose).""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic.Anthropic"): + de = DuplicateExtractor(_PROXY_RANGES, source=_PROXY_SOURCE, verbose=True) + + assert de._new_source is None + captured = capsys.readouterr() + assert "trivial proxy wrapper" in captured.err + + +def test_proxy_wrapper_guard_skips_group_silent(monkeypatch): + """Groups that would leave a trivial proxy are skipped with verbose=False.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic.Anthropic"): + de = DuplicateExtractor(_PROXY_RANGES, source=_PROXY_SOURCE, verbose=False) + + assert de._new_source is None diff --git a/tests/duplicate_extractor/test_function_matching.py b/tests/duplicate_extractor/test_function_matching.py new file mode 100644 index 0000000..8c9c8eb --- /dev/null +++ b/tests/duplicate_extractor/test_function_matching.py @@ -0,0 +1,306 @@ +from unittest.mock import patch +import textwrap +from crispen.refactors.duplicate_extractor import ( + DuplicateExtractor, + _ApiTimeout, + _llm_veto_func_match, +) + + +# _setup() has no params; called by main() → in func_body_fps. +# foo.body fingerprint == _setup.body fingerprint. +# Diff range (2, 9) covers both _setup.body (2-4) AND foo.body (7-9). +# _setup.body hits the func.name==seq.scope True branch (skipped). +# foo.body hits the False branch and proceeds to veto → replace. +_FUNC_MATCH_SOURCE = textwrap.dedent( + """\ + def _setup(): + x = compute(data) + y = transform(x) + z = finalize(y) + + def foo(): + x = compute(data) + y = transform(x) + z = finalize(y) + + def main(): + _setup() + """ +) +_FUNC_MATCH_RANGES = [(2, 9)] # covers _setup.body AND foo.body + +# _process(val) has one param; called by main() → in func_body_fps. +# foo.body fingerprint == _process.body fingerprint (names normalized). +# Diff range covers foo.body only. +_FUNC_MATCH_PARAM_SOURCE = textwrap.dedent( + """\ + def _process(val): + y = transform(val) + z = finalize(y) + return z + + def foo(): + y = transform(data) + z = finalize(y) + return z + + def main(): + _process(data) + """ +) +_FUNC_MATCH_PARAM_RANGES = [(6, 9)] # overlaps foo.body only + +# Source with a function-match AND an independent duplicate group. +# bar/baz use an if-else structure so no sub-window of their bodies matches +# _setup's 3-chained-assignment fingerprint. +_FUNC_MATCH_THEN_DUP_SOURCE = textwrap.dedent( + """\ + def _setup(): + x = compute(data) + y = transform(x) + z = finalize(y) + + def foo(): + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(): + a = setup(items) + if condition: + result = process(items) + else: + result = fallback(items) + store(result) + + def baz(): + if quick_check: + pass + if condition: + result = process(items) + else: + result = fallback(items) + store(result) + + def main(): + _setup() + """ +) +_FUNC_MATCH_THEN_DUP_RANGES = [(2, 30)] # covers foo, bar, baz bodies + + +def test_func_match_no_arg_replaces_body(monkeypatch): + """No-param module-level function: algorithmic replacement, no call-gen LLM.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + return_value=(True, "same operation", ""), + ), + ): + de = DuplicateExtractor( + _FUNC_MATCH_RANGES, source=_FUNC_MATCH_SOURCE, verbose=True + ) + assert de._new_source is not None + assert "_setup" in de.changes_made[0] + + +def test_func_match_verbose_false(monkeypatch): + """verbose=False covers all False branches of new if-self.verbose guards.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + return_value=(True, "same operation", ""), + ), + ): + de = DuplicateExtractor( + _FUNC_MATCH_RANGES, source=_FUNC_MATCH_SOURCE, verbose=False + ) + assert de._new_source is not None + + +def test_func_match_veto_rejects(monkeypatch): + """Veto rejects func match → no replacement.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + return_value=(False, "different", ""), + ), + ): + de = DuplicateExtractor( + _FUNC_MATCH_RANGES, source=_FUNC_MATCH_SOURCE, verbose=True + ) + assert de._new_source is None + + +def test_func_match_veto_timeout(monkeypatch): + """Veto times out → seq skipped; subsequent dup group also times out.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + side_effect=_ApiTimeout("timed out"), + ), + ): + de = DuplicateExtractor( + _FUNC_MATCH_RANGES, source=_FUNC_MATCH_SOURCE, verbose=True + ) + assert de._new_source is None + + +def test_func_match_verify_fails(monkeypatch): + """_verify_extraction returns False → func match skipped; dup group veto rejects.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + # Call 1: func match veto → (True, "ok") + # Call 2: dup group veto → (False, "different") so extract is never called + side_effects = [(True, "ok", ""), (False, "different", "")] + + def _mock_run(func, timeout, *args, **kwargs): + return side_effects.pop(0) + + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + side_effect=_mock_run, + ), + patch( + "crispen.refactors.duplicate_extractor.extractor._verify_extraction", + return_value=False, + ), + ): + de = DuplicateExtractor(_FUNC_MATCH_RANGES, source=_FUNC_MATCH_SOURCE) + assert de._new_source is None + + +def test_func_match_param_call_gen_success(monkeypatch): + """Parametrised function: LLM generates call expression successfully.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + # Call 1: func match veto → (True, "ok") + # Call 2: _llm_generate_call → replacement string + side_effects: list = [(True, "ok", ""), " _process(data)\n"] + + def _mock_run(func, timeout, *args, **kwargs): + return side_effects.pop(0) + + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + side_effect=_mock_run, + ), + ): + de = DuplicateExtractor( + _FUNC_MATCH_PARAM_RANGES, + source=_FUNC_MATCH_PARAM_SOURCE, + verbose=True, + ) + assert de._new_source is not None + + +def test_func_match_param_call_gen_timeout(monkeypatch): + """Call generation times out → seq skipped; dup group veto rejects.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + # Call 1: func match veto → (True, "ok") + # Call 2: _llm_generate_call → timeout + # Call 3: dup group veto → (False, "reject") so no extract called + side_effects: list = [ + (True, "ok", ""), + _ApiTimeout("timed out"), + (False, "reject", ""), + ] + + def _mock_run(func, timeout, *args, **kwargs): + result = side_effects.pop(0) + if isinstance(result, BaseException): + raise result + return result + + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + side_effect=_mock_run, + ), + ): + de = DuplicateExtractor( + _FUNC_MATCH_PARAM_RANGES, + source=_FUNC_MATCH_PARAM_SOURCE, + verbose=True, + ) + assert de._new_source is None + + +def test_func_match_then_dup_extract(monkeypatch): + """Func match succeeds; remaining dup group triggers standard veto/extract.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + extraction_dict = { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper():\n pass\n", + "call_site_replacements": [" _helper()\n", " _helper()\n"], + } + # Call 1: func match veto → (True, "ok", "") + # Call 2: dup group veto → (True, "ok", "") + # Call 3: dup group extract → extraction dict + # Call 4: LLM verify → (True, []) + side_effects: list = [ + (True, "ok", ""), + (True, "ok", ""), + extraction_dict, + (True, []), + ] + + def _mock_run(func, timeout, *args, **kwargs): + return side_effects.pop(0) + + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + side_effect=_mock_run, + ), + ): + de = DuplicateExtractor( + _FUNC_MATCH_THEN_DUP_RANGES, + source=_FUNC_MATCH_THEN_DUP_SOURCE, + ) + assert de._new_source is not None + # One func-match change + one dup-extract change + assert len(de.changes_made) == 2 + + +def test_match_functions_false_skips_func_match_pass(monkeypatch): + """match_functions=False: func-match veto never called even when match exists.""" + + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + veto_func_match_called: list = [] + + def _mock_run_with_timeout(fn, timeout, *args, **kwargs): + if fn is _llm_veto_func_match: + veto_func_match_called.append(True) + # Reject any extraction-pass LLM call so no new source is produced. + return (False, "rejected", "") + + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + side_effect=_mock_run_with_timeout, + ), + ): + de = DuplicateExtractor( + _FUNC_MATCH_RANGES, + source=_FUNC_MATCH_SOURCE, + verbose=False, + match_functions=False, + ) + assert veto_func_match_called == [] + assert de._new_source is None diff --git a/tests/duplicate_extractor/test_import_dedup.py b/tests/duplicate_extractor/test_import_dedup.py new file mode 100644 index 0000000..65ccfb6 --- /dev/null +++ b/tests/duplicate_extractor/test_import_dedup.py @@ -0,0 +1,233 @@ +from crispen.refactors.duplicate_extractor import _lift_and_dedup_imports + + +def test_lift_and_dedup_no_changes_needed(): + src = "import os\nfrom typing import Any, Dict\nx = 1\n" + assert _lift_and_dedup_imports(src) == src + + +def test_lift_and_dedup_exact_from_duplicate(): + src = "from typing import Any\nfrom typing import Any\n" + assert _lift_and_dedup_imports(src) == "from typing import Any\n" + + +def test_lift_and_dedup_partial_overlap_adds_new_names(): + # Original F811 trigger: helper adds Any+Dict+Optional, file had Any+Dict + src = "from typing import Any, Dict\nfrom typing import Any, Dict, Optional\n" + assert _lift_and_dedup_imports(src) == "from typing import Any, Dict, Optional\n" + + +def test_lift_and_dedup_second_adds_only_new_names(): + src = "from typing import Any\nfrom typing import Optional\n" + assert _lift_and_dedup_imports(src) == "from typing import Any, Optional\n" + + +def test_lift_and_dedup_multiple_modules_independent(): + src = ( + "from typing import Any\n" + "from os.path import join\n" + "from typing import Dict\n" + "from os.path import exists\n" + ) + result = _lift_and_dedup_imports(src) + assert result == "from typing import Any, Dict\nfrom os.path import join, exists\n" + + +def test_lift_and_dedup_plain_import_deduped(): + # Unlike the old _dedup_from_imports, plain 'import X' dups are now removed + src = "import os\nimport os\n" + assert _lift_and_dedup_imports(src) == "import os\n" + + +def test_lift_and_dedup_skips_multiline_parens(): + src = "from typing import (\n Any,\n Dict,\n)\nfrom typing import Any\n" + # Paren form not matched; single-line import stands alone — no change + assert _lift_and_dedup_imports(src) == src + + +def test_lift_and_dedup_skips_wildcard(): + src = "from typing import *\nfrom typing import *\n" + assert _lift_and_dedup_imports(src) == src + + +def test_lift_and_dedup_skips_commented_import_line(): + # Inline comment prevents matching; both lines are left alone + src = "from typing import Any # noqa\nfrom typing import Any\n" + assert _lift_and_dedup_imports(src) == src + + +def test_lift_and_dedup_skips_indented_imports(): + # Indented imports (TYPE_CHECKING blocks, try/except, etc.) are not touched + src = " from typing import Any\n from typing import Dict\n" + assert _lift_and_dedup_imports(src) == src + + +def test_lift_and_dedup_empty_names_skipped(): + # Malformed import with no names: left unchanged + src = "from typing import ,\nfrom typing import ,\n" + assert _lift_and_dedup_imports(src) == src + + +def test_lift_and_dedup_non_import_lines_preserved(): + src = "from typing import Any\nx = 1\nfrom typing import Dict\ny = 2\n" + result = _lift_and_dedup_imports(src) + assert result == "from typing import Any, Dict\nx = 1\ny = 2\n" + + +def test_lift_and_dedup_lifts_misplaced_existing_module(): + # Helper inserted before second_fn lands after def first_fn → misplaced + # The import merges into the block and the misplaced copy is removed. + src = ( + "from typing import Any\n" + "\n" + "def first_fn():\n" + " pass\n" + "\n" + "from typing import Optional\n" # misplaced — helper preamble + "def _helper():\n" + " pass\n" + "\n" + "def second_fn():\n" + " pass\n" + ) + result = _lift_and_dedup_imports(src) + assert result == ( + "from typing import Any, Optional\n" + "\n" + "def first_fn():\n" + " pass\n" + "\n" + "def _helper():\n" + " pass\n" + "\n" + "def second_fn():\n" + " pass\n" + ) + + +def test_lift_and_dedup_lifts_misplaced_new_module(): + # Helper introduces a brand-new import mid-file → moved to after block. + src = ( + "from typing import Any\n" + "\n" + "def first_fn():\n" + " pass\n" + "\n" + "from collections import OrderedDict\n" # misplaced — new module + "def _helper():\n" + " pass\n" + "\n" + "def second_fn():\n" + " pass\n" + ) + result = _lift_and_dedup_imports(src) + assert result == ( + "from typing import Any\n" + "from collections import OrderedDict\n" # lifted after last block import + "\n" + "def first_fn():\n" + " pass\n" + "\n" + "def _helper():\n" + " pass\n" + "\n" + "def second_fn():\n" + " pass\n" + ) + + +def test_lift_and_dedup_lifts_misplaced_plain_import_new_module(): + # Covers: misplaced plain 'import X' (i >= first_funcdef_idx branch) and + # the new_plain_modules emission path inside _emit_new_imports. + src = ( + "from typing import Any\n" + "\n" + "def first_fn():\n" + " pass\n" + "\n" + "import os\n" # misplaced plain import — new module + "def _helper():\n" + " pass\n" + ) + result = _lift_and_dedup_imports(src) + assert result == ( + "from typing import Any\n" + "import os\n" # lifted after last block import + "\n" + "def first_fn():\n" + " pass\n" + "\n" + "def _helper():\n" + " pass\n" + ) + + +def test_lift_and_dedup_sorts_new_imports_by_pep8_section(): + # New lifted imports are sorted future→stdlib→third-party→local regardless + # of the order they were encountered. + src = ( + "from typing import Any\n" # block stdlib import + "\n" + "def first_fn():\n" + " pass\n" + "\n" + "import requests\n" # misplaced third-party + "from collections import OrderedDict\n" # misplaced stdlib + "def _helper():\n" + " pass\n" + ) + result = _lift_and_dedup_imports(src) + assert result == ( + "from typing import Any\n" + "from collections import OrderedDict\n" # stdlib before third-party + "import requests\n" + "\n" + "def first_fn():\n" + " pass\n" + "\n" + "def _helper():\n" + " pass\n" + ) + + +def test_lift_and_dedup_blank_lines_in_block_dropped(): + # Blank lines between import lines in the block are removed when the block + # is rebuilt — covers the blank-line-dropping branch in pass 5. + src = ( + "import os\n" + "\n" # blank between block imports → dropped on rebuild + "from typing import Any\n" + "from typing import Dict\n" # duplicate module → merged + "x = 1\n" + ) + result = _lift_and_dedup_imports(src) + # PEP 8 sort: both are stdlib (group 1); from_order precedes plain_order in + # all_final_imports so stable sort keeps 'from typing' before 'import os'. + assert result == ("from typing import Any, Dict\n" "import os\n" "x = 1\n") + + +def test_lift_and_dedup_no_block_imports_inserts_before_first_funcdef(): + # File has no imports at all; helper adds one mid-file → moved to very top. + src = ( + "def first_fn():\n" + " pass\n" + "\n" + "from collections import OrderedDict\n" # misplaced + "def _helper():\n" + " pass\n" + "\n" + "def second_fn():\n" + " pass\n" + ) + result = _lift_and_dedup_imports(src) + assert result == ( + "from collections import OrderedDict\n" # inserted before first funcdef + "def first_fn():\n" + " pass\n" + "\n" + "def _helper():\n" + " pass\n" + "\n" + "def second_fn():\n" + " pass\n" + ) diff --git a/tests/duplicate_extractor/test_llm_operations.py b/tests/duplicate_extractor/test_llm_operations.py new file mode 100644 index 0000000..745e840 --- /dev/null +++ b/tests/duplicate_extractor/test_llm_operations.py @@ -0,0 +1,412 @@ +from unittest.mock import MagicMock, patch +import textwrap +from crispen.errors import CrispenAPIError +from crispen.refactors.duplicate_extractor import ( + DuplicateExtractor, + _ApiTimeout, + _FunctionInfo, + _SeqInfo, + _generate_no_arg_call, + _llm_generate_call, + _llm_veto_func_match, + _run_with_timeout, +) +import pytest +from .test_extractor_core import ( + _DUP_RANGES, + _DUP_SOURCE, + _make_extract_response, + _make_verify_response, + _make_veto_response, +) + + +def _make_seq_info(start: int, end: int, src: str = "") -> _SeqInfo: + return _SeqInfo( + stmts=[], + start_line=start, + end_line=end, + scope="foo", + source=src, + fingerprint="", + ) + + +def test_llm_veto_skips_non_matching_blocks(monkeypatch): + from crispen.refactors.duplicate_extractor import _llm_veto + + client = MagicMock() + non_matching = MagicMock() + non_matching.type = "text" # not tool_use → if condition False + matching = MagicMock() + matching.type = "tool_use" + matching.name = "evaluate_duplicate" + matching.input = {"is_valid_duplicate": True, "reason": "same"} + response = MagicMock() + response.content = [non_matching, matching] + client.messages.create.return_value = response + + group = [_make_seq_info(1, 3), _make_seq_info(5, 7)] + is_valid, reason, _ = _llm_veto(client, group) + assert is_valid is True + + +def test_llm_extract_skips_non_matching_blocks(monkeypatch): + from crispen.refactors.duplicate_extractor import _llm_extract + + client = MagicMock() + non_matching = MagicMock() + non_matching.type = "text" # not tool_use → if condition False + matching = MagicMock() + matching.type = "tool_use" + matching.name = "extract_helper" + matching.input = { + "function_name": "helper", + "placement": "module_level", + "helper_source": "def helper(): pass\n", + "call_site_replacements": ["helper()\n"], + } + response = MagicMock() + response.content = [non_matching, matching] + client.messages.create.return_value = response + + group = [_make_seq_info(1, 3)] + result = _llm_extract(client, group, "a = 1\n") + assert result is not None + assert result["function_name"] == "helper" + + +def _make_veto_func_match_response(is_valid: bool, reason: str = "test") -> MagicMock: + block = MagicMock() + block.type = "tool_use" + block.name = "evaluate_duplicate" + block.input = {"is_valid_duplicate": is_valid, "reason": reason} + resp = MagicMock() + resp.content = [block] + return resp + + +def _make_call_gen_response(replacement: str) -> MagicMock: + block = MagicMock() + block.type = "tool_use" + block.name = "generate_call" + block.input = {"replacement": replacement} + resp = MagicMock() + resp.content = [block] + return resp + + +# --------------------------------------------------------------------------- +# engine integration: CrispenAPIError propagates +def test_verbose_false_suppresses_stderr(monkeypatch): + # verbose=False must take all four if-self.verbose False branches without printing. + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + import os + + def foo(): + if debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + + de = DuplicateExtractor([(12, 14)], source=source, verbose=False) + + assert de._new_source is not None + assert "_helper" in de._new_source + + +def test_engine_propagates_api_error(tmp_path, monkeypatch): + from crispen.config import CrispenConfig + from crispen.engine import run_engine + + f = tmp_path / "code.py" + f.write_text(_DUP_SOURCE, encoding="utf-8") + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("MOONSHOT_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + monkeypatch.setattr("crispen.engine.load_config", lambda: CrispenConfig()) + + with pytest.raises(CrispenAPIError): + list(run_engine({str(f): _DUP_RANGES})) + + +def test_cli_exits_on_api_error(tmp_path, monkeypatch): + import io + from crispen.cli import main + from crispen.config import CrispenConfig + + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("MOONSHOT_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + monkeypatch.setattr("crispen.cli.load_config", lambda: CrispenConfig()) + monkeypatch.setattr("crispen.engine.load_config", lambda: CrispenConfig()) + + # Write file so engine can read it + f = tmp_path / "dup.py" + f.write_text(_DUP_SOURCE, encoding="utf-8") + + diff = textwrap.dedent( + f"""\ + --- a/{f} + +++ b/{f} + @@ -10,3 +10,3 @@ + - x = compute(data) + + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + monkeypatch.setattr("sys.stdin", io.StringIO(diff)) + + with pytest.raises(SystemExit) as exc_info: + main() + assert exc_info.value.code == 1 + + +def test_run_with_timeout_fires_on_slow_func(): + import threading + + barrier = threading.Event() + try: + with pytest.raises(_ApiTimeout): + _run_with_timeout(barrier.wait, timeout=0.01) + finally: + barrier.set() # allow the daemon thread to exit cleanly + + +def test_veto_timeout_skips_group(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + side_effect=_ApiTimeout("veto timed out"), + ), + ): + de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) + assert de._new_source is None + assert de.changes_made == [] + + +def test_extract_timeout_skips_group(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + # First call (veto) returns success; second call (extract) times out. + side_effects = [(True, "same logic", ""), _ApiTimeout("extract timed out")] + + def _mock_run(func, timeout, *args, **kwargs): + result = side_effects.pop(0) + if isinstance(result, BaseException): + raise result + return result + + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + side_effect=_mock_run, + ), + ): + de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) + assert de._new_source is None + + +def test_llm_veto_func_match_accepted(): + client = MagicMock() + client.messages.create.return_value = _make_veto_func_match_response( + True, "same op" + ) + seq = _make_seq_info(7, 9, " x = 1\n") + func = _FunctionInfo( + name="fn", + source="def fn(): pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=[], + ) + is_valid, reason, _ = _llm_veto_func_match(client, seq, func, "source") + assert is_valid is True + assert reason == "same op" + + +def test_llm_veto_func_match_rejected(): + client = MagicMock() + client.messages.create.return_value = _make_veto_func_match_response( + False, "different" + ) + seq = _make_seq_info(7, 9, " x = 1\n") + func = _FunctionInfo( + name="fn", + source="def fn(): pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=[], + ) + is_valid, reason, _ = _llm_veto_func_match(client, seq, func, "source") + assert is_valid is False + + +def test_llm_veto_func_match_api_error(): + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_anthropic.APIError = Exception + client = MagicMock() + client.messages.create.side_effect = Exception("api error") + seq = _make_seq_info(7, 9, " x = 1\n") + func = _FunctionInfo( + name="fn", + source="def fn(): pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=[], + ) + with pytest.raises(CrispenAPIError): + _llm_veto_func_match(client, seq, func, "source") + + +def test_llm_veto_func_match_skips_non_matching_blocks(): + """Non-matching content block is skipped; matching block still found.""" + client = MagicMock() + non_matching = MagicMock() + non_matching.type = "text" # not tool_use → False branch of the if + matching = MagicMock() + matching.type = "tool_use" + matching.name = "evaluate_duplicate" + matching.input = {"is_valid_duplicate": True, "reason": "same"} + response = MagicMock() + response.content = [non_matching, matching] + client.messages.create.return_value = response + seq = _make_seq_info(7, 9, " x = 1\n") + func = _FunctionInfo( + name="fn", + source="def fn(): pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=[], + ) + is_valid, reason, _ = _llm_veto_func_match(client, seq, func, "source") + assert is_valid is True + + +def test_generate_no_arg_call_indented(): + seq = _make_seq_info(7, 9, " x = 1\n y = 2\n") + func = _FunctionInfo( + name="setup", + source="def setup(): pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=[], + ) + result = _generate_no_arg_call(seq, func) + assert result == " setup()\n" + + +def test_generate_no_arg_call_no_indent(): + seq = _make_seq_info(1, 2, "x = 1\ny = 2\n") + func = _FunctionInfo( + name="setup", + source="def setup(): pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=[], + ) + result = _generate_no_arg_call(seq, func) + assert result == "setup()\n" + + +def test_llm_generate_call_success(): + client = MagicMock() + client.messages.create.return_value = _make_call_gen_response( + " _process(data)\n" + ) + seq = _make_seq_info(7, 9, " y = 1\n") + func = _FunctionInfo( + name="_process", + source="def _process(val):\n pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=["val"], + ) + result = _llm_generate_call(client, seq, func, "source") + assert result == " _process(data)\n" + + +def test_llm_generate_call_api_error(): + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_anthropic.APIError = Exception + client = MagicMock() + client.messages.create.side_effect = Exception("api error") + seq = _make_seq_info(7, 9, " y = 1\n") + func = _FunctionInfo( + name="_process", + source="def _process(val):\n pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=["val"], + ) + with pytest.raises(CrispenAPIError): + _llm_generate_call(client, seq, func, "source") + + +def test_llm_generate_call_skips_non_matching_blocks(): + """Non-matching content block is skipped; matching block still found.""" + client = MagicMock() + non_matching = MagicMock() + non_matching.type = "text" # not tool_use → False branch of the if + matching = MagicMock() + matching.type = "tool_use" + matching.name = "generate_call" + matching.input = {"replacement": " _process(data)\n"} + response = MagicMock() + response.content = [non_matching, matching] + client.messages.create.return_value = response + seq = _make_seq_info(7, 9, " y = 1\n") + func = _FunctionInfo( + name="_process", + source="def _process(val):\n pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=["val"], + ) + result = _llm_generate_call(client, seq, func, "source") + assert result == " _process(data)\n" diff --git a/tests/duplicate_extractor/test_name_collision.py b/tests/duplicate_extractor/test_name_collision.py new file mode 100644 index 0000000..45171f4 --- /dev/null +++ b/tests/duplicate_extractor/test_name_collision.py @@ -0,0 +1,300 @@ +from unittest.mock import MagicMock, patch +import textwrap +from crispen.refactors.duplicate_extractor import ( + DuplicateExtractor, + _collect_ast_store_names, + _extract_defined_names, + _names_assigned_in, + _names_in_edit_texts, + _scope_end_line, + _strip_helper_docstring, +) +from .test_extractor_core import _make_extract_response, _make_veto_response + + +def test_names_in_edit_texts_collects_from_all_edits(): + groups = [ + ( + "_helper", + [ + (1, 3, "def _helper(last_import_line):\n return last_import_line\n"), + (5, 6, "result = _helper(x)\n"), + ], + "msg", + ) + ] + names = _names_in_edit_texts(groups) + assert "last_import_line" in names + assert "_helper" in names + assert "result" in names + assert "x" in names + + +def test_names_in_edit_texts_skips_syntax_errors(): + groups = [("_h", [(1, 2, "def (\n")], "msg")] + # Should not raise — returns whatever names were parseable. + names = _names_in_edit_texts(groups) + assert isinstance(names, set) + + +def test_names_assigned_in_simple(): + assert _names_assigned_in("x = 1\n") == {"x"} + + +def test_names_assigned_in_tuple_unpack(): + assert _names_assigned_in("x, y = f()\n") == {"x", "y"} + + +def test_names_assigned_in_augassign(): + assert _names_assigned_in("x += 1\n") == {"x"} + + +def test_names_assigned_in_no_assign(): + assert _names_assigned_in("f()\n") == set() + + +def test_names_assigned_in_syntax_error(): + assert _names_assigned_in("def (\n") == set() + + +def test_extract_defined_names_basic(): + source = textwrap.dedent( + """\ + def foo(): + pass + + async def bar(): + pass + + class Baz: + pass + """ + ) + assert _extract_defined_names(source) == {"foo", "bar", "Baz"} + + +def test_extract_defined_names_syntax_error(): + assert _extract_defined_names("def (\n") == set() + + +# Source that already defines _helper AND has duplicate blocks. +_COLLISION_SOURCE = textwrap.dedent( + """\ + def _helper(x): + return x + + def foo(): + if debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ +) +_COLLISION_RANGES = [(12, 14)] # overlaps bar's duplicate block + + +def test_extraction_name_collision_skipped(monkeypatch, capsys): + # LLM returns function_name="_helper", which is already defined → skipped. + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(x, y):\n pass\n", + "call_site_replacements": [ + " _helper(data, x)\n", + " _helper(data, x)\n", + ], + } + ), + ] + de = DuplicateExtractor( + _COLLISION_RANGES, + source=_COLLISION_SOURCE, + verbose=True, + extraction_retries=0, + llm_verify_retries=0, + ) + + assert de._new_source is None + assert de.changes_made == [] + err = capsys.readouterr().err + assert "name collision" in err + assert "_helper" in err + + +def test_extraction_name_collision_silent(monkeypatch, capsys): + # Same collision, verbose=False → no stderr output. + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(x, y):\n pass\n", + "call_site_replacements": [ + " _helper(data, x)\n", + " _helper(data, x)\n", + ], + } + ), + ] + de = DuplicateExtractor( + _COLLISION_RANGES, + source=_COLLISION_SOURCE, + verbose=False, + extraction_retries=0, + llm_verify_retries=0, + ) + + assert de._new_source is None + assert de.changes_made == [] + err = capsys.readouterr().err + assert "name collision" not in err + + +def test_strip_helper_docstring_with_docstring(): + source = 'def _helper(x):\n """Strip me."""\n return x\n' + result = _strip_helper_docstring(source) + assert '"""Strip me."""' not in result + assert "return x" in result + + +def test_strip_helper_docstring_no_docstring(): + source = "def _helper(x):\n return x\n" + result = _strip_helper_docstring(source) + assert result == source + + +def test_strip_helper_docstring_parse_error(): + bad = "def f(:\n pass\n" + result = _strip_helper_docstring(bad) + assert result == bad + + +def test_strip_helper_docstring_non_function(): + source = "x = 1\n" + result = _strip_helper_docstring(source) + assert result == source + + +def test_strip_helper_docstring_docstring_only_body(): + # Function whose body is only a docstring — don't strip (would leave empty body). + source = 'def _helper():\n """Only doc."""\n' + result = _strip_helper_docstring(source) + assert '"""Only doc."""' in result + + +def test_collect_ast_store_names_simple_name(): + import ast + + node = ast.parse("x = 1").body[0].targets[0] + names: list = [] + _collect_ast_store_names(node, names) + assert names == ["x"] + + +def test_collect_ast_store_names_tuple(): + import ast + + node = ast.parse("a, b = 1, 2").body[0].targets[0] + names: list = [] + _collect_ast_store_names(node, names) + assert set(names) == {"a", "b"} + + +def test_collect_ast_store_names_nested_tuple(): + import ast + + node = ast.parse("(a, (b, c)) = x").body[0].targets[0] + names: list = [] + _collect_ast_store_names(node, names) + assert set(names) == {"a", "b", "c"} + + +def test_collect_ast_store_names_non_name_non_tuple_noop(): + # ast.Attribute target (e.g. self.x) → nothing collected. + import ast + + node = ast.parse("self.x = 1").body[0].targets[0] + names: list = [] + _collect_ast_store_names(node, names) + assert names == [] + + +def _make_source_lines(src: str): + return src.splitlines(keepends=True) + + +def test_scope_end_line_module_returns_full_length(): + lines = _make_source_lines("x = 1\ny = 2\n") + assert _scope_end_line(lines, "", 1) == len(lines) + + +def test_scope_end_line_function_scope(): + src = "def foo():\n x = 1\n y = 2\n\ndef bar():\n z = 3\n" + lines = _make_source_lines(src) + # Block ends at line 2 (inside foo). foo ends at line 3. + assert _scope_end_line(lines, "foo", 2) == 3 + + +def test_scope_end_line_does_not_bleed_into_next_function(): + src = "def foo():\n x = 1\n\ndef bar():\n x = 2\n" + lines = _make_source_lines(src) + # Searching for `x` after line 2 should stop at end of foo (line 2), not + # reach bar where `x` also appears. + end = _scope_end_line(lines, "foo", 2) + assert end == 2 # foo ends at line 2; bar's x is excluded + + +def test_scope_end_line_picks_innermost_matching_scope(): + # Two functions named "inner" — one nested inside outer, one at module level. + src = ( + "def outer():\n" + " def inner():\n" + " a = 1\n" + " inner()\n" + "\n" + "def inner():\n" + " b = 2\n" + ) + lines = _make_source_lines(src) + # Block at line 3 is inside the nested inner (lines 2-3). That is the + # smallest matching span, so end_lineno == 3 is returned. + assert _scope_end_line(lines, "inner", 3) == 3 + + +def test_scope_end_line_class_scope(): + src = "class Foo:\n x = 1\n y = 2\n\nclass Bar:\n x = 3\n" + lines = _make_source_lines(src) + assert _scope_end_line(lines, "Foo", 2) == 3 + + +def test_scope_end_line_no_match_returns_full_length(): + src = "def foo():\n x = 1\n" + lines = _make_source_lines(src) + # Scope name doesn't match any definition. + assert _scope_end_line(lines, "bar", 1) == len(lines) + + +def test_scope_end_line_syntax_error_returns_full_length(): + lines = _make_source_lines("def (\n x = 1\n") + assert _scope_end_line(lines, "foo", 1) == len(lines) diff --git a/tests/duplicate_extractor/test_node_utils.py b/tests/duplicate_extractor/test_node_utils.py new file mode 100644 index 0000000..8d49262 --- /dev/null +++ b/tests/duplicate_extractor/test_node_utils.py @@ -0,0 +1,235 @@ +from crispen.refactors.duplicate_extractor import ( + _SeqInfo, + _has_def, + _node_weight, + _normalize_source, + _overlaps_diff, + _sequence_weight, +) +import libcst as cst + + +def _parse_stmt(src: str) -> cst.BaseStatement: + return cst.parse_module(src).body[0] + + +def test_node_weight_simple_one(): + assert _node_weight(_parse_stmt("a = 1\n")) == 1 + + +def test_node_weight_simple_two_semicolons(): + # Two small stmts on one line separated by semicolon + stmt = _parse_stmt("a = 1; b = 2\n") + assert _node_weight(stmt) == 2 + + +def test_node_weight_indented_block(): + block = _parse_stmt("if True:\n a = 1\n b = 2\n").body + assert _node_weight(block) == 2 + + +def test_node_weight_else(): + if_node = _parse_stmt("if True:\n a = 1\nelse:\n b = 2\n") + else_node = if_node.orelse + assert _node_weight(else_node) == 1 + + +def test_node_weight_finally(): + try_node = _parse_stmt("try:\n a = 1\nfinally:\n b = 2\n") + finally_node = try_node.finalbody + assert _node_weight(finally_node) == 1 + + +def test_node_weight_functiondef(): + stmt = _parse_stmt("def foo():\n pass\n") + assert _node_weight(stmt) == 1 + + +def test_node_weight_classdef(): + stmt = _parse_stmt("class Foo:\n pass\n") + assert _node_weight(stmt) == 1 + + +def test_node_weight_non_statement(): + name_node = cst.Name("foo") + assert _node_weight(name_node) == 0 + + +def test_node_weight_if_no_else(): + # weight = 1 (if) + 2 (body) + stmt = _parse_stmt("if x:\n a = 1\n b = 2\n") + assert _node_weight(stmt) == 3 + + +def test_node_weight_if_with_else(): + # weight = 1 (if) + 1 (body) + 1 (else body) + stmt = _parse_stmt("if x:\n a = 1\nelse:\n b = 2\n") + assert _node_weight(stmt) == 3 + + +def test_node_weight_for(): + # weight = 1 (for) + 1 (body) + stmt = _parse_stmt("for i in x:\n a = 1\n") + assert _node_weight(stmt) == 2 + + +def test_node_weight_for_with_else(): + # weight = 1 (for) + 1 (body) + 1 (else body) + stmt = _parse_stmt("for i in x:\n a = 1\nelse:\n b = 2\n") + assert _node_weight(stmt) == 3 + + +def test_node_weight_while(): + stmt = _parse_stmt("while x:\n a = 1\n") + assert _node_weight(stmt) == 2 + + +def test_node_weight_try_with_handler(): + # weight = 1 (try) + 1 (body) + 1 (handler body) + stmt = _parse_stmt("try:\n a = 1\nexcept:\n b = 2\n") + assert _node_weight(stmt) == 3 + + +def test_node_weight_try_with_handler_and_finally(): + # weight = 1 + 1 + 1 + 1 (finally body) + stmt = _parse_stmt("try:\n a = 1\nexcept:\n b = 2\nfinally:\n c = 3\n") + assert _node_weight(stmt) == 4 + + +def test_node_weight_try_with_orelse(): + # weight = 1 + 1 (body) + 1 (handler) + 1 (else body) + stmt = _parse_stmt("try:\n a = 1\nexcept:\n b = 2\nelse:\n c = 3\n") + assert _node_weight(stmt) == 4 + + +def test_node_weight_with(): + stmt = _parse_stmt("with open('f') as fh:\n a = 1\n") + assert _node_weight(stmt) == 2 + + +def test_sequence_weight_empty(): + assert _sequence_weight([]) == 0 + + +def test_sequence_weight_mixed(): + stmts = [ + _parse_stmt("a = 1\n"), + _parse_stmt("if x:\n b = 2\n"), + ] + assert _sequence_weight(stmts) == 1 + 2 + + +def test_has_def_no_def(): + stmts = [_parse_stmt("a = 1\n"), _parse_stmt("b = 2\n")] + assert _has_def(stmts) is False + + +def test_has_def_with_functiondef(): + stmts = [_parse_stmt("a = 1\n"), _parse_stmt("def foo():\n pass\n")] + assert _has_def(stmts) is True + + +def test_has_def_with_classdef(): + stmts = [_parse_stmt("class Foo:\n pass\n")] + assert _has_def(stmts) is True + + +def test_normalize_source_normalizes_vars(): + src = "result = compute(data)\noutput = transform(result)\n" + norm = _normalize_source(src) + # All names (both assigned and free) are replaced with positional placeholders + assert "result" not in norm + assert "output" not in norm + assert "compute" not in norm + assert "data" not in norm + + +def test_normalize_source_same_fingerprint(): + src_a = "x = compute(data)\ny = transform(x)\n" + src_b = "val = compute(data)\nres = transform(val)\n" + assert _normalize_source(src_a) == _normalize_source(src_b) + + +def test_normalize_source_different_ops(): + # Structurally different code (different number of statements) should differ + src_a = "x = a + b\n" + src_b = "x = a + b\ny = x * 2\n" + assert _normalize_source(src_a) != _normalize_source(src_b) + + +def test_normalize_source_invalid_syntax(): + src = "def f(: pass" + # Falls back to original source + assert _normalize_source(src) == src + + +def test_normalize_source_load_context_replaced(): + # Var assigned then used: both should be normalized the same + src_a = "x = 1\ny = x + 1\n" + src_b = "a = 1\nb = a + 1\n" + assert _normalize_source(src_a) == _normalize_source(src_b) + + +def test_normalize_source_load_not_in_map(): + # Free variables (Load context, never stored) are also normalized, + # so two blocks with different free variable names get the same fingerprint. + src_a = "y = a + 1\n" + src_b = "z = b + 1\n" + assert _normalize_source(src_a) == _normalize_source(src_b) + + +def test_normalize_source_repeated_store(): + # Same name assigned twice: _placeholder called with cached key (False branch) + src = "x = 1\nx = 2\n" + norm = _normalize_source(src) + # Both assignments normalize to the same placeholder + assert norm.count("_v0") == 2 + + +def test_normalize_source_del_context(): + # Del context falls through to return node unchanged + src = "del x\n" + norm = _normalize_source(src) + assert "x" in norm + + +def test_normalize_source_free_variables_match(): + # Blocks differing only in free variable names should get the same fingerprint. + # This is the core case: `p = a * 2; if p > 100: p += 1` vs the same with q/b. + src_a = "p = a * 2\nif p > 100:\n p += 1\n" + src_b = "q = b * 2\nif q > 100:\n q += 1\n" + assert _normalize_source(src_a) == _normalize_source(src_b) + + +def test_normalize_source_indented_blocks_match(): + # Source collected from inside a function is indented; dedent must happen + # before ast.parse so that structurally identical blocks still match. + src_a = " p = a * 2\n if p > 100:\n p += 1\n" + src_b = " q = b * 2\n if q > 100:\n q += 1\n" + assert _normalize_source(src_a) == _normalize_source(src_b) + + +def _make_seq(start: int, end: int) -> _SeqInfo: + return _SeqInfo( + stmts=[], + start_line=start, + end_line=end, + scope="", + source="", + fingerprint="", + ) + + +def test_overlaps_diff_yes(): + seq = _make_seq(5, 10) + assert _overlaps_diff(seq, [(8, 12)]) is True + + +def test_overlaps_diff_no(): + seq = _make_seq(5, 10) + assert _overlaps_diff(seq, [(11, 20)]) is False + + +def test_overlaps_diff_exact_boundary(): + seq = _make_seq(5, 10) + assert _overlaps_diff(seq, [(10, 15)]) is True diff --git a/tests/duplicate_extractor/test_post_block_guards.py b/tests/duplicate_extractor/test_post_block_guards.py new file mode 100644 index 0000000..d39efd2 --- /dev/null +++ b/tests/duplicate_extractor/test_post_block_guards.py @@ -0,0 +1,298 @@ +from unittest.mock import MagicMock, patch +import textwrap +from crispen.refactors.duplicate_extractor import ( + DuplicateExtractor, + _SeqInfo, + _helper_imports_local_name, + _replacement_steals_post_block_line, +) +from .test_extractor_core import _make_extract_response, _make_veto_response + + +_POST_STEAL_SOURCE = textwrap.dedent( + """\ + def foo(): + x = compute(data) + y = transform(x) + z = finalize(y) + return z + + def bar(): + x = compute(data) + y = transform(x) + z = finalize(y) + logger.info("done") + """ +) +_POST_STEAL_RANGES = [(8, 10)] # overlaps bar's 3-statement block + + +def test_replacement_steals_post_block_skipped(monkeypatch): + """Replacement whose last line matches the post-block line is rejected.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "_do_work", + "placement": "module_level", + "helper_source": ( + "def _do_work(data):\n" + " x = compute(data)\n" + " y = transform(x)\n" + " z = finalize(y)\n" + ), + "call_site_replacements": [ + " _do_work(data)\n return z\n", # steals "return z" + " _do_work(data)\n", + ], + } + ), + ] + de = DuplicateExtractor( + _POST_STEAL_RANGES, + source=_POST_STEAL_SOURCE, + extraction_retries=0, + llm_verify_retries=0, + ) + + assert de._new_source is None + + +def test_replacement_steals_post_block_skipped_verbose_false(monkeypatch): + """verbose=False covers the False branch of the verbose guard.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "_do_work", + "placement": "module_level", + "helper_source": ( + "def _do_work(data):\n" + " x = compute(data)\n" + " y = transform(x)\n" + " z = finalize(y)\n" + ), + "call_site_replacements": [ + " _do_work(data)\n return z\n", # steals "return z" + " _do_work(data)\n", + ], + } + ), + ] + de = DuplicateExtractor( + _POST_STEAL_RANGES, + source=_POST_STEAL_SOURCE, + verbose=False, + extraction_retries=0, + llm_verify_retries=0, + ) + + assert de._new_source is None + + +def _make_steal_seq(end_line: int) -> _SeqInfo: + return _SeqInfo( + stmts=[], start_line=1, end_line=end_line, scope="f", source="", fingerprint="" + ) + + +def test_replacement_steals_post_block_at_eof(): + # Block is the last line of the file — no post-block line exists. + source_lines = ["x = 1\n"] + seq = _make_steal_seq(1) # next_idx=1 >= len=1 → skip + assert not _replacement_steals_post_block_line( + [seq], ["y = helper()\n"], source_lines + ) + + +def test_replacement_steals_post_block_blank_after(): + # Post-block line is blank but there is a non-blank line further down. + # The check must scan past the blank to find the real post-block code. + source_lines = ["x = 1\n", "\n", "y = 2\n"] + seq = _make_steal_seq(1) # next_idx=1 → "\n" → scan → next_idx=2 → "y = 2" + assert _replacement_steals_post_block_line([seq], ["y = 2\n"], source_lines) + + +def test_replacement_steals_post_block_blank_after_no_match(): + # Blank after block, but replacement doesn't steal the non-blank post-block line. + source_lines = ["x = 1\n", "\n", "y = 2\n"] + seq = _make_steal_seq(1) + assert not _replacement_steals_post_block_line( + [seq], ["z = helper()\n"], source_lines + ) + + +def test_replacement_steals_post_block_all_blank_after(): + # Only blank lines follow the block — no real post-block line to steal. + source_lines = ["x = 1\n", "\n", "\n"] + seq = _make_steal_seq(1) + assert not _replacement_steals_post_block_line( + [seq], ["z = helper()\n"], source_lines + ) + + +def test_replacement_steals_post_block_no_match(): + # Replacement last line doesn't match post-block line. + source_lines = ["x = 1\n", "y = 2\n"] + seq = _make_steal_seq(1) # next_idx=1 → "y = 2" + assert not _replacement_steals_post_block_line( + [seq], ["z = helper()\n"], source_lines + ) + + +def test_replacement_steals_post_block_match(): + # Replacement last line matches post-block line → steal detected. + source_lines = ["x = 1\n", "y = 2\n"] + seq = _make_steal_seq(1) # next_idx=1 → "y = 2" + assert _replacement_steals_post_block_line( + [seq], ["z = helper()\ny = 2\n"], source_lines + ) + + +def test_helper_imports_local_name_true(): + helper = "def _h():\n import mock_client\n mock_client.run()\n" + original = "def test(mock_client):\n mock_client.run()\n" + assert _helper_imports_local_name(helper, original) is True + + +def test_helper_imports_local_name_already_imported_in_original(): + # mock_client is already a top-level import → not a local-only name. + helper = "def _h():\n import mock_client\n mock_client.run()\n" + original = "import mock_client\ndef test(x):\n mock_client.run()\n" + assert _helper_imports_local_name(helper, original) is False + + +def test_helper_imports_local_name_no_imports_in_helper(): + helper = "def _h():\n pass\n" + original = "def test(mock_client):\n pass\n" + assert _helper_imports_local_name(helper, original) is False + + +def test_helper_imports_local_name_syntax_error_helper(): + assert _helper_imports_local_name("def (:\n", "def test(x):\n pass\n") is False + + +def test_helper_imports_local_name_syntax_error_original(): + assert _helper_imports_local_name("def _h():\n import x\n", "(:\n") is False + + +def test_helper_imports_local_name_from_import_in_helper(): + # "from X import Y" in helper: the tracked name is "Y", not "X". + # If "Y" is a param in the original, it is flagged. + helper = "def _h():\n from pkg import mock_client\n mock_client.run()\n" + original = "def test(mock_client):\n mock_client.run()\n" + assert _helper_imports_local_name(helper, original) is True + + +def test_helper_imports_local_name_from_import_in_original(): + # Top-level "from pkg import something" in the original covers the branch + # in the orig_top_imports loop and prevents false-positive flagging. + helper = "def _h():\n import something\n something.run()\n" + original = "from pkg import something\ndef test(x):\n something.run()\n" + assert _helper_imports_local_name(helper, original) is False + + +def test_helper_imports_local_name_vararg(): + # Function with *args: vararg name tracked as potential local. + helper = "def _h():\n import args\n" + original = "def test(*args):\n pass\n" + assert _helper_imports_local_name(helper, original) is True + + +def test_helper_imports_local_name_kwarg(): + # Function with **kwargs: kwarg name tracked as potential local. + helper = "def _h():\n import kwargs\n" + original = "def test(**kwargs):\n pass\n" + assert _helper_imports_local_name(helper, original) is True + + +_PARAM_DUP_SOURCE = textwrap.dedent( + """\ + def test_a(mock_client): + if debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def test_b(mock_client): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ +) +_PARAM_DUP_RANGES = [(10, 12)] # overlaps test_b's duplicate block + + +def _make_import_local_extract_response(): + return _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + # helper imports mock_client instead of taking it as a parameter + "helper_source": ( + "def _helper():\n" + " import mock_client\n" + " x = compute(data)\n" + " y = transform(x)\n" + " z = finalize(y)\n" + ), + "call_site_replacements": [ + " _helper()\n", + " _helper()\n", + ], + } + ) + + +def test_helper_imports_local_guard_skips(monkeypatch, capsys): + """Extraction rejected when helper imports a name that is a param in original.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_import_local_extract_response(), + ] + de = DuplicateExtractor( + _PARAM_DUP_RANGES, + source=_PARAM_DUP_SOURCE, + extraction_retries=0, + llm_verify_retries=0, + ) + assert de._new_source is None + assert "helper imports a name that is a parameter/local" in capsys.readouterr().err + + +def test_helper_imports_local_guard_skips_silent(monkeypatch): + """verbose=False: extraction rejected with no stderr output.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_import_local_extract_response(), + ] + de = DuplicateExtractor( + _PARAM_DUP_RANGES, + source=_PARAM_DUP_SOURCE, + verbose=False, + extraction_retries=0, + llm_verify_retries=0, + ) + assert de._new_source is None diff --git a/tests/duplicate_extractor/test_staticmethod_placement.py b/tests/duplicate_extractor/test_staticmethod_placement.py new file mode 100644 index 0000000..2fa5cf4 --- /dev/null +++ b/tests/duplicate_extractor/test_staticmethod_placement.py @@ -0,0 +1,599 @@ +from unittest.mock import MagicMock, patch +import textwrap +from crispen.refactors.duplicate_extractor import DuplicateExtractor +from .test_extractor_core import ( + _make_extract_response, + _make_verify_response, + _make_veto_response, +) + + +def test_staticmethod_placement(monkeypatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + class MyClass: + def foo(self): + if self.debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(self): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper = " @staticmethod\n def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "_helper", + "placement": "staticmethod:MyClass", + "helper_source": helper, + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + + de = DuplicateExtractor([(11, 13)], source=source) + + assert de._new_source is not None + + +def test_staticmethod_placement_zero_indent_helper_auto_indented(monkeypatch): + """0-indent helper with staticmethod: placement is auto-indented into the class.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + class MyClass: + def foo(self): + if self.debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(self): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + # LLM generates a 0-indent (module-level) def even though it requested + # staticmethod:MyClass placement. Without auto-indent this would end the + # class body at the docstring, making foo/bar nested inside the helper. + helper_zero_indent = "def _helper(self, data):\n return compute(data)\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "_helper", + "placement": "staticmethod:MyClass", + "helper_source": helper_zero_indent, + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + + de = DuplicateExtractor([(11, 13)], source=source) + + assert de._new_source is not None + # foo and bar must remain real class methods, not nested inside the helper. + import ast as _ast + + tree = _ast.parse(de._new_source) + class_def = next( + n + for n in _ast.walk(tree) + if isinstance(n, _ast.ClassDef) and n.name == "MyClass" + ) + top_level_methods = { + n.name for n in class_def.body if isinstance(n, _ast.FunctionDef) + } + assert "foo" in top_level_methods + assert "bar" in top_level_methods + assert "_helper" in top_level_methods + + +def test_cross_class_duplicates_use_module_level_placement(monkeypatch): + """Duplicates in different classes must be extracted as module-level functions.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + class ClassA: + def foo(self): + if self.debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + class ClassB: + def bar(self): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + responses = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + mock_client.messages.create.side_effect = responses + de = DuplicateExtractor([(3, 5)], source=source) + + assert de._new_source is not None + # The extraction call prompt should tell the LLM to use module_level + extract_prompt = mock_client.messages.create.call_args_list[1][1]["messages"][0][ + "content" + ] + assert "module_level" in extract_prompt + assert "staticmethod" not in extract_prompt + + +def test_cross_class_staticmethod_placement_rejected(monkeypatch): + """LLM returning staticmethod placement for a cross-class group is rejected.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + class ClassA: + def foo(self): + if self.debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + class ClassB: + def bar(self): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + # First extraction attempt: LLM ignores prompt and returns staticmethod + # placement for a cross-class group → rejected; second attempt: correct. + responses = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "_helper", + "placement": "staticmethod:ClassA", + "helper_source": ( + " @staticmethod\n def _helper(data):\n pass\n" + ), + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + mock_client.messages.create.side_effect = responses + de = DuplicateExtractor([(3, 5)], source=source) + + assert de._new_source is not None + # Three LLM calls: veto + two extraction attempts + assert mock_client.messages.create.call_count == 4 + + +def test_cross_class_staticmethod_placement_rejected_non_verbose(monkeypatch): + """Defensive cross-class check works when verbose=False (no print side-effect).""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + class ClassA: + def foo(self): + if self.debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + class ClassB: + def bar(self): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + responses = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "_helper", + "placement": "staticmethod:ClassA", + "helper_source": ( + " @staticmethod\n def _helper(data):\n pass\n" + ), + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + mock_client.messages.create.side_effect = responses + de = DuplicateExtractor([(3, 5)], source=source, verbose=False) + + assert de._new_source is not None + + +def test_same_class_module_level_placement_rejected(monkeypatch): + """module_level placement with self.() call sites is rejected and retried.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + class MyClass: + def foo(self): + if self.debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(self): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper_module = "def _helper(data):\n pass\n" + helper_static = " @staticmethod\n def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + responses = [ + _make_veto_response(True), + # First attempt: module_level placement but call sites use self._helper() + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper_module, + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + # Second attempt: correct staticmethod placement + _make_extract_response( + { + "function_name": "_helper", + "placement": "staticmethod:MyClass", + "helper_source": helper_static, + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + mock_client.messages.create.side_effect = responses + de = DuplicateExtractor([(11, 13)], source=source) + + assert de._new_source is not None + # veto + two extraction attempts + verify + assert mock_client.messages.create.call_count == 4 + + +def test_same_class_module_level_placement_rejected_non_verbose(monkeypatch): + """Same inconsistency rejection works when verbose=False.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + class MyClass: + def foo(self): + if self.debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(self): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper_static = " @staticmethod\n def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + responses = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(data):\n pass\n", + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + _make_extract_response( + { + "function_name": "_helper", + "placement": "staticmethod:MyClass", + "helper_source": helper_static, + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + mock_client.messages.create.side_effect = responses + de = DuplicateExtractor([(11, 13)], source=source, verbose=False) + + assert de._new_source is not None + + +def test_cross_class_module_level_self_call_rejected(monkeypatch): + """module_level with self.() call sites in a cross-class group is rejected.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + class ClassA: + def foo(self): + if self.debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + class ClassB: + def bar(self): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + responses = [ + _make_veto_response(True), + # First attempt: module_level but call sites use self._helper() + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + # Second attempt: correct call sites + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + mock_client.messages.create.side_effect = responses + de = DuplicateExtractor([(3, 5)], source=source) + + assert de._new_source is not None + assert mock_client.messages.create.call_count == 4 + + +def test_staticmethod_wrong_class_rejected(monkeypatch): + """LLM naming the wrong class in staticmethod:X is rejected and retried.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + class ClassA: + def setup(self): + pass + + class ClassB: + def foo(self): + if self.debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(self): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper_static = " @staticmethod\n def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + responses = [ + _make_veto_response(True), + # First attempt: LLM names the wrong class + _make_extract_response( + { + "function_name": "_helper", + "placement": "staticmethod:ClassA", + "helper_source": helper_static, + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + # Second attempt: correct class name + _make_extract_response( + { + "function_name": "_helper", + "placement": "staticmethod:ClassB", + "helper_source": helper_static, + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + mock_client.messages.create.side_effect = responses + de = DuplicateExtractor([(14, 16)], source=source) + + assert de._new_source is not None + assert mock_client.messages.create.call_count == 4 + + +def test_staticmethod_wrong_class_rejected_non_verbose(monkeypatch): + """Wrong-class staticmethod rejection works when verbose=False.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + class ClassA: + def setup(self): + pass + + class ClassB: + def foo(self): + if self.debug: + pass + x = compute(data) + y = transform(x) + z = finalize(y) + + def bar(self): + result = None + x = compute(data) + y = transform(x) + z = finalize(y) + """ + ) + helper_static = " @staticmethod\n def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + responses = [ + _make_veto_response(True), + _make_extract_response( + { + "function_name": "_helper", + "placement": "staticmethod:ClassA", + "helper_source": helper_static, + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + _make_extract_response( + { + "function_name": "_helper", + "placement": "staticmethod:ClassB", + "helper_source": helper_static, + "call_site_replacements": [ + " self._helper(data)\n", + " self._helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + mock_client.messages.create.side_effect = responses + de = DuplicateExtractor([(14, 16)], source=source, verbose=False) + + assert de._new_source is not None diff --git a/tests/duplicate_extractor/test_timing_detailed.py b/tests/duplicate_extractor/test_timing_detailed.py new file mode 100644 index 0000000..ec74227 --- /dev/null +++ b/tests/duplicate_extractor/test_timing_detailed.py @@ -0,0 +1,280 @@ +from unittest.mock import MagicMock, patch +from crispen.refactors.duplicate_extractor import ( + DuplicateExtractor, + _FunctionInfo, + _llm_generate_call, + _llm_veto_func_match, +) +from .test_extractor_core import ( + _DUP_RANGES, + _DUP_SOURCE, + _make_extract_response, + _make_verify_response, + _make_veto_response, +) +from .test_llm_operations import ( + _make_call_gen_response, + _make_seq_info, + _make_veto_func_match_response, +) +from .test_function_matching import _FUNC_MATCH_PARAM_RANGES, _FUNC_MATCH_PARAM_SOURCE + + +def test_llm_veto_with_timing_out(monkeypatch): + """_llm_veto appends result to _timing_out when provided.""" + from crispen.refactors.duplicate_extractor import _llm_veto + + client = MagicMock() + client.messages.create.return_value = _make_veto_response(True, "ok") + group = [_make_seq_info(1, 3), _make_seq_info(5, 7)] + timing: list = [] + _llm_veto(client, group, _timing_out=timing) + assert len(timing) == 1 + assert timing[0].tool_input == {"is_valid_duplicate": True, "reason": "ok"} + + +def test_llm_veto_func_match_with_timing_out(): + """_llm_veto_func_match appends result to _timing_out when provided.""" + client = MagicMock() + client.messages.create.return_value = _make_veto_func_match_response(True, "same") + seq = _make_seq_info(7, 9, " x = 1\n") + func = _FunctionInfo( + name="fn", + source="def fn(): pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=[], + ) + timing: list = [] + _llm_veto_func_match(client, seq, func, "source", _timing_out=timing) + assert len(timing) == 1 + assert timing[0].tool_input["is_valid_duplicate"] is True + + +def test_llm_generate_call_with_timing_out(): + """_llm_generate_call appends result to _timing_out when provided.""" + client = MagicMock() + client.messages.create.return_value = _make_call_gen_response(" fn(data)\n") + seq = _make_seq_info(7, 9, " y = 1\n") + func = _FunctionInfo( + name="fn", + source="def fn(val):\n pass\n", + scope="", + body_source=" pass\n", + body_stmt_count=1, + params=["val"], + ) + timing: list = [] + result = _llm_generate_call(client, seq, func, "source", _timing_out=timing) + assert result == " fn(data)\n" + assert len(timing) == 1 + + +def test_llm_verify_extraction_with_timing_out(): + """_llm_verify_extraction appends result to _timing_out when provided.""" + from crispen.refactors.duplicate_extractor import _llm_verify_extraction + + client = MagicMock() + client.messages.create.return_value = _make_verify_response(True, []) + group = [_make_seq_info(1, 3), _make_seq_info(5, 7)] + timing: list = [] + is_correct, issues = _llm_verify_extraction( + client, + group, + "def _helper(): pass\n", + [" _helper()\n", " _helper()\n"], + "a = 1\nb = 2\n", + _timing_out=timing, + ) + assert is_correct is True + assert len(timing) == 1 + + +def test_llm_verify_extraction_without_timing_out(): + """_llm_verify_extraction works correctly when _timing_out is None.""" + from crispen.refactors.duplicate_extractor import _llm_verify_extraction + + client = MagicMock() + client.messages.create.return_value = _make_verify_response(True, []) + group = [_make_seq_info(1, 3), _make_seq_info(5, 7)] + is_correct, issues = _llm_verify_extraction( + client, + group, + "def _helper(): pass\n", + [" _helper()\n", " _helper()\n"], + "a = 1\nb = 2\n", + ) + assert is_correct is True + assert issues == [] + + +def test_func_match_veto_timing_recorded(monkeypatch): + """When func-match veto accepts, record_llm_call is invoked for the veto call.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + # veto accepts → call-gen runs (func has params) → done (no dup groups) + mock_client.messages.create.side_effect = [ + _make_veto_func_match_response(True, "same"), + _make_call_gen_response(" _process(data)\n"), + ] + de = DuplicateExtractor( + _FUNC_MATCH_PARAM_RANGES, + source=_FUNC_MATCH_PARAM_SOURCE, + ) + # record_llm_call ran for veto (the timing branch was True) + assert de.stats.llm_elapsed_by_category.get("veto", 0) >= 0 + + +def test_func_match_call_gen_timing_recorded(monkeypatch): + """When func-match call-gen runs, record_llm_call is invoked for the edit call.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + # veto accepts → call-gen runs → done (no dup groups) + mock_client.messages.create.side_effect = [ + _make_veto_func_match_response(True, "same"), + _make_call_gen_response(" _process(data)\n"), + ] + de = DuplicateExtractor( + _FUNC_MATCH_PARAM_RANGES, + source=_FUNC_MATCH_PARAM_SOURCE, + ) + assert de.stats.llm_edit_calls >= 1 + + +def test_func_match_veto_detailed_timing_suffix(monkeypatch, capsys): + """timing='detailed' prints timing suffix after func-match veto result.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_func_match_response(True, "same"), + _make_call_gen_response(" _process(data)\n"), + ] + DuplicateExtractor( + _FUNC_MATCH_PARAM_RANGES, + source=_FUNC_MATCH_PARAM_SOURCE, + verbose=True, + timing="detailed", + ) + err = capsys.readouterr().err + assert "ACCEPTED" in err + assert "[" in err # timing suffix present + + +def test_func_match_replacement_detailed_timing_suffix(monkeypatch, capsys): + """timing='detailed' prints timing suffix after func-match replacement line.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_func_match_response(True, "same"), + _make_call_gen_response(" _process(data)\n"), + ] + DuplicateExtractor( + _FUNC_MATCH_PARAM_RANGES, + source=_FUNC_MATCH_PARAM_SOURCE, + verbose=True, + timing="detailed", + ) + err = capsys.readouterr().err + assert "replacing" in err + assert "[" in err # timing suffix on replacement line + + +def test_dup_veto_detailed_timing_suffix(monkeypatch, capsys): + """timing='detailed' prints timing suffix after dup-group veto result.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.return_value = _make_veto_response( + False, "different logic" + ) + DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + verbose=True, + timing="detailed", + ) + err = capsys.readouterr().err + assert "VETOED" in err + assert "[" in err # timing suffix present + + +def test_verify_detailed_timing_suffix(monkeypatch, capsys): + """timing='detailed' prints timing suffix after verify result.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + verbose=True, + timing="detailed", + ) + err = capsys.readouterr().err + assert "verify ACCEPTED" in err + assert "[" in err # timing suffix present + + +def test_extraction_detailed_timing_message(monkeypatch, capsys): + """timing='detailed' prints extraction timing message after extraction call.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + DuplicateExtractor( + _DUP_RANGES, + source=_DUP_SOURCE, + verbose=True, + timing="detailed", + ) + err = capsys.readouterr().err + assert "→ extraction [" in err diff --git a/tests/duplicate_extractor/test_unused_assignments.py b/tests/duplicate_extractor/test_unused_assignments.py new file mode 100644 index 0000000..373a5f5 --- /dev/null +++ b/tests/duplicate_extractor/test_unused_assignments.py @@ -0,0 +1,374 @@ +from unittest.mock import MagicMock, patch +import textwrap +from crispen.refactors.duplicate_extractor import ( + DuplicateExtractor, + _pyflakes_strip_unused_simple_assigns, + _replace_unused_in_target, + _strip_unused_call_assignments, +) +from .test_extractor_core import ( + _make_extract_response, + _make_verify_response, + _make_veto_response, +) + + +def test_pyflakes_strip_unused_simple_assigns_removes_literal_init(): + # last_import_line = 0 becomes unused after extraction. + source = textwrap.dedent( + """\ + def foo(source): + last_import_line = 0 + lines = source.splitlines() + return lines + """ + ) + result = _pyflakes_strip_unused_simple_assigns(source, {"last_import_line"}) + assert "last_import_line" not in result + assert "lines = source.splitlines()" in result + + +def test_pyflakes_strip_unused_simple_assigns_keeps_call_rhs(): + # x = func() must NOT be stripped — it has side effects. + source = textwrap.dedent( + """\ + def foo(): + x = side_effect() + return 1 + """ + ) + result = _pyflakes_strip_unused_simple_assigns(source, {"x"}) + assert "x = side_effect()" in result + + +def test_pyflakes_strip_unused_simple_assigns_no_change_when_used(): + source = textwrap.dedent( + """\ + def foo(source): + last_import_line = 0 + for line in source.splitlines(): + last_import_line += 1 + return last_import_line + """ + ) + result = _pyflakes_strip_unused_simple_assigns(source, {"last_import_line"}) + assert result == source + + +def test_pyflakes_strip_unused_simple_assigns_fallback_on_empty_block(): + # If stripping would leave a block with no statements (syntax error), + # the original source is returned unchanged. + source = textwrap.dedent( + """\ + def foo(): + x = 0 + """ + ) + # After stripping x = 0 the function body is empty — SyntaxError. + result = _pyflakes_strip_unused_simple_assigns(source, {"x"}) + assert result == source + + +def test_pyflakes_strip_unused_simple_assigns_module_level_unchanged(): + # Module-level assignments are not flagged as UnusedVariable by pyflakes. + source = "x = 0\n" + result = _pyflakes_strip_unused_simple_assigns(source, {"x"}) + assert result == source + + +def test_pyflakes_strip_unused_simple_assigns_skips_unrelated_names(): + # A variable unused after extraction but NOT in allowed_names is preserved. + source = textwrap.dedent( + """\ + def foo(source): + unrelated = 0 + lines = source.splitlines() + return lines + """ + ) + # "unrelated" is not in the allowed set → must not be removed. + result = _pyflakes_strip_unused_simple_assigns(source, {"last_import_line"}) + assert "unrelated = 0" in result + + +def test_pyflakes_strip_unused_simple_assigns_empty_allowed(): + # Empty allowed_names means nothing can be stripped. + source = textwrap.dedent( + """\ + def foo(source): + x = 0 + lines = source.splitlines() + return lines + """ + ) + result = _pyflakes_strip_unused_simple_assigns(source, set()) + assert result == source + + +def test_replace_unused_in_target_name_used(): + import ast + + target = ast.parse("result = 1").body[0].targets[0] + new_t, all_r, any_r = _replace_unused_in_target(target, "print(result)\n") + assert all_r is False and any_r is False + assert ast.unparse(new_t) == "result" + + +def test_replace_unused_in_target_name_unused(): + import ast + + target = ast.parse("result = 1").body[0].targets[0] + new_t, all_r, any_r = _replace_unused_in_target(target, "return None\n") + assert all_r is True and any_r is True + assert ast.unparse(new_t) == "_" + + +def test_replace_unused_in_target_tuple_all_unused(): + import ast + + target = ast.parse("a, b = 1").body[0].targets[0] + new_t, all_r, any_r = _replace_unused_in_target(target, "return None\n") + assert all_r is True and any_r is True + assert ast.unparse(new_t) == "(_, _)" + + +def test_replace_unused_in_target_tuple_some_unused(): + import ast + + target = ast.parse("a, b = 1").body[0].targets[0] + new_t, all_r, any_r = _replace_unused_in_target(target, "print(a)\n") + assert all_r is False and any_r is True + assert ast.unparse(new_t) == "(a, _)" + + +def test_replace_unused_in_target_tuple_all_used(): + import ast + + target = ast.parse("a, b = 1").body[0].targets[0] + new_t, all_r, any_r = _replace_unused_in_target(target, "print(a, b)\n") + assert all_r is False and any_r is False + + +def test_replace_unused_in_target_attribute_treated_as_used(): + import ast + + target = ast.parse("self.x = 1").body[0].targets[0] + new_t, all_r, any_r = _replace_unused_in_target(target, "return None\n") + assert all_r is False and any_r is False + + +def test_strip_unused_call_assignments_removes_unused_single(): + # `result` never appears after the block → assignment stripped. + replacement = " result = _helper(x, y)\n" + following = [" do_something()\n", " return z\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == " _helper(x, y)\n" + + +def test_strip_unused_call_assignments_keeps_used_single(): + # `result` is referenced after the block → assignment kept. + replacement = " result = _helper(x, y)\n" + following = [" print(result)\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == replacement + + +def test_strip_unused_call_assignments_removes_unused_tuple(): + # Both names unused after the block → assignment stripped entirely. + replacement = " a, b = _helper(x)\n" + following = [" return None\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == " _helper(x)\n" + + +def test_strip_unused_call_assignments_partial_tuple_replaces_with_underscore(): + # One name used, one unused → replace unused with _. + replacement = " a, b = _helper(x)\n" + following = [" print(a)\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == " (a, _) = _helper(x)\n" + + +def test_strip_unused_call_assignments_attribute_target_unchanged(): + # Target is an attribute (self.x = call()) → treated as used → left unchanged. + replacement = " self.result = _helper(x)\n" + following = [] + out = _strip_unused_call_assignments(replacement, following) + assert out == replacement + + +def test_strip_unused_call_assignments_non_call_rhs_unchanged(): + # RHS is not a Call → leave unchanged. + replacement = " result = x + y\n" + following = [" return None\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == replacement + + +def test_strip_unused_call_assignments_chained_all_unused_stripped(): + # Chained assignment where every name is unused → stripped to just the call. + replacement = " a = b = _helper(x)\n" + following = [" return None\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == " _helper(x)\n" + + +def test_strip_unused_call_assignments_chained_some_used_unchanged(): + # Chained assignment where one name is used → left unchanged. + replacement = " a = b = _helper(x)\n" + following = [" print(a)\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == replacement + + +def test_strip_unused_call_assignments_chained_no_names_unchanged(): + # Chained assignment whose targets yield no names (e.g. attributes) → unchanged. + replacement = " self.a = self.b = _helper(x)\n" + following = [] + out = _strip_unused_call_assignments(replacement, following) + assert out == replacement + + +def test_strip_unused_call_assignments_no_assignment_unchanged(): + # Replacement is already just a call → returned as-is. + replacement = " _helper(x, y)\n" + following = [" return None\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == replacement + + +def test_strip_unused_call_assignments_syntax_error_unchanged(): + # Unparseable replacement → returned unchanged. + replacement = " def (\n" + following = [] + out = _strip_unused_call_assignments(replacement, following) + assert out == replacement + + +def test_strip_unused_call_assignments_multiline_replacement(): + # Multi-statement replacement: only the unused assignment is stripped. + replacement = " result = _helper(x)\n do_other()\n" + following = [" return None\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == " _helper(x)\n do_other()\n" + + +def test_strip_unused_call_assignments_preserves_indentation(): + # Indentation of stripped replacement matches original block indent. + replacement = " result = _helper(x)\n" + following = [] + out = _strip_unused_call_assignments(replacement, following) + assert out == " _helper(x)\n" + + +def test_strip_unused_call_assignments_leading_blank_line(): + # Replacement with a blank leading line: indent is read from first content line. + replacement = "\n result = _helper(x)\n" + following = [] + out = _strip_unused_call_assignments(replacement, following) + assert out == "\n _helper(x)\n" + + +def test_strip_unused_call_assignments_await_unused_stripped(): + # `result = await _helper(x)` and `result` never used → strip assignment. + replacement = " result = await _helper(x)\n" + following = [" return None\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == " await _helper(x)\n" + + +def test_strip_unused_call_assignments_await_used_kept(): + # `result = await _helper(x)` and `result` is used → keep assignment. + replacement = " result = await _helper(x)\n" + following = [" print(result)\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == replacement + + +def test_strip_unused_call_assignments_await_tuple_unused_stripped(): + # `a, b = await _helper(x)` and neither name is used → strip assignment. + replacement = " a, b = await _helper(x)\n" + following = [" return None\n"] + out = _strip_unused_call_assignments(replacement, following) + assert out == " await _helper(x)\n" + + +def test_strip_unused_call_assignments_await_non_call_unchanged(): + # `result = await some_awaitable` (not a call) → left unchanged. + replacement = " result = await some_awaitable\n" + following = [] + out = _strip_unused_call_assignments(replacement, following) + assert out == replacement + + +def test_restrip_drops_assignment_unused_only_after_all_call_sites_replaced( + monkeypatch, +): + # Regression: when two call sites reference the same variable name, the + # per-call-site strip (which uses original following lines) sees the name + # in the other call site's original block and keeps the assignment. After + # all replacements are assembled the variable is truly unused, so the + # re-strip pass must drop it. + # + # Source: test_f has two identical 2-line blocks. + # LLM returns: + # - call site 1 replacement: ``data = assert_error(result)`` + # - call site 2 replacement: ``assert_error(result2)`` (no assignment) + # After initial per-call-site strip, call site 1 keeps the assignment + # because "data" appears in the original following source (inside call + # site 2's original block). The re-strip must then drop it. + # Using function parameters avoids the SequenceCollector merging the + # assignment lines into the duplicate block. + # Use 3-statement blocks (weight=3 ≥ min_weight) so the SequenceCollector + # finds the duplicate group. Mirroring the real lever-mcp pattern: + # json.loads + two asserts. Both result and result2 are function + # parameters so the SequenceCollector cannot absorb the assignment lines + # into the duplicate block. + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + source = textwrap.dedent( + """\ + def test_f(result, result2): + rd = json.loads(result) + assert rd["value"] is None + assert "error" in rd + rd = json.loads(result2) + assert rd["value"] is None + assert "error" in rd + """ + ) + helper = textwrap.dedent( + """\ + def assert_error_result(result): + rd = json.loads(result) + assert rd["value"] is None + assert "error" in rd + """ + ) + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "identical blocks"), + _make_extract_response( + { + "function_name": "assert_error_result", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + # LLM assigns the return value at call site 1 … + " rd = assert_error_result(result)\n", + # … but not at call site 2 (helper returns None). + " assert_error_result(result2)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor([(2, 4), (5, 7)], source=source) + + assert de._new_source is not None + # The re-strip must have dropped the unused assignment at call site 1. + assert "rd = assert_error_result(result)" not in de._new_source + assert "assert_error_result(result)" in de._new_source + assert "assert_error_result(result2)" in de._new_source diff --git a/tests/duplicate_extractor/test_veto_notes_retry.py b/tests/duplicate_extractor/test_veto_notes_retry.py new file mode 100644 index 0000000..8ab5b16 --- /dev/null +++ b/tests/duplicate_extractor/test_veto_notes_retry.py @@ -0,0 +1,312 @@ +from unittest.mock import MagicMock, patch +from crispen.refactors.duplicate_extractor import DuplicateExtractor, _ApiTimeout +from .test_extractor_core import ( + _DUP_RANGES, + _DUP_SOURCE, + _make_extract_response, + _make_verify_response, + _make_veto_response, +) + + +def _make_veto_response_with_notes( + is_valid: bool, reason: str, notes: str +) -> MagicMock: + block = MagicMock() + block.type = "tool_use" + block.name = "evaluate_duplicate" + block.input = { + "is_valid_duplicate": is_valid, + "reason": reason, + "extraction_notes": notes, + } + resp = MagicMock() + resp.content = [block] + return resp + + +def test_veto_notes_passed_to_extract(monkeypatch): + """extraction_notes from veto are forwarded to the extract prompt.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response_with_notes(True, "same logic", "watch out for x"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) + + assert de._new_source is not None + extract_call = mock_client.messages.create.call_args_list[1] + extract_prompt = extract_call.kwargs["messages"][0]["content"] + assert "watch out for x" in extract_prompt + + +def test_extraction_retry_on_alg_failure_verbose(monkeypatch, capsys): + """First extract has wrong call count -> retry -> second succeeds. verbose=True.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [" _helper(data)\n"], # wrong count + } + ), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor( + _DUP_RANGES, source=_DUP_SOURCE, verbose=True, extraction_retries=1 + ) + + assert de._new_source is not None + err = capsys.readouterr().err + assert "retrying" in err + + +def test_extraction_retry_on_alg_failure_silent(monkeypatch): + """First extract has wrong call count -> retry -> second succeeds. verbose=False.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [" _helper(data)\n"], # wrong count + } + ), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor( + _DUP_RANGES, source=_DUP_SOURCE, verbose=False, extraction_retries=1 + ) + + assert de._new_source is not None + + +def test_llm_verify_timeout_verbose(monkeypatch, capsys): + """Verify times out (verbose=True) -> extraction is accepted and logged.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + from crispen.refactors.duplicate_extractor import _llm_verify_extraction + + extraction_dict = { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(data):\n pass\n", + "call_site_replacements": [" _helper(data)\n", " _helper(data)\n"], + } + side_effects: list = [(True, "same logic", ""), extraction_dict] + + def _mock_run(func, timeout, *args, **kwargs): + if func is _llm_verify_extraction: + raise _ApiTimeout("verify timed out") + return side_effects.pop(0) + + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + side_effect=_mock_run, + ), + ): + de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, verbose=True) + + assert de._new_source is not None + err = capsys.readouterr().err + assert "verify timed out" in err + + +def test_llm_verify_rejects_then_retries_verbose(monkeypatch, capsys): + """Verify rejects first attempt; retry extract passes. verbose=True.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(False, ["wrong variable name"]), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor( + _DUP_RANGES, source=_DUP_SOURCE, verbose=True, llm_verify_retries=1 + ) + + assert de._new_source is not None + err = capsys.readouterr().err + assert "REJECTED" in err + assert "wrong variable name" in err + assert "retrying" in err + + +def test_llm_verify_rejects_then_retries_silent(monkeypatch): + """Verify rejects first attempt; retry extract passes. verbose=False.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(False, ["wrong variable name"]), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(True, []), + ] + de = DuplicateExtractor( + _DUP_RANGES, source=_DUP_SOURCE, verbose=False, llm_verify_retries=1 + ) + + assert de._new_source is not None + + +def test_llm_verify_exhausted_skips_group(monkeypatch): + """All verify attempts fail -> group skipped.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + helper = "def _helper(data):\n pass\n" + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True, "same logic"), + _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": helper, + "call_site_replacements": [ + " _helper(data)\n", + " _helper(data)\n", + ], + } + ), + _make_verify_response(False, ["issue"]), + ] + de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, llm_verify_retries=0) + + assert de._new_source is None + + +def test_llm_verify_timeout_silent(monkeypatch): + """Verify times out (verbose=False) -> extraction is accepted silently.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + from crispen.refactors.duplicate_extractor import _llm_verify_extraction + + extraction_dict = { + "function_name": "_helper", + "placement": "module_level", + "helper_source": "def _helper(data):\n pass\n", + "call_site_replacements": [" _helper(data)\n", " _helper(data)\n"], + } + side_effects: list = [(True, "same logic", ""), extraction_dict] + + def _mock_run(func, timeout, *args, **kwargs): + if func is _llm_verify_extraction: + raise _ApiTimeout("verify timed out") + return side_effects.pop(0) + + with ( + patch("crispen.llm_client.anthropic.Anthropic"), + patch( + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", + side_effect=_mock_run, + ), + ): + de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, verbose=False) + + assert de._new_source is not None diff --git a/tests/duplicate_extractor/test_yield_return_guards.py b/tests/duplicate_extractor/test_yield_return_guards.py new file mode 100644 index 0000000..d6bb300 --- /dev/null +++ b/tests/duplicate_extractor/test_yield_return_guards.py @@ -0,0 +1,190 @@ +from unittest.mock import MagicMock, patch +import textwrap +from crispen.refactors.duplicate_extractor import ( + DuplicateExtractor, + _replacement_contains_return, + _seq_ends_with_return, + _seq_source_contains_yield, +) +from .test_edit_operations import _make_seq_with_source +from .test_collectors import _collect_sequences +from .test_extractor_core import _make_extract_response, _make_veto_response + + +def test_seq_ends_with_return_true(): + assert ( + _seq_ends_with_return(_make_seq_with_source(" x = 1\n return x\n")) + is True + ) + + +def test_seq_ends_with_return_false_no_return(): + assert ( + _seq_ends_with_return(_make_seq_with_source(" x = 1\n y = 2\n")) is False + ) + + +def test_seq_ends_with_return_syntax_error(): + assert _seq_ends_with_return(_make_seq_with_source(" (\n")) is False + + +def test_seq_ends_with_return_empty_body(): + # Pure whitespace → ast.parse produces an empty module body. + assert _seq_ends_with_return(_make_seq_with_source(" \n")) is False + + +def test_seq_ends_with_return_bare_return(): + # Bare `return` is equivalent to returning None — not flagged. + assert ( + _seq_ends_with_return(_make_seq_with_source(" x = 1\n return\n")) is False + ) + + +def test_seq_ends_with_return_return_none(): + # Explicit `return None` is also equivalent to implicit None — not flagged. + assert ( + _seq_ends_with_return(_make_seq_with_source(" x = 1\n return None\n")) + is False + ) + + +def test_seq_source_contains_yield_async_with_yield(): + # The exact pattern that triggered the bug: async with ... as c: yield c + src = " async with Client(mcp) as c:\n yield c\n" + assert _seq_source_contains_yield(src) is True + + +def test_seq_source_contains_yield_plain_yield(): + assert _seq_source_contains_yield(" yield x\n") is True + + +def test_seq_source_contains_yield_from(): + assert _seq_source_contains_yield(" yield from something()\n") is True + + +def test_seq_source_contains_yield_no_yield(): + assert _seq_source_contains_yield(" x = 1\n y = 2\n") is False + + +def test_seq_source_contains_yield_nested_funcdef_not_counted(): + # yield inside a nested def must NOT trigger the guard + src = " def inner():\n yield 1\n" + assert _seq_source_contains_yield(src) is False + + +def test_seq_source_contains_yield_syntax_error(): + assert _seq_source_contains_yield(" (\n") is False + + +def test_collector_skips_yield_sequences(): + # Sequences whose source contains yield should never be collected. + source = textwrap.dedent( + """\ + async def make_client(): + x = setup() + async with Client(x) as c: + yield c + + async def make_client2(): + x = setup() + async with Client(x) as c: + yield c + """ + ) + seqs = _collect_sequences(source) + for seq in seqs: + assert not _seq_source_contains_yield(seq.source) + + +def test_replacement_contains_return_true(): + assert _replacement_contains_return(" return x\n") is True + + +def test_replacement_contains_return_false(): + assert _replacement_contains_return(" _helper()\n") is False + + +def test_replacement_contains_return_syntax_error(): + # Unclosed paren inside the wrapper → SyntaxError → False. + assert _replacement_contains_return(" (\n") is False + + +_RETURN_BLOCK_SOURCE = textwrap.dedent( + """\ + def foo(): + if debug: + pass + x = compute(data) + y = transform(x) + return y + + def bar(): + result = None + x = compute(data) + y = transform(x) + return y + """ +) +_RETURN_BLOCK_RANGES = [(10, 12)] # overlaps bar's duplicate block (x/y/return lines) + + +def _make_return_block_extract_response(): + return _make_extract_response( + { + "function_name": "_helper", + "placement": "module_level", + "helper_source": ( + "def _helper():\n" + " x = compute(data)\n" + " y = transform(x)\n" + " return y\n" + ), + # replacement drops the return — this is the bug being guarded + "call_site_replacements": [ + " _helper()\n", + " _helper()\n", + ], + } + ) + + +def test_block_ends_with_return_guard_skips(monkeypatch, capsys): + """Extraction rejected when block ends with return but replacement omits it.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_return_block_extract_response(), + ] + de = DuplicateExtractor( + _RETURN_BLOCK_RANGES, + source=_RETURN_BLOCK_SOURCE, + extraction_retries=0, + llm_verify_retries=0, + ) + assert de._new_source is None + assert "block ends with return but replacement omits it" in capsys.readouterr().err + + +def test_block_ends_with_return_guard_skips_silent(monkeypatch): + """verbose=False: extraction rejected with no stderr output.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + with patch("crispen.llm_client.anthropic") as mock_anthropic: + mock_client = MagicMock() + mock_anthropic.Anthropic.return_value = mock_client + mock_anthropic.APIError = Exception + mock_client.messages.create.side_effect = [ + _make_veto_response(True), + _make_return_block_extract_response(), + ] + de = DuplicateExtractor( + _RETURN_BLOCK_RANGES, + source=_RETURN_BLOCK_SOURCE, + verbose=False, + extraction_retries=0, + llm_verify_retries=0, + ) + assert de._new_source is None diff --git a/tests/engine/__init__.py b/tests/engine/__init__.py new file mode 100644 index 0000000..a8ba749 --- /dev/null +++ b/tests/engine/__init__.py @@ -0,0 +1 @@ +"""Tests for the engine module.""" diff --git a/tests/engine/core/__init__.py b/tests/engine/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/engine/core/test_engine_run.py b/tests/engine/core/test_engine_run.py new file mode 100644 index 0000000..1f55e9e --- /dev/null +++ b/tests/engine/core/test_engine_run.py @@ -0,0 +1,382 @@ +from unittest.mock import patch +import textwrap +from crispen.config import CrispenConfig +from crispen.engine import ( + _LLM_REFACTOR_KEYS, + _categorize_into_stats, + _should_run, + run_engine, +) +from crispen.errors import CrispenAPIError +from crispen.file_limiter.runner import FileLimiterResult +from crispen.refactors.base import Refactor +from crispen.stats import RunStats +import pytest +from ..helpers import _CrispenApiErrorRefactor, _FL_PATCH, _RaisingTransformer, _run + + +def test_config_header_printed_when_llm_refactors_enabled(tmp_path, capsys): + f = tmp_path / "simple.py" + f.write_text("x = 1\n", encoding="utf-8") + list(run_engine({str(f): [(1, 1)]}, config=CrispenConfig())) + err = capsys.readouterr().err + assert "--- crispen ---" in err + assert "provider:" in err + assert "model:" in err + + +def test_config_header_suppressed_when_all_llm_refactors_disabled(tmp_path, capsys): + f = tmp_path / "simple.py" + f.write_text("x = 1\n", encoding="utf-8") + cfg = CrispenConfig(disabled_refactors=list(_LLM_REFACTOR_KEYS)) + list(run_engine({str(f): [(1, 1)]}, config=cfg)) + assert "--- crispen ---" not in capsys.readouterr().err + + +def test_config_header_suppressed_when_changed_empty(capsys): + list(run_engine({}, config=CrispenConfig())) + assert "--- crispen ---" not in capsys.readouterr().err + + +def test_skip_missing_file(tmp_path): + missing = str(tmp_path / "nonexistent.py") + msgs = _run({missing: [(1, 10)]}) + assert len(msgs) == 1 + assert "SKIP" in msgs[0] + assert "file not found" in msgs[0] + + +def test_no_changes_no_messages(tmp_path): + f = tmp_path / "simple.py" + f.write_text("x = 1\n", encoding="utf-8") + msgs = _run({str(f): [(1, 1)]}) + assert msgs == [] + + +def test_applies_refactor_and_writes(tmp_path): + source = textwrap.dedent( + """\ + if not x: + a() + else: + b() + """ + ) + f = tmp_path / "code.py" + f.write_text(source, encoding="utf-8") + msgs = _run({str(f): [(1, 4)]}) + assert any("IfNotElse" in m for m in msgs) + assert "if x:" in f.read_text(encoding="utf-8") + + +def test_rewritten_source_used_when_available(tmp_path): + """get_rewritten_source() is preferred over new_tree.code when non-None.""" + rewritten = "x = 999 # rewritten\n" + + class _RewritingRefactor(Refactor): + @classmethod + def name(cls): + return "Rewriter" + + def get_rewritten_source(self): + return rewritten + + def get_changes(self): + return ["Rewriter: rewrote the file"] + + f = tmp_path / "code.py" + f.write_text("x = 1\n", encoding="utf-8") + with patch("crispen.engine._REFACTORS", [_RewritingRefactor]): + msgs = _run({str(f): [(1, 1)]}) + assert any("Rewriter" in m for m in msgs) + assert f.read_text(encoding="utf-8") == rewritten + + +def test_skip_parse_error(tmp_path): + f = tmp_path / "bad.py" + f.write_text("def f(:\n pass\n", encoding="utf-8") + msgs = _run({str(f): [(1, 2)]}) + assert any("parse error" in m for m in msgs) + + +def test_skip_transform_error(tmp_path): + f = tmp_path / "code.py" + f.write_text("x = 1\n", encoding="utf-8") + with patch("crispen.engine._REFACTORS", [_RaisingTransformer]): + msgs = _run({str(f): [(1, 1)]}) + assert any("transform error" in m for m in msgs) + + +def test_crispen_api_error_propagates(tmp_path): + f = tmp_path / "code.py" + f.write_text("x = 1\n", encoding="utf-8") + with patch("crispen.engine._REFACTORS", [_CrispenApiErrorRefactor]): + with pytest.raises(CrispenAPIError): + list(run_engine({str(f): [(1, 1)]})) + + +def test_tuple_dataclass_transform_error_handled(tmp_path): + f = tmp_path / "code.py" + f.write_text("x = 1\n", encoding="utf-8") + + class _FailingTD: + def __init__(self, *a, **kw): + raise RuntimeError("simulated TupleDataclass failure") + + with patch("crispen.engine.helpers.TupleDataclass", _FailingTD): + msgs = _run({str(f): [(1, 1)]}) + assert any("TupleDataclass" in m and "transform error" in m for m in msgs) + + +def test_run_engine_accepts_explicit_config(tmp_path): + """run_engine works when config is provided explicitly.""" + f = tmp_path / "code.py" + f.write_text("x = 1\n", encoding="utf-8") + config = CrispenConfig() + msgs = list(run_engine({str(f): [(1, 1)]}, config=config)) + assert msgs == [] + + +def test_run_engine_config_none_loads_default(tmp_path): + """run_engine with config=None (default) loads config from disk.""" + f = tmp_path / "code.py" + f.write_text("x = 1\n", encoding="utf-8") + # config=None triggers load_config() internally + msgs = list(run_engine({str(f): [(1, 1)]}, config=None)) + assert msgs == [] + + +def test_categorize_if_not_else(): + s = RunStats() + _categorize_into_stats(s, "IfNotElse: flipped if/else at line 3") + assert s.if_not_else == 1 + assert s.total_edits == 1 + + +def test_categorize_tuple_to_dataclass(): + s = RunStats() + _categorize_into_stats( + s, "TupleDataclass: replaced 3-tuple with FooResult at line 5" + ) + assert s.tuple_to_dataclass == 1 + + +def test_categorize_duplicate_matched(): + s = RunStats() + _categorize_into_stats(s, "DuplicateExtractor: replaced '_f' body with call to 'g'") + assert s.duplicate_matched == 1 + assert s.duplicate_extracted == 0 + + +def test_categorize_duplicate_extracted(): + s = RunStats() + _categorize_into_stats( + s, "DuplicateExtractor: extracted '_helper' from 2 duplicate blocks" + ) + assert s.duplicate_extracted == 1 + assert s.duplicate_matched == 0 + + +def test_categorize_function_split(): + s = RunStats() + _categorize_into_stats(s, "split 'big_func': extracted _step_two") + assert s.function_split == 1 + + +def test_categorize_other_message_ignored(): + s = RunStats() + _categorize_into_stats(s, "CallerUpdater: expanded FooResult unpacking at line 7") + assert s.total_edits == 0 + + +def test_run_engine_stats_populated(tmp_path): + source = "if not x:\n a()\nelse:\n b()\n" + f = tmp_path / "code.py" + f.write_text(source, encoding="utf-8") + s = RunStats() + list(run_engine({str(f): [(1, 4)]}, config=CrispenConfig(), stats=s)) + assert s.if_not_else == 1 + assert s.files_edited == [str(f)] + assert s.lines_added + s.lines_deleted > 0 + + +def test_run_engine_stats_none_default(tmp_path): + """When stats is None (default), engine runs without error.""" + f = tmp_path / "code.py" + f.write_text("x = 1\n", encoding="utf-8") + msgs = list(run_engine({str(f): [(1, 1)]}, config=CrispenConfig())) + assert msgs == [] + + +def test_should_run_defaults_allow_all(): + cfg = CrispenConfig() + for name in ( + "if_not_else", + "duplicate_extractor", + "function_splitter", + "tuple_dataclass", + "file_limiter", + ): + assert _should_run(name, cfg) is True + + +def test_should_run_enabled_list_allows_listed(): + cfg = CrispenConfig(enabled_refactors=["if_not_else", "function_splitter"]) + assert _should_run("if_not_else", cfg) is True + assert _should_run("function_splitter", cfg) is True + + +def test_should_run_enabled_list_blocks_unlisted(): + cfg = CrispenConfig(enabled_refactors=["if_not_else"]) + assert _should_run("duplicate_extractor", cfg) is False + assert _should_run("tuple_dataclass", cfg) is False + assert _should_run("file_limiter", cfg) is False + + +def test_should_run_disabled_list_blocks_listed(): + cfg = CrispenConfig(disabled_refactors=["function_splitter", "file_limiter"]) + assert _should_run("function_splitter", cfg) is False + assert _should_run("file_limiter", cfg) is False + + +def test_should_run_enabled_takes_precedence_over_disabled(): + # enabled_refactors non-empty → disabled_refactors is ignored + cfg = CrispenConfig( + enabled_refactors=["if_not_else"], + disabled_refactors=["if_not_else"], + ) + assert _should_run("if_not_else", cfg) is True + + +def test_engine_disabled_refactors_skips_if_not_else(tmp_path): + """With if_not_else disabled the pattern is left unchanged.""" + source = textwrap.dedent( + """\ + if not x: + a() + else: + b() + """ + ) + f = tmp_path / "code.py" + f.write_text(source, encoding="utf-8") + msgs = list( + run_engine( + {str(f): [(1, 4)]}, + config=CrispenConfig(disabled_refactors=["if_not_else"]), + ) + ) + assert not any("IfNotElse" in m for m in msgs) + assert f.read_text(encoding="utf-8") == source + + +def test_engine_enabled_refactors_runs_only_listed(tmp_path): + """enabled_refactors=["if_not_else"] — other refactors don't touch the file.""" + source = textwrap.dedent( + """\ + if not x: + a() + else: + b() + """ + ) + f = tmp_path / "code.py" + f.write_text(source, encoding="utf-8") + + called = [] + + class _Spy(Refactor): + @classmethod + def name(cls): + return "Spy" + + def get_changes(self): + called.append("Spy") + return [] + + with patch("crispen.engine._REFACTORS", [_Spy]): + with patch("crispen.engine._REFACTOR_KEY", {_Spy: "spy"}): + list( + run_engine( + {str(f): [(1, 4)]}, + config=CrispenConfig(enabled_refactors=["if_not_else"]), + ) + ) + + # _Spy is not in enabled_refactors, so it must not have been called. + assert called == [] + + +def test_engine_file_limiter_skipped_when_disabled(tmp_path): + """file_limiter in disabled_refactors prevents FileLimiter from running.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + success_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"utils.py": "# new\n"}, + messages=["FileLimiter: moved"], + abort=False, + ) + with patch(_FL_PATCH, return_value=success_result) as mock_fl: + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig( + max_file_lines=5, + disabled_refactors=["file_limiter"], + ), + ) + ) + mock_fl.assert_not_called() + + +def test_engine_match_function_disabled_passes_flag_to_duplicate_extractor(tmp_path): + """disabled_refactors=["match_function"] passes match_functions=False to DE.""" + f = tmp_path / "code.py" + f.write_text("x = 1\n", encoding="utf-8") + + constructed_with: dict = {} + + original_init = __import__( + "crispen.refactors.duplicate_extractor", fromlist=["DuplicateExtractor"] + ).DuplicateExtractor.__init__ + + def _spy_init(self, *args, **kwargs): + constructed_with.update(kwargs) + original_init(self, *args, **kwargs) + + with patch("crispen.engine.DuplicateExtractor.__init__", side_effect=_spy_init): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(disabled_refactors=["match_function"]), + ) + ) + + assert constructed_with.get("match_functions") is False + + +def test_engine_match_function_enabled_by_default(tmp_path): + """Without any filter, match_functions=True is passed to DuplicateExtractor.""" + f = tmp_path / "code.py" + f.write_text("x = 1\n", encoding="utf-8") + + constructed_with: dict = {} + + original_init = __import__( + "crispen.refactors.duplicate_extractor", fromlist=["DuplicateExtractor"] + ).DuplicateExtractor.__init__ + + def _spy_init(self, *args, **kwargs): + constructed_with.update(kwargs) + original_init(self, *args, **kwargs) + + with patch("crispen.engine.DuplicateExtractor.__init__", side_effect=_spy_init): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(), + ) + ) + + assert constructed_with.get("match_functions") is True diff --git a/tests/engine/core/test_file_limiter.py b/tests/engine/core/test_file_limiter.py new file mode 100644 index 0000000..4eb24a3 --- /dev/null +++ b/tests/engine/core/test_file_limiter.py @@ -0,0 +1,157 @@ +from unittest.mock import patch +from crispen.config import CrispenConfig +from crispen.engine import run_engine +from crispen.file_limiter.runner import FileLimiterResult +from crispen.stats import RunStats +from ..helpers import _FL_PATCH + + +def test_file_limiter_empty_original_source_deletes_file(tmp_path): + """FileLimiter returns empty original_source → original file is deleted.""" + f = tmp_path / "big.py" + original = "".join(f"var_{i} = {i}\n" for i in range(10)) + f.write_text(original, encoding="utf-8") + # All content was moved out; original_source is empty (all entities migrated). + # new_files content is kept short (≤ max_file_lines) so it doesn't re-enter + # the recursive queue (file_limiter_recursive defaults to True). + moved_source = "# moved content\n" + drained_result = FileLimiterResult( + original_source="", + new_files={"utils.py": moved_source}, + messages=[f"{f}: FileLimiter: moved all → utils.py"], + abort=False, + ) + with patch(_FL_PATCH, return_value=drained_result): + msgs = list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + assert any("FileLimiter" in m for m in msgs) + # Original file must be deleted, not left as a blank file. + assert not f.exists() + # New file must exist with the moved content. + assert (tmp_path / "utils.py").exists() + + +def test_file_limiter_recursive_empty_original_source_deletes_file(tmp_path): + """Recursive FileLimiter with empty original_source deletes the recursive file.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) + # chunk_a content is short so it doesn't re-enter the recursive queue. + small = "# chunk_a content\n" + first_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"chunk.py": oversized}, + messages=[], + abort=False, + ) + # Recursive call drains chunk.py entirely; original_source is empty. + second_result = FileLimiterResult( + original_source="", + new_files={"chunk_a.py": small}, + messages=[], + abort=False, + ) + + with patch(_FL_PATCH, side_effect=[first_result, second_result]): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), + ) + ) + + # chunk.py was drained and must be deleted. + assert not (tmp_path / "chunk.py").exists() + # New file from the recursive split must exist. + assert (tmp_path / "chunk_a.py").exists() + + +def test_file_limiter_empty_init_py_preserved(tmp_path): + """__init__.py is never deleted even when FileLimiter drains it to empty.""" + f = tmp_path / "__init__.py" + original = "".join(f"def func_{i}():\n pass\n\n" for i in range(10)) + f.write_text(original, encoding="utf-8") + drained = FileLimiterResult( + original_source="", + new_files={"utils.py": "# moved\n"}, + messages=[f"{f}: FileLimiter: moved all → utils.py"], + abort=False, + ) + with patch(_FL_PATCH, return_value=drained): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + # __init__.py must still exist (empty is fine; deletion would break the package). + assert f.exists() + assert f.read_text(encoding="utf-8") == "" + + +def test_file_limiter_llm_timing_recorded_in_stats(tmp_path): + """When FileLimiterResult has llm_elapsed > 0, record_llm_call is invoked.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + timed_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"utils.py": "# new\n"}, + messages=[f"{f}: FileLimiter: moved → utils.py"], + abort=False, + llm_elapsed=1.5, + llm_input_tokens=100, + llm_output_tokens=50, + ) + stats = RunStats() + with patch(_FL_PATCH, return_value=timed_result): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + stats=stats, + ) + ) + assert "file_limiter" in stats.llm_elapsed_by_category + + +def test_file_limiter_recursive_llm_timing_recorded_in_stats(tmp_path): + """Recursive FileLimiterResult with llm_elapsed > 0 triggers record_llm_call.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + first_result = FileLimiterResult( + original_source="# reduced original\n", + new_files={"chunk.py": "".join(f"x_{i} = {i}\n" for i in range(10))}, + messages=[f"{f}: moved vars → chunk.py"], + abort=False, + ) + second_result = FileLimiterResult( + original_source="# reduced chunk\n", + new_files={"chunk_a.py": "# a\n"}, + messages=[], + abort=False, + llm_elapsed=2.0, + llm_input_tokens=200, + llm_output_tokens=80, + ) + stats = RunStats() + call_count = 0 + + def _fl_side_effect(**kwargs): + nonlocal call_count + call_count += 1 + return first_result if call_count == 1 else second_result + + with patch(_FL_PATCH, side_effect=_fl_side_effect): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), + stats=stats, + ) + ) + assert "file_limiter" in stats.llm_elapsed_by_category diff --git a/tests/engine/core/test_patch_update.py b/tests/engine/core/test_patch_update.py new file mode 100644 index 0000000..e427d6f --- /dev/null +++ b/tests/engine/core/test_patch_update.py @@ -0,0 +1,504 @@ +from unittest.mock import patch +from crispen.config import CrispenConfig +from crispen.engine import _build_patch_map, run_engine +from crispen.file_limiter.runner import FileLimiterResult +from ..helpers import _FL_PATCH + + +def test_build_patch_map_single_caller_uses_caller(tmp_path): + """Entity imported and used by exactly one new file → caller's module used.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "def MyFunc(): pass\n", + "caller.py": "from .sub import MyFunc\nMyFunc()\n", + }, + entity_to_target={"MyFunc": "sub.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg) + assert result == {"mypkg.module.MyFunc": "mypkg.caller.MyFunc"} + + +def test_build_patch_map_forking_entity_skipped(tmp_path): + """Entity used by multiple new files (forking) → skipped.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "def MyFunc(): pass\n", + "caller_a.py": "from .sub import MyFunc\nMyFunc()\n", + "caller_b.py": "from .sub import MyFunc\nMyFunc()\n", + }, + entity_to_target={"MyFunc": "sub.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg) + assert result == {} + + +def test_build_patch_map_empty_new_file_skipped(tmp_path): + """New file with empty source is skipped when building import index.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "class MyClass: pass\n", + "empty.py": "", + }, + entity_to_target={"MyClass": "sub.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg) + assert result == {"mypkg.module.MyClass": "mypkg.sub.MyClass"} + + +def test_build_patch_map_new_module_none(tmp_path): + """When target file's module path can't be resolved → entity is skipped.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + f = tmp_path / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={"utils.py": "class MyClass: pass\n"}, + entity_to_target={"MyClass": "utils.py"}, + ) + with patch( + "crispen.engine.file_limiter._module_path_for_file", + side_effect=["mypkg.module", None], + ): + result = _build_patch_map(str(f), fl_result, tmp_path) + assert result == {} + + +def test_build_patch_map_assignment_defined_in_multiple_files_skipped(tmp_path): + """Variable appearing in two new files' assignments → ambiguous → skipped.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + pre_split = "_TIMEOUT = 30\n" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "core.py": "_TIMEOUT = 30\n", + "utils.py": "_TIMEOUT = 60\n", + }, + abort=False, + entity_to_target={}, + ) + result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) + assert "mypkg.big._TIMEOUT" not in result + + +def test_build_patch_map_assignment_not_in_original_skipped(tmp_path): + """Variable introduced by code generation (not in pre_split_source) → skipped.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + pre_split = "def run(): pass\n" # _TIMEOUT not in original + fl_result = FileLimiterResult( + original_source="", + new_files={"core.py": "_TIMEOUT = 30\ndef run(): pass\n"}, + abort=False, + entity_to_target={"run": "core.py"}, + ) + result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) + assert "mypkg.big._TIMEOUT" not in result + + +def test_patch_update_no_combined_map(tmp_path): + """'update' mode but FL returned empty entity_to_target → no updates.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + other = tmp_path / "test_other.py" + other.write_text( + '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" + ) + + no_entity_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"utils.py": "class MyClass: pass\n"}, + messages=["big.py: FileLimiter: moved MyClass → utils.py"], + abort=False, + entity_to_target={}, # empty! + ) + with patch(_FL_PATCH, return_value=no_entity_result): + list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="basic", + ), + _repo_root=str(tmp_path), + ) + ) + assert ( + other.read_text(encoding="utf-8") + == '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n' + ) + + +def test_patch_update_updates_per_file_source(tmp_path): + """'update' mode, FL moved entities → per_file source with old path is updated.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + big_source = "".join(f"var_{i} = {i}\n" for i in range(10)) + f.write_text(big_source, encoding="utf-8") + # Another diff file with an old @patch string + other_diff = pkg / "test_big.py" + other_diff.write_text( + '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" + ) + + fl_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"utils.py": "class MyClass: pass\n"}, + messages=["big.py: FileLimiter: moved MyClass → utils.py"], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) + with patch(_FL_PATCH, return_value=fl_result): + msgs = list( + run_engine( + {str(f): [(1, 10)], str(other_diff): [(1, 2)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="basic", + ), + _repo_root=str(tmp_path), + ) + ) + # The per_file source for other_diff should have the updated string + updated_text = other_diff.read_text(encoding="utf-8") + assert "mypkg.utils.MyClass" in updated_text + assert any("patch_update" in m for m in msgs) + + +def test_patch_update_updates_other_file(tmp_path): + """'update' mode, a separate file outside per_file gets updated on disk.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + big_source = "".join(f"var_{i} = {i}\n" for i in range(10)) + f.write_text(big_source, encoding="utf-8") + # A file NOT in the diff that has the old @patch string + other = tmp_path / "test_other.py" + other.write_text( + '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" + ) + + fl_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"utils.py": "class MyClass: pass\n"}, + messages=["big.py: FileLimiter: moved MyClass → utils.py"], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) + with patch(_FL_PATCH, return_value=fl_result): + msgs = list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="basic", + ), + _repo_root=str(tmp_path), + ) + ) + updated_text = other.read_text(encoding="utf-8") + assert "mypkg.utils.MyClass" in updated_text + assert any("patch_update" in m for m in msgs) + + +def test_patch_update_skips_excluded_dir(tmp_path): + """Files under .venv/ are excluded from Phase 4 scanning.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + big_source = "".join(f"var_{i} = {i}\n" for i in range(10)) + f.write_text(big_source, encoding="utf-8") + venv_dir = tmp_path / ".venv" + venv_dir.mkdir() + venv_file = venv_dir / "test.py" + venv_content = '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n' + venv_file.write_text(venv_content, encoding="utf-8") + + fl_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"utils.py": "class MyClass: pass\n"}, + messages=["big.py: FileLimiter: moved MyClass → utils.py"], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) + with patch(_FL_PATCH, return_value=fl_result): + list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="basic", + ), + _repo_root=str(tmp_path), + ) + ) + # .venv/test.py must not be modified + assert venv_file.read_text(encoding="utf-8") == venv_content + + +def test_patch_update_no_repo_root(tmp_path): + """When repo_root can't be found, Phase 4 skips entirely.""" + # No .git or pyproject.toml → _find_repo_root returns None + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + other = tmp_path / "test_other.py" + other.write_text( + '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" + ) + + fl_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"utils.py": "class MyClass: pass\n"}, + messages=[], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) + with patch(_FL_PATCH, return_value=fl_result): + list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="basic", + ), + # No _repo_root passed, no .git in tmp_path → repo_root=None + ) + ) + # test_other.py should be unchanged + assert ( + other.read_text(encoding="utf-8") + == '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n' + ) + + +def test_patch_update_oserror_skipped(tmp_path): + """Phase 4 continues gracefully when read_text raises OSError.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + big_source = "".join(f"var_{i} = {i}\n" for i in range(10)) + f.write_text(big_source, encoding="utf-8") + # A file that will raise OSError when read + other = tmp_path / "test_other.py" + other.write_text( + '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" + ) + + fl_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"utils.py": "class MyClass: pass\n"}, + messages=[], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) + + other_abs = str(other.resolve()) + + original_pathlib_read = None + + def _patched_read_text(self, encoding="utf-8"): + if str(self.resolve()) == other_abs: + raise OSError("permission denied") + return original_pathlib_read(self, encoding=encoding) + + import pathlib + + original_pathlib_read = pathlib.Path.read_text + + with patch.object(pathlib.Path, "read_text", _patched_read_text): + with patch(_FL_PATCH, return_value=fl_result): + # Should not raise even though read_text raises OSError + msgs = list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="basic", + ), + _repo_root=str(tmp_path), + ) + ) + + # No patch_update message for other since it raised OSError + assert not any("test_other" in m and "patch_update" in m for m in msgs) + + +def test_patch_update_accumulates_from_recursive_fl(tmp_path): + """Entities from recursive FL results also contribute to the patch map.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + big_source = "".join(f"var_{i} = {i}\n" for i in range(10)) + f.write_text(big_source, encoding="utf-8") + + # Other file outside per_file with old @patch strings for both files + other = tmp_path / "test_other.py" + other.write_text( + '@patch("mypkg.big.MyClass")\n' + '@patch("mypkg.utils.HelperClass")\n' + "def test_it(): pass\n", + encoding="utf-8", + ) + + # First FL result: big.py → utils.py (MyClass moved there) + first_result = FileLimiterResult( + original_source="# reduced\n", + new_files={ + "utils.py": "class MyClass: pass\n" * 10 + }, # over limit for recursion + messages=[], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) + + # utils.py written by first result; set up that file + utils_path = pkg / "utils.py" + + # Second FL result: utils.py → helpers.py (HelperClass moved there) + second_result = FileLimiterResult( + original_source="# utils reduced\n", + new_files={"helpers.py": "class HelperClass: pass\n"}, + messages=[], + abort=False, + entity_to_target={"HelperClass": "helpers.py"}, + ) + + call_count = 0 + + def _fl_side_effect(**kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + # Write utils.py so the recursive call can find it + utils_path.write_text("class MyClass: pass\n" * 10, encoding="utf-8") + return first_result + return second_result + + with patch(_FL_PATCH, side_effect=_fl_side_effect): + msgs = list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_recursive=True, + file_limiter_patch_update="basic", + ), + _repo_root=str(tmp_path), + ) + ) + + # Verify combined_patch_map was non-empty by checking at least one + # patch_update message was generated (from the other file or per_file). + updated_text = other.read_text(encoding="utf-8") + # At minimum, MyClass should be updated (from first pass) + assert "mypkg.utils.MyClass" in updated_text or any( + "patch_update" in m for m in msgs + ) + + +def test_patch_update_chain_flattening(tmp_path): + """Transitive chains in combined_patch_map are flattened before apply. + + When a first split produces A→B and a recursive split produces B→C, + apply_patch_strings must map A directly to C, not to the intermediate B. + """ + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + # big.py imports get_api_key and uses it; also defines func_0 (moved entity) + big_source = "from llm import get_api_key\n" + "".join( + f"def func_{i}(): get_api_key()\n" for i in range(10) + ) + f.write_text(big_source, encoding="utf-8") + + # Other file has @patch pointing at the imported alias in big.py + other = tmp_path / "test_other.py" + other.write_text( + '@patch("mypkg.big.get_api_key")\ndef test_it(): pass\n', + encoding="utf-8", + ) + + # First FL result: big.py → utils.py. + # utils.py is over max_file_lines so it will be queued for recursive split. + # It imports and uses get_api_key so the alias ends up in utils's map entry. + utils_source = "from llm import get_api_key\n" + "".join( + f"def helper_{i}(): get_api_key()\n" for i in range(10) + ) + first_result = FileLimiterResult( + original_source="# big reduced\n", + new_files={"utils.py": utils_source}, + messages=[], + abort=False, + entity_to_target={ + "func_0": "utils.py" + }, # non-empty to trigger _build_patch_map + ) + + # Second FL result (recursive split of utils.py) → helpers.py + helpers_source = "from llm import get_api_key\n" "def helper_0(): get_api_key()\n" + second_result = FileLimiterResult( + original_source="# utils reduced\n", + new_files={"helpers.py": helpers_source}, + messages=[], + abort=False, + entity_to_target={"helper_0": "helpers.py"}, # non-empty + ) + + call_count = 0 + + def _fl_side_effect(**kwargs): + nonlocal call_count + call_count += 1 + return first_result if call_count == 1 else second_result + + with patch(_FL_PATCH, side_effect=_fl_side_effect): + list( + run_engine( + {str(f): [(1, len(big_source.splitlines()))]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_recursive=True, + file_limiter_patch_update="basic", + ), + _repo_root=str(tmp_path), + ) + ) + + # Round 1 map: mypkg.big.get_api_key → mypkg.utils.get_api_key + # Round 2 map: mypkg.utils.get_api_key → mypkg.helpers.get_api_key + # After flattening: mypkg.big.get_api_key → mypkg.helpers.get_api_key + # Without flattening the test file would still hold the intermediate path. + updated = other.read_text(encoding="utf-8") + assert ( + "mypkg.helpers.get_api_key" in updated + ), f"Expected chain-flattened path but got: {updated!r}" diff --git a/tests/engine/core/test_utils_misc.py b/tests/engine/core/test_utils_misc.py new file mode 100644 index 0000000..2d15d23 --- /dev/null +++ b/tests/engine/core/test_utils_misc.py @@ -0,0 +1,169 @@ +from unittest.mock import patch +import threading +from crispen.engine import ( + _collect_assignment_names, + _collect_code_referenced_names, + _collect_imported_names, + _collect_top_level_names, + _find_outside_callers, + _module_path_for_file, + _visit_with_timeout, +) + + +def test_visit_with_timeout_completes(): + """Fast visit completes within timeout → returns True.""" + from unittest.mock import MagicMock + + wrapper = MagicMock() + finder = MagicMock() + assert _visit_with_timeout(wrapper, finder, 5.0) is True + wrapper.visit.assert_called_once_with(finder) + + +def test_visit_with_timeout_fires(): + """Slow visit that never completes → returns False after timeout.""" + block = threading.Event() + + class _HangWrapper: + def visit(self, finder): + block.wait() # blocks until released + + result = _visit_with_timeout(_HangWrapper(), object(), 0.01) + block.set() # unblock the daemon thread for cleanup + assert result is False + + +def test_find_outside_callers_scope_analysis_timeout(tmp_path): + """When _visit_with_timeout times out, all target qnames are blocked.""" + (tmp_path / "other.py").write_text("x = 1\n") + with patch("crispen.engine.helpers._visit_with_timeout", return_value=False): + result = _find_outside_callers(str(tmp_path), {"some.func"}, set()) + assert result == {"some.func"} + + +def test_find_outside_callers_deadline_expired(tmp_path): + """Total budget already exhausted before any file is visited: all blocked.""" + (tmp_path / "other.py").write_text("x = 1\n") + # A negative timeout makes the deadline fall in the past immediately. + with patch("crispen.engine.helpers._SCOPE_ANALYSIS_TIMEOUT", -1): + result = _find_outside_callers(str(tmp_path), {"some.func"}, set()) + assert result == {"some.func"} + + +def test_module_path_for_file_returns_dotted_path(tmp_path): + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + sub = tmp_path / "tests" / "lua" + sub.mkdir(parents=True) + f = sub / "test_foo.py" + f.write_text("", encoding="utf-8") + assert _module_path_for_file(str(f)) == "tests.lua.test_foo" + + +def test_module_path_for_file_init_strips_init_segment(tmp_path): + """__init__.py resolves to the package name, not package.__init__.""" + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + pkg = tmp_path / "mypkg" / "subpkg" + pkg.mkdir(parents=True) + f = pkg / "__init__.py" + f.write_text("", encoding="utf-8") + assert _module_path_for_file(str(f)) == "mypkg.subpkg" + + +def test_collect_top_level_names_various(): + """Covers functions, classes, assignments, aug/ann assigns, imports, from-imports, + non-Name aug-assign targets, and unrecognised statement types.""" + source = ( + "import os\n" + "import libcst as cst\n" + "from pathlib import Path\n" + "from typing import List as L\n" + "from os import *\n" # star import skipped + "_CONST = 42\n" + "x: int = 1\n" + "counter += 1\n" + "a, b = 1, 2\n" # tuple target → ast.Tuple, not ast.Name + "some_obj.attr += 1\n" # AugAssign with Attribute target → skipped + "if True: pass\n" # ast.If → matches no elif, skipped + "def my_func(): pass\n" + "class MyClass: pass\n" + "async def async_func(): pass\n" + ) + result = _collect_top_level_names(source) + assert "os" in result + assert "cst" in result + assert "Path" in result + assert "L" in result + assert "_CONST" in result + assert "x" in result + assert "counter" in result + assert "my_func" in result + assert "MyClass" in result + assert "async_func" in result + # Tuple-unpacking targets (a, b = …) are ast.Tuple, not ast.Name → skipped + assert "a" not in result + assert "b" not in result + # Attribute aug-assign (some_obj.attr += 1) → target is Attribute, skipped + assert "some_obj" not in result + + +def test_collect_top_level_names_syntax_error(): + """Invalid Python source → empty set.""" + assert _collect_top_level_names("def broken(:") == set() + + +def test_collect_imported_names_various(): + """Covers import, import-as, from-import, from-import-as, star (skip).""" + source = ( + "import os\n" + "import os.path\n" + "import json as json_mod\n" + "from pathlib import Path\n" + "from typing import List as L\n" + "from os import *\n" + ) + result = _collect_imported_names(source) + assert result == {"os", "path", "json_mod", "Path", "L"} + + +def test_collect_imported_names_syntax_error(): + """Invalid Python source → empty set.""" + assert _collect_imported_names("def broken(:") == set() + + +def test_collect_assignment_names_basic(): + """Covers plain assignment, annotated assignment, augmented assignment.""" + source = ( + "_CONST = 42\n" + "x: int = 1\n" + "counter += 1\n" + "a, b = 1, 2\n" # tuple target → skipped + "obj.attr += 1\n" # attribute aug-assign → skipped + "def my_func(): pass\n" # function → skipped + "import os\n" # import → skipped + ) + result = _collect_assignment_names(source) + assert result == {"_CONST", "x", "counter"} + + +def test_collect_assignment_names_syntax_error(): + """Invalid Python source → empty set.""" + assert _collect_assignment_names("def broken(:") == set() + + +def test_collect_code_referenced_names_finds_load_uses(): + """Names used in code expressions are returned.""" + src = "from .sub import MyFunc\nresult = MyFunc()\n" + assert "MyFunc" in _collect_code_referenced_names(src) + + +def test_collect_code_referenced_names_excludes_import_aliases(): + """Import alias names are not ast.Name nodes → not returned.""" + src = "from .sub import MyFunc\n" + assert "MyFunc" not in _collect_code_referenced_names(src) + + +def test_collect_code_referenced_names_excludes_funcdef_name(): + """Function definition names are not ast.Name Load nodes.""" + src = "def MyFunc(): pass\n" + assert "MyFunc" not in _collect_code_referenced_names(src) diff --git a/tests/engine/cross_file/__init__.py b/tests/engine/cross_file/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/engine/cross_file/test_cross_file_transform.py b/tests/engine/cross_file/test_cross_file_transform.py new file mode 100644 index 0000000..49a74f4 --- /dev/null +++ b/tests/engine/cross_file/test_cross_file_transform.py @@ -0,0 +1,537 @@ +from unittest.mock import patch +import textwrap +from crispen.config import CrispenConfig +from crispen.engine import ( + _EXCLUDED_DIR_NAMES, + _apply_tuple_dataclass, + _blocked_private_scopes, + _find_outside_callers, + _has_callers_outside_ranges, + run_engine, +) +import libcst as cst +from ..helpers import _make_pkg, _run + + +def test_find_outside_callers_empty_qnames(tmp_path): + result = _find_outside_callers(str(tmp_path), set(), set()) + assert result == set() + + +def test_find_outside_callers_no_outside_py_files(tmp_path): + f = tmp_path / "code.py" + f.write_text("x = 1\n") + result = _find_outside_callers(str(tmp_path), {"pkg.func"}, {str(f.resolve())}) + # All .py files are in the diff → nothing to scan outside + assert result == set() + + +def test_find_outside_callers_finds_caller(tmp_path): + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + service = pkg / "service.py" + service.write_text("def get_user():\n return (1, 2, 3)\n") + outside = tmp_path / "outside.py" + outside.write_text("from mypkg.service import get_user\nget_user()\n") + + qname = "mypkg.service.get_user" + diff_files = {str(service.resolve())} + result = _find_outside_callers(str(tmp_path), {qname}, diff_files) + assert qname in result + + +def test_find_outside_callers_no_match(tmp_path): + outside = tmp_path / "other.py" + outside.write_text("x = 1\n") + qname = "mypkg.service.get_user" + result = _find_outside_callers(str(tmp_path), {qname}, set()) + assert qname not in result + + +def test_cross_file_transforms_public_func_and_caller(tmp_path): + pkg = _make_pkg(tmp_path, "mypkg") + + service = pkg / "service.py" + service.write_text( + "def get_user():\n return (name, age, score)\n", encoding="utf-8" + ) + + api = pkg / "api.py" + api.write_text( + "from mypkg.service import get_user\n" + "def main():\n" + " a, b, c = get_user()\n", + encoding="utf-8", + ) + + changed = {str(service): [(1, 2)], str(api): [(1, 4)]} + msgs = list( + run_engine( + changed, + _repo_root=str(tmp_path), + config=CrispenConfig(min_tuple_size=3), + ) + ) + + assert any("TupleDataclass" in m for m in msgs) + assert any("CallerUpdater" in m for m in msgs) + + service_text = service.read_text(encoding="utf-8") + assert "GetUserResult(" in service_text + assert "@dataclass" in service_text + + api_text = api.read_text(encoding="utf-8") + assert "_ = get_user()" in api_text + assert "_.name" in api_text + + +def test_cross_file_skips_when_outside_caller_exists(tmp_path): + pkg = _make_pkg(tmp_path, "mypkg") + + service = pkg / "service.py" + service.write_text( + "def get_user():\n return (name, age, score)\n", encoding="utf-8" + ) + + # This file is NOT in the diff but calls get_user. + outside = pkg / "outside.py" + outside.write_text( + "from mypkg.service import get_user\na, b, c = get_user()\n", + encoding="utf-8", + ) + + changed = {str(service): [(1, 2)]} + msgs = list( + run_engine( + changed, + _repo_root=str(tmp_path), + config=CrispenConfig(min_tuple_size=3), + ) + ) + + assert any("callers exist outside the diff" in m for m in msgs) + assert "return (name, age, score)" in service.read_text(encoding="utf-8") + + +def test_find_outside_callers_call_qname_not_target(tmp_path): + # outside file calls other_func (resolves to mypkg.other.other_func), + # but target is mypkg.service.get_user → hits the 118->117 branch. + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + (pkg / "other.py").write_text("def other_func(): pass\n") + caller = tmp_path / "caller.py" + caller.write_text("from mypkg.other import other_func\nother_func()\n") + + result = _find_outside_callers(str(tmp_path), {"mypkg.service.get_user"}, set()) + assert "mypkg.service.get_user" not in result + + +def test_find_outside_callers_manager_build_fails(tmp_path): + (tmp_path / "other.py").write_text("x = 1\n") + with patch( + "crispen.engine.helpers.FullRepoManager", side_effect=RuntimeError("fail") + ): + result = _find_outside_callers(str(tmp_path), {"some.func"}, set()) + # Conservative: all target qnames are blocked. + assert result == {"some.func"} + + +def test_find_outside_callers_wrapper_fails(tmp_path): + (tmp_path / "other.py").write_text("x = 1\n") + with patch("crispen.engine.helpers.FullRepoManager") as MockFRM: + MockFRM.return_value.get_metadata_wrapper_for_path.side_effect = RuntimeError( + "fail" + ) + result = _find_outside_callers(str(tmp_path), {"some.func"}, set()) + assert result == set() + + +def test_cross_file_file_not_under_repo_root(tmp_path): + # repo_root is a separate directory; changed file is not under it. + repo_root = tmp_path / "repo" + repo_root.mkdir() + f = tmp_path / "code.py" + f.write_text("def public_func():\n return (1, 2, 3)\n", encoding="utf-8") + # _compute_qname raises ValueError → all_candidates stays empty → 317->406 branch. + msgs = list( + run_engine( + {str(f): [(1, 2)]}, + _repo_root=str(repo_root), + config=CrispenConfig(min_tuple_size=3), + ) + ) + assert not any("callers" in m for m in msgs) + + +def test_no_public_candidates_with_repo_root(tmp_path): + f = tmp_path / "code.py" + f.write_text("x = 1\n", encoding="utf-8") + msgs = list(run_engine({str(f): [(1, 1)]}, _repo_root=str(tmp_path))) + assert msgs == [] + + +def test_cross_file_one_approved_one_blocked(tmp_path): + pkg = _make_pkg(tmp_path, "mypkg") + + a = pkg / "a.py" + a.write_text("def approved_func():\n return (1, 2, 3)\n", encoding="utf-8") + + b = pkg / "b.py" + b.write_text("def blocked_func():\n return (1, 2, 3)\n", encoding="utf-8") + + # outside.py calls blocked_func and is NOT in the diff. + outside = pkg / "outside.py" + outside.write_text( + "from mypkg.b import blocked_func\nblocked_func()\n", encoding="utf-8" + ) + + changed = {str(a): [(1, 2)], str(b): [(1, 2)]} + msgs = list( + run_engine( + changed, _repo_root=str(tmp_path), config=CrispenConfig(min_tuple_size=3) + ) + ) + + # blocked_func is skipped; its identity entry in alias_map hits the 349->348 branch. + assert any( + "blocked_func" in m and "callers exist outside the diff" in m for m in msgs + ) + # approved_func is transformed. + assert any("TupleDataclass" in m for m in msgs) + + +def test_cross_file_caller_updater_file_not_under_repo_root(tmp_path): + subdir = tmp_path / "repo" + subdir.mkdir() + (subdir / "__init__.py").write_text("") + + inside = subdir / "service.py" + inside.write_text("def approved():\n return (1, 2, 3)\n", encoding="utf-8") + + # This file is in the diff but outside repo_root (subdir). + outside_code = tmp_path / "outside_code.py" + outside_code.write_text("x = 1\n", encoding="utf-8") + + changed = {str(inside): [(1, 2)], str(outside_code): [(1, 1)]} + # No crash; outside_code.py's _file_to_module raises ValueError → continue. + list( + run_engine( + changed, _repo_root=str(subdir), config=CrispenConfig(min_tuple_size=3) + ) + ) + + +def test_cross_file_caller_updater_parse_error(tmp_path): + pkg = _make_pkg(tmp_path, "mypkg") + + service = pkg / "service.py" + service.write_text("def approved():\n return (1, 2, 3)\n", encoding="utf-8") + + changed = {str(service): [(1, 2)]} + + original_parse = cst.parse_module + + def patched_parse(source): + # After Phase 2 transforms the source, it will contain "@dataclass". + # Fail on that call to exercise the 374-375 parse-error branch. + if "@dataclass" in source: + raise cst.ParserSyntaxError( + "fake error", lines=("@dataclass",), raw_line=0, raw_column=0 + ) + return original_parse(source) + + with patch("crispen.engine.cst.parse_module", patched_parse): + # Should not crash; CallerUpdater pass silently continues. + list( + run_engine( + changed, + _repo_root=str(tmp_path), + config=CrispenConfig(min_tuple_size=3), + ) + ) + + +def test_cross_file_caller_updater_raises(tmp_path): + pkg = _make_pkg(tmp_path, "mypkg") + + service = pkg / "service.py" + service.write_text("def approved():\n return (1, 2, 3)\n", encoding="utf-8") + + changed = {str(service): [(1, 2)]} + + with patch("crispen.engine.CallerUpdater", side_effect=RuntimeError("fail")): + # Should not crash; the exception is caught. + list( + run_engine( + changed, + _repo_root=str(tmp_path), + config=CrispenConfig(min_tuple_size=3), + ) + ) + + +def test_cross_file_init_alias_detected_as_outside_caller(tmp_path): + pkg = _make_pkg(tmp_path, "mypkg") + + # Re-export get_user through __init__.py + (pkg / "__init__.py").write_text( + "from mypkg.service import get_user\n", encoding="utf-8" + ) + + service = pkg / "service.py" + service.write_text( + "def get_user():\n return (name, age, score)\n", encoding="utf-8" + ) + + # Outside file imports via the alias (pkg.get_user) + outside = tmp_path / "outside.py" + outside.write_text( + "from mypkg import get_user\na, b, c = get_user()\n", encoding="utf-8" + ) + + changed = {str(service): [(1, 2)]} + msgs = list( + run_engine( + changed, _repo_root=str(tmp_path), config=CrispenConfig(min_tuple_size=3) + ) + ) + + assert any("callers exist outside the diff" in m for m in msgs) + + +def test_find_outside_callers_excludes_venv_dirs(tmp_path): + """Files inside excluded directories (.venv, __pycache__, etc.) are skipped.""" + for dirname in _EXCLUDED_DIR_NAMES: + excluded = tmp_path / dirname / "lib" + excluded.mkdir(parents=True, exist_ok=True) + (excluded / "pkg.py").write_text( + "from mypkg.service import get_user\nget_user()\n" + ) + # Even though each excluded dir has a caller, none should be counted. + result = _find_outside_callers(str(tmp_path), {"mypkg.service.get_user"}, set()) + assert "mypkg.service.get_user" not in result + + +def test_phase1_private_caller_updated(tmp_path): + """Private function callers in the same file are updated after Phase 1.""" + source = textwrap.dedent( + """\ + def _make_result(): + return (1, 2, 3) + + def use_it(): + a, b, c = _make_result() + """ + ) + f = tmp_path / "code.py" + f.write_text(source, encoding="utf-8") + msgs = _run({str(f): [(1, 100)]}) + result = f.read_text(encoding="utf-8") + assert "_ = _make_result()" in result + assert any("CallerUpdater" in m for m in msgs) + + +def test_phase1_private_no_callers_no_caller_updater_msg(tmp_path): + """Private transform with no callers produces no CallerUpdater message.""" + source = "def _make_result():\n return (1, 2, 3)\n" + f = tmp_path / "code.py" + f.write_text(source, encoding="utf-8") + msgs = _run({str(f): [(1, 100)]}) + assert any("TupleDataclass" in m for m in msgs) + assert not any("CallerUpdater" in m for m in msgs) + + +def test_phase1_private_caller_updater_exception_ignored(tmp_path): + """If CallerUpdater raises during Phase 1, the engine continues gracefully.""" + source = textwrap.dedent( + """\ + def _make_result(): + return (1, 2, 3) + + def use_it(): + a, b, c = _make_result() + """ + ) + f = tmp_path / "code.py" + f.write_text(source, encoding="utf-8") + with patch("crispen.engine.CallerUpdater", side_effect=RuntimeError("fail")): + msgs = _run({str(f): [(1, 100)]}) + # TupleDataclass still ran successfully + assert any("TupleDataclass" in m for m in msgs) + + +def test_has_callers_outside_ranges_found(): + source = "def f(): pass\nf()\n" # call on line 2, range is only line 1 + assert _has_callers_outside_ranges(source, "f", [(1, 1)]) is True + + +def test_has_callers_outside_ranges_not_found(): + source = "def f(): pass\nf()\n" # call on line 2, range covers line 2 + assert _has_callers_outside_ranges(source, "f", [(1, 2)]) is False + + +def test_has_callers_outside_ranges_syntax_error(): + assert _has_callers_outside_ranges("def f(:", "f", [(1, 1)]) is False + + +def test_blocked_private_scopes_finds_outside_callers(): + # _helper called at line 3, diff range only covers line 1 + source = "def _helper(): pass\n\n_helper()\n" + blocked = _blocked_private_scopes(source, [(1, 1)]) + assert "_helper" in blocked + + +def test_blocked_private_scopes_ignores_in_range_callers(): + # _helper called at line 3, diff range covers line 3 + source = "def _helper(): pass\n\n_helper()\n" + blocked = _blocked_private_scopes(source, [(1, 3)]) + assert "_helper" not in blocked + + +def test_blocked_private_scopes_syntax_error(): + blocked = _blocked_private_scopes("def f(:", [(1, 1)]) + assert blocked == set() + + +def test_blocked_private_scopes_ignores_public(): + # Public functions (no leading _) should not appear in blocked set + source = "def helper(): pass\n\nhelper()\n" + blocked = _blocked_private_scopes(source, [(1, 1)]) + assert "helper" not in blocked + + +def test_update_diff_file_callers_false_blocks_private_with_outside_caller(tmp_path): + """Private function with a caller outside diff ranges is NOT transformed.""" + source = textwrap.dedent( + """\ + def _make_result(): + return (a, b, c) + + def use_in_diff(): + x, y, z = _make_result() + + def use_outside_diff(): + p, q, r = _make_result() + """ + ) + f = tmp_path / "code.py" + f.write_text(source, encoding="utf-8") + config = CrispenConfig(min_tuple_size=3, update_diff_file_callers=False) + # Diff only covers the function definition and use_in_diff + msgs = list(run_engine({str(f): [(1, 5)]}, config=config)) + # Should NOT have been transformed (outside callers exist) + assert not any("TupleDataclass" in m for m in msgs) + assert "return (a, b, c)" in f.read_text(encoding="utf-8") + + +def test_update_diff_file_callers_false_allows_private_with_only_diff_callers( + tmp_path, +): + """Private function with all callers inside diff is transformed.""" + source = textwrap.dedent( + """\ + def _make_result(): + return (a, b, c) + + def use_in_diff(): + x, y, z = _make_result() + """ + ) + f = tmp_path / "code.py" + f.write_text(source, encoding="utf-8") + config = CrispenConfig(min_tuple_size=3, update_diff_file_callers=False) + msgs = list(run_engine({str(f): [(1, 5)]}, config=config)) + # Only diff caller exists → transformation should proceed + assert any("TupleDataclass" in m for m in msgs) + + +def test_update_diff_file_callers_false_blocks_public_with_diff_file_outside_caller( + tmp_path, +): + """Public function with callers outside diff in diff file is skipped.""" + pkg = _make_pkg(tmp_path, "mypkg") + + service = pkg / "service.py" + service.write_text( + "def get_user():\n return (name, age, score)\n", encoding="utf-8" + ) + + api = pkg / "api.py" + api.write_text( + "from mypkg.service import get_user\n" + "def in_diff():\n" + " a, b, c = get_user()\n" + "def not_in_diff():\n" + " x, y, z = get_user()\n", + encoding="utf-8", + ) + + # api.py diff only covers lines 1-3 (in_diff function) + changed = {str(service): [(1, 2)], str(api): [(1, 3)]} + config = CrispenConfig(min_tuple_size=3, update_diff_file_callers=False) + msgs = list(run_engine(changed, _repo_root=str(tmp_path), config=config)) + + # get_user has a caller outside the diff (not_in_diff at lines 4-5) + assert any("callers exist outside the diff" in m for m in msgs) + + +def test_update_diff_file_callers_false_allows_public_with_all_callers_in_diff( + tmp_path, +): + """Public function with all callers inside diff (no diff-file outside callers).""" + pkg = _make_pkg(tmp_path, "mypkg") + + service = pkg / "service.py" + service.write_text( + "def get_user():\n return (name, age, score)\n", encoding="utf-8" + ) + + api = pkg / "api.py" + api.write_text( + "from mypkg.service import get_user\n" + "def main():\n" + " a, b, c = get_user()\n", + encoding="utf-8", + ) + + changed = {str(service): [(1, 2)], str(api): [(1, 3)]} + config = CrispenConfig(min_tuple_size=3, update_diff_file_callers=False) + msgs = list(run_engine(changed, _repo_root=str(tmp_path), config=config)) + + # All callers within diff → transformation should proceed even with + # update_diff_file_callers=False (no callers outside diff ranges) + assert any("TupleDataclass" in m for m in msgs) + assert any("CallerUpdater" in m for m in msgs) + + +def test_phase2_apply_tuple_dataclass_td_none(tmp_path): + """Phase 2 _apply_tuple_dataclass returning td=None is handled gracefully.""" + pkg = _make_pkg(tmp_path, "mypkg") + + service = pkg / "service.py" + service.write_text("def approved():\n return (1, 2, 3)\n", encoding="utf-8") + + orig_apply = _apply_tuple_dataclass + call_count = {"n": 0} + + def patched_apply(filepath, ranges, source, verbose, approved_public_funcs, **kw): + call_count["n"] += 1 + if call_count["n"] == 2: + # Phase 2 call: return td=None to exercise the td2 is None branch + return (source, [], None) + return orig_apply( + filepath, ranges, source, verbose, approved_public_funcs, **kw + ) + + with patch("crispen.engine._apply_tuple_dataclass", patched_apply): + msgs = list( + run_engine( + {str(service): [(1, 2)]}, + _repo_root=str(tmp_path), + config=CrispenConfig(min_tuple_size=3), + ) + ) + # Should not crash; Phase 2 gracefully skips categorization + assert isinstance(msgs, list) diff --git a/tests/engine/cross_file/test_inline_imports.py b/tests/engine/cross_file/test_inline_imports.py new file mode 100644 index 0000000..22ccb39 --- /dev/null +++ b/tests/engine/cross_file/test_inline_imports.py @@ -0,0 +1,355 @@ +from unittest.mock import patch +from crispen.config import CrispenConfig +from crispen.engine import ( + _add_fl_context, + _apply_tuple_dataclass, + _module_path_for_file, + _patch_inline_imports_after_test_deletion, + _redirect_inline_module_imports, + run_engine, +) +from crispen.errors import CrispenAPIError +from crispen.file_limiter.runner import FileLimiterResult +import pytest +from ..helpers import _FL_PATCH + + +def test_apply_tuple_dataclass_parse_error(): + bad_source = "def f(:\n pass\n" + source_out, msgs, td = _apply_tuple_dataclass( + "fake.py", [(1, 10)], bad_source, False, set() + ) + assert any("parse error" in m for m in msgs) + assert td is None + assert source_out == bad_source + + +def test_apply_tuple_dataclass_crispen_api_error(): + with patch("crispen.engine.helpers.MetadataWrapper") as MockWrapper: + MockWrapper.return_value.visit.side_effect = CrispenAPIError("test api error") + with pytest.raises(CrispenAPIError): + _apply_tuple_dataclass("f.py", [(1, 1)], "x = 1\n", False, set()) + + +def test_module_path_for_file_no_markers_returns_none(tmp_path): + f = tmp_path / "test_foo.py" + f.write_text("", encoding="utf-8") + # No pyproject.toml / .git anywhere up the path — within tmp_path hierarchy. + # We can't guarantee no markers exist above tmp_path in the real filesystem, + # so only assert that the function returns a string or None without raising. + result = _module_path_for_file(str(f)) + assert result is None or isinstance(result, str) + + +def test_redirect_inline_module_imports_basic(): + source = "def run():\n from pkg.old import Foo, Bar\n Foo()\n" + result = _redirect_inline_module_imports( + source, "pkg.old", {"Foo": "pkg.new_foo", "Bar": "pkg.new_bar"} + ) + assert "from pkg.new_foo import Foo" in result + assert "from pkg.new_bar import Bar" in result + assert "from pkg.old import" not in result + + +def test_redirect_inline_module_imports_partial_redirect(): + # Only 'Foo' has a new location; 'Baz' is unknown and kept in old module. + source = "def run():\n from pkg.old import Foo, Baz\n Foo()\n" + result = _redirect_inline_module_imports(source, "pkg.old", {"Foo": "pkg.new_foo"}) + assert "from pkg.new_foo import Foo" in result + assert "from pkg.old import Baz" in result + + +def test_redirect_inline_module_imports_no_matching_import(): + source = "def run():\n from pkg.other import Foo\n Foo()\n" + result = _redirect_inline_module_imports(source, "pkg.old", {"Foo": "pkg.new"}) + assert result == source + + +def test_redirect_inline_module_imports_module_level(): + # Module-level import is also redirected. + source = "from pkg.old import Foo\n\ndef run():\n Foo()\n" + result = _redirect_inline_module_imports(source, "pkg.old", {"Foo": "pkg.new"}) + assert "from pkg.new import Foo" in result + assert "from pkg.old import" not in result + + +def test_redirect_inline_module_imports_syntax_error(): + source = "def (invalid" + result = _redirect_inline_module_imports(source, "pkg.old", {"Foo": "pkg.new"}) + assert result == source + + +def test_redirect_inline_module_imports_no_moved_names(): + # Import exists but none of the names are in the map — leave unchanged. + source = "def run():\n from pkg.old import Baz\n" + result = _redirect_inline_module_imports(source, "pkg.old", {"Foo": "pkg.new"}) + assert result == source + + +def test_patch_inline_imports_after_test_deletion_updates_per_file(tmp_path): + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + sub = tmp_path / "pkg" + sub.mkdir() + # Simulate the deleted test file path and new files created by its split. + deleted_path = str(tmp_path / "pkg" / "test_old.py") + deleted_dir = tmp_path / "pkg" + new_files = { + "sub/test_new.py": "class TestFoo:\n pass\n", + } + (deleted_dir / "sub").mkdir() + (deleted_dir / "sub" / "test_new.py").write_text(new_files["sub/test_new.py"]) + + src = "def run():\n from pkg.test_old import TestFoo\n TestFoo()\n" + per_file = { + "parent.py": { + "source": src, + "original": src, + } + } + fl_new_file_final: dict = {} + + _patch_inline_imports_after_test_deletion( + deleted_path, deleted_dir, new_files, per_file, fl_new_file_final + ) + + updated = per_file["parent.py"]["source"] + assert "from pkg.sub.test_new import TestFoo" in updated + assert "from pkg.test_old import" not in updated + + +def test_patch_inline_imports_after_test_deletion_updates_new_files(tmp_path): + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + sub = tmp_path / "pkg" + sub.mkdir() + deleted_path = str(tmp_path / "pkg" / "test_old.py") + deleted_dir = tmp_path / "pkg" + + sibling_content = ( + "def helper():\n from pkg.test_old import TestFoo\n TestFoo()\n" + ) + sibling_path = str(tmp_path / "pkg" / "test_sibling.py") + (tmp_path / "pkg" / "test_sibling.py").write_text(sibling_content, encoding="utf-8") + + (deleted_dir / "sub").mkdir() + new_file_content = "class TestFoo:\n pass\n" + (deleted_dir / "sub" / "test_new.py").write_text(new_file_content) + + new_files = {"sub/test_new.py": new_file_content} + per_file: dict = {} + fl_new_file_final = {sibling_path: sibling_content} + + _patch_inline_imports_after_test_deletion( + deleted_path, deleted_dir, new_files, per_file, fl_new_file_final + ) + + updated = fl_new_file_final[sibling_path] + assert "from pkg.sub.test_new import TestFoo" in updated + assert "from pkg.test_old import" not in updated + # File was re-written to disk. + assert ( + "from pkg.sub.test_new import TestFoo" + in (tmp_path / "pkg" / "test_sibling.py").read_text() + ) + + +def test_patch_inline_imports_after_test_deletion_no_markers_skips(tmp_path): + # No pyproject.toml — module path unresolvable; function must not raise. + deleted_path = str(tmp_path / "pkg" / "test_old.py") + deleted_dir = tmp_path / "pkg" + deleted_dir.mkdir(parents=True) + per_file = { + "parent.py": { + "source": "def run():\n from pkg.test_old import TestFoo\n", + "original": "def run():\n from pkg.test_old import TestFoo\n", + } + } + # Should not raise; source is unchanged because old_mod is None. + _patch_inline_imports_after_test_deletion( + deleted_path, deleted_dir, {}, per_file, {} + ) + assert ( + per_file["parent.py"]["source"] + == "def run():\n from pkg.test_old import TestFoo\n" + ) + + +def test_patch_inline_imports_after_test_deletion_new_mod_none_skips(tmp_path): + # new_mod resolves to None for a path outside the project root → that entry + # is skipped and name_to_new_mod stays empty → function returns early. + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + sub = tmp_path / "pkg" + sub.mkdir() + deleted_path = str(tmp_path / "pkg" / "test_old.py") + deleted_dir = tmp_path / "pkg" + # Relative path that escapes the project root when resolved from deleted_dir. + outside_rel = "../../../outside/test_new.py" + per_file = { + "parent.py": { + "source": "def run():\n from pkg.test_old import TestFoo\n", + "original": "def run():\n from pkg.test_old import TestFoo\n", + } + } + _patch_inline_imports_after_test_deletion( + deleted_path, + deleted_dir, + {outside_rel: "class TestFoo:\n pass\n"}, + per_file, + {}, + ) + # Source unchanged because name_to_new_mod was empty. + assert ( + per_file["parent.py"]["source"] + == "def run():\n from pkg.test_old import TestFoo\n" + ) + + +def test_patch_inline_imports_after_test_deletion_syntax_error_in_new_file(tmp_path): + # SyntaxError in a new_file content → that entry is skipped. + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + sub = tmp_path / "pkg" / "sub" + sub.mkdir(parents=True) + (sub / "test_new.py").write_text("def (invalid", encoding="utf-8") + deleted_path = str(tmp_path / "pkg" / "test_old.py") + deleted_dir = tmp_path / "pkg" + per_file = { + "parent.py": { + "source": "def run():\n from pkg.test_old import TestFoo\n", + "original": "def run():\n from pkg.test_old import TestFoo\n", + } + } + _patch_inline_imports_after_test_deletion( + deleted_path, deleted_dir, {"sub/test_new.py": "def (invalid"}, per_file, {} + ) + # Source unchanged; SyntaxError in the new file caused it to be skipped. + assert ( + per_file["parent.py"]["source"] + == "def run():\n from pkg.test_old import TestFoo\n" + ) + + +def test_patch_inline_imports_after_test_deletion_no_class_or_func_in_new_file( + tmp_path, +): + # New file has no ClassDef/FunctionDef → name_to_new_mod stays empty → skip. + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + sub = tmp_path / "pkg" / "sub" + sub.mkdir(parents=True) + deleted_path = str(tmp_path / "pkg" / "test_old.py") + deleted_dir = tmp_path / "pkg" + per_file = { + "parent.py": { + "source": "def run():\n from pkg.test_old import TestFoo\n", + "original": "def run():\n from pkg.test_old import TestFoo\n", + } + } + _patch_inline_imports_after_test_deletion( + deleted_path, deleted_dir, {"sub/test_new.py": "X = 1\n"}, per_file, {} + ) + assert ( + per_file["parent.py"]["source"] + == "def run():\n from pkg.test_old import TestFoo\n" + ) + + +def test_patch_inline_imports_after_test_deletion_source_unchanged_no_import(tmp_path): + # per_file source has no import from old_mod → update is a no-op. + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + sub = tmp_path / "pkg" / "sub" + sub.mkdir(parents=True) + (sub / "test_new.py").write_text("class TestFoo:\n pass\n", encoding="utf-8") + deleted_path = str(tmp_path / "pkg" / "test_old.py") + deleted_dir = tmp_path / "pkg" + original_source = "def run():\n pass\n" + per_file = { + "parent.py": { + "source": original_source, + "original": original_source, + } + } + _patch_inline_imports_after_test_deletion( + deleted_path, + deleted_dir, + {"sub/test_new.py": "class TestFoo:\n pass\n"}, + per_file, + {}, + ) + # Source is identical to original (no-op branch taken). + assert per_file["parent.py"]["source"] == original_source + + +def test_file_limiter_recursive_test_deletion_patches_parent_inline_imports(tmp_path): + """When a recursive split deletes a test file, inline imports in the parent + file that point to the deleted module are updated to the new locations.""" + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + + # Parent test file with inline imports referencing a module that will be + # created by the first split and then deleted by the recursive split. + parent_src = ( + "def run_comprehensive_tests():\n" + " from tests.test_collections import TestA, TestB\n" + " TestA()\n" + " TestB()\n" + "\n" + "if __name__ == '__main__':\n" + " run_comprehensive_tests()\n" + ) + parent_file = tmp_path / "tests" / "test_suite.py" + parent_file.parent.mkdir(parents=True) + parent_file.write_text(parent_src, encoding="utf-8") + + # First pass: parent → creates test_collections.py (oversized). + # The original source has the inline import already present (as if + # _inject_inline_test_imports_original added it). + first_result = FileLimiterResult( + original_source=parent_src, # unchanged (inline import already injected) + new_files={ + "test_collections.py": ( + "class TestA:\n pass\n\nclass TestB:\n pass\n" + ) + }, + messages=[], + abort=False, + ) + + # Recursive pass: test_collections.py → split into sub/test_a.py + sub/test_b.py. + # All entities migrated; original_source is empty → file will be deleted. + recursive_result = FileLimiterResult( + original_source="", + new_files={ + "sub/test_a.py": "class TestA:\n pass\n", + "sub/test_b.py": "class TestB:\n pass\n", + }, + messages=[], + abort=False, + ) + + with patch(_FL_PATCH, side_effect=[first_result, recursive_result]): + list( + run_engine( + {str(parent_file): [(1, len(parent_src.splitlines()))]}, + config=CrispenConfig(max_file_lines=2, file_limiter_recursive=True), + ) + ) + + # test_collections.py was deleted. + assert not (parent_file.parent / "test_collections.py").exists() + + # Parent file now has updated inline imports pointing to the new locations. + updated = parent_file.read_text(encoding="utf-8") + assert "from tests.sub.test_a import TestA" in updated + assert "from tests.sub.test_b import TestB" in updated + assert "from tests.test_collections import" not in updated + + +def test_add_fl_context_no_module_path(): + """When module path cannot be determined, _add_fl_context does nothing.""" + fl_list = [] + fl_result = FileLimiterResult( + original_source="", + new_files={}, + abort=False, + entity_to_target={"X": "a.py"}, + ) + # A path with no ancestor containing pyproject.toml / .git → returns None. + _add_fl_context(fl_list, "/no/project/root/here/file.py", "", fl_result, {}) + assert fl_list == [] diff --git a/tests/engine/cross_file/test_patch_map.py b/tests/engine/cross_file/test_patch_map.py new file mode 100644 index 0000000..e4b485e --- /dev/null +++ b/tests/engine/cross_file/test_patch_map.py @@ -0,0 +1,378 @@ +from unittest.mock import patch +from crispen.engine import ( + _build_alias_map, + _build_patch_map, + _compute_qname, + _file_to_module, + _find_repo_root, +) +from crispen.file_limiter.runner import FileLimiterResult + + +def test_find_repo_root_finds_git(tmp_path): + (tmp_path / ".git").mkdir() + subdir = tmp_path / "src" + subdir.mkdir() + f = subdir / "code.py" + f.write_text("x = 1\n") + root = _find_repo_root({str(f): [(1, 1)]}) + assert root == str(tmp_path) + + +def test_find_repo_root_not_found(tmp_path): + f = tmp_path / "code.py" + f.write_text("x = 1\n") + root = _find_repo_root({str(f): [(1, 1)]}) + assert root is None + + +def test_file_to_module_regular_file(tmp_path): + f = tmp_path / "mypkg" / "service.py" + f.parent.mkdir() + f.write_text("x = 1\n") + assert _file_to_module(str(tmp_path), str(f)) == "mypkg.service" + + +def test_file_to_module_init(tmp_path): + f = tmp_path / "mypkg" / "__init__.py" + f.parent.mkdir() + f.write_text("") + assert _file_to_module(str(tmp_path), str(f)) == "mypkg" + + +def test_compute_qname(tmp_path): + f = tmp_path / "pkg" / "mod.py" + f.parent.mkdir() + f.write_text("") + assert _compute_qname(str(tmp_path), str(f), "my_func") == "pkg.mod.my_func" + + +def test_build_alias_map_identity_only(tmp_path): + # No __init__.py in tmp_path → only identity mapping returned. + alias_map = _build_alias_map(str(tmp_path), {"a.b.func"}) + assert alias_map == {"a.b.func": "a.b.func"} + + +def test_build_alias_map_with_reexport(tmp_path): + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("from mypkg.service import get_user\n") + alias_map = _build_alias_map(str(tmp_path), {"mypkg.service.get_user"}) + assert "mypkg.get_user" in alias_map + assert alias_map["mypkg.get_user"] == "mypkg.service.get_user" + + +def test_build_alias_map_star_import_skipped(tmp_path): + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("from mypkg.service import *\n") + alias_map = _build_alias_map(str(tmp_path), {"mypkg.service.get_user"}) + # Star import does not create an alias + assert "mypkg.get_user" not in alias_map + + +def test_build_alias_map_ambiguous_name_skipped(tmp_path): + # Two canonical qnames share the same function name → alias is ambiguous. + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("from mypkg.service import get_user\n") + alias_map = _build_alias_map( + str(tmp_path), + {"mypkg.service.get_user", "mypkg.other.get_user"}, + ) + # Ambiguous: skip adding the alias + assert "mypkg.get_user" not in alias_map + + +def test_build_alias_map_invalid_init_skipped(tmp_path): + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("def f(:\n pass\n") # invalid Python + alias_map = _build_alias_map(str(tmp_path), {"mypkg.service.get_user"}) + # Gracefully skips the unreadable __init__.py + assert alias_map == {"mypkg.service.get_user": "mypkg.service.get_user"} + + +def test_build_alias_map_skips_compound_statement(tmp_path): + # A function definition is a compound statement, not SimpleStatementLine (line 76). + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("def helper():\n pass\n") + alias_map = _build_alias_map(str(tmp_path), {"mypkg.service.get_user"}) + assert alias_map == {"mypkg.service.get_user": "mypkg.service.get_user"} + + +def test_build_alias_map_skips_non_import_in_simple_stmt(tmp_path): + # An assignment in SimpleStatementLine is not ImportFrom (line 79). + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("__version__ = '1.0'\n") + alias_map = _build_alias_map(str(tmp_path), {"mypkg.service.get_user"}) + assert alias_map == {"mypkg.service.get_user": "mypkg.service.get_user"} + + +def test_build_patch_map_import_alias_single_importer(tmp_path): + """Import alias from original used in exactly one new file → added.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + pre_split = "from external import Helper\ndef MyFunc(): pass\n" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "def MyFunc(): pass\n", + "utils.py": "from external import Helper\nHelper()\n", + }, + entity_to_target={"MyFunc": "sub.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg, pre_split) + assert result["mypkg.module.Helper"] == "mypkg.utils.Helper" + assert result["mypkg.module.MyFunc"] == "mypkg.sub.MyFunc" + + +def test_build_patch_map_import_alias_skips_entity_names(tmp_path): + """Import alias that is also an entity name is not double-processed.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + # "Helper" appears both as entity_to_target key and in pre_split imports + pre_split = "from external import Helper\n" + fl_result = FileLimiterResult( + original_source="", + new_files={"utils.py": "from external import Helper\n"}, + entity_to_target={"Helper": "utils.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg, pre_split) + # Entity loop handles Helper (definer=utils.py, no external callers → utils.py) + assert result == {"mypkg.module.Helper": "mypkg.utils.Helper"} + + +def test_build_patch_map_import_alias_module_none(tmp_path): + """Import alias target module can't be resolved → alias is skipped.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + pre_split = "from external import Helper\ndef MyFunc(): pass\n" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "def MyFunc(): pass\n", + "utils.py": "from external import Helper\nHelper()\n", + }, + entity_to_target={"MyFunc": "sub.py"}, + ) + # Third call (for alias importer utils.py) returns None + with patch( + "crispen.engine.file_limiter._module_path_for_file", + side_effect=["mypkg.module", "mypkg.sub", None], + ): + result = _build_patch_map(str(f), fl_result, pkg, pre_split) + # MyFunc was added (second call succeeded); Helper was skipped (third → None) + assert result == {"mypkg.module.MyFunc": "mypkg.sub.MyFunc"} + assert "mypkg.module.Helper" not in result + + +def test_build_patch_map_import_only_caller_falls_back_to_definer(tmp_path): + """Entity imported but not used by any new file → falls back to definer.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "def MyFunc(): pass\n", + "__init__.py": "from .sub import MyFunc\n", + }, + entity_to_target={"MyFunc": "sub.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg) + # __init__.py only re-exports (no Load usage) → 0 real callers → fall back to sub.py + assert result == {"mypkg.module.MyFunc": "mypkg.sub.MyFunc"} + + +def test_build_patch_map_reexport_ignored_real_caller_wins(tmp_path): + """Re-export stub ignored; the one file that actually calls the entity is used.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "def MyFunc(): pass\n", + "caller.py": "from .sub import MyFunc\nMyFunc()\n", + "__init__.py": "from .sub import MyFunc\n", + }, + entity_to_target={"MyFunc": "sub.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg) + # __init__.py has no Load usage; caller.py does → single real caller + assert result == {"mypkg.module.MyFunc": "mypkg.caller.MyFunc"} + + +def test_build_patch_map_init_real_usage_counted_as_caller(tmp_path): + """__init__.py that actually calls an entity is counted as a real caller. + + The module path strips .__init__ so the patch target is the public + package namespace (mypkg.MyFunc, not mypkg.__init__.MyFunc). + """ + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "def MyFunc(): pass\n", + "__init__.py": "from .sub import MyFunc\n_x = MyFunc()\n", + }, + entity_to_target={"MyFunc": "sub.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg) + # __init__.py has a Load reference → real caller; .__init__ stripped from path + assert result == {"mypkg.module.MyFunc": "mypkg.MyFunc"} + + +def test_build_patch_map_init_real_usage_plus_other_caller_forks(tmp_path): + """__init__.py calling entity + another caller → forking → skipped.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "def MyFunc(): pass\n", + "caller.py": "from .sub import MyFunc\nMyFunc()\n", + "__init__.py": "from .sub import MyFunc\nMyFunc()\n", + }, + entity_to_target={"MyFunc": "sub.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg) + # 2 real callers (caller.py + __init__.py) → forking → skipped + assert "mypkg.module.MyFunc" not in result + + +def test_build_patch_map_import_alias_reexport_stub_skipped(tmp_path): + """Import alias whose only importer is a re-export stub is skipped.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + pre_split = "from external import Helper\ndef F(): pass\n" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "def F(): pass\n", + "__init__.py": "from external import Helper\n", + }, + entity_to_target={"F": "sub.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg, pre_split) + # __init__.py imports Helper but has no Load usage → 0 real importers → skipped + assert "mypkg.module.Helper" not in result + + +def test_build_patch_map_assignment_no_callers(tmp_path): + """Module-level variable in original file, only used in its defining new file.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + pre_split = "_TIMEOUT = 30\ndef run(): pass\n" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "core.py": "_TIMEOUT = 30\ndef run(): pass\n", + }, + abort=False, + entity_to_target={"run": "core.py"}, + ) + result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) + # 0 callers → _TIMEOUT stays in its definer core.py + assert result["mypkg.big._TIMEOUT"] == "mypkg.core._TIMEOUT" + + +def test_build_patch_map_assignment_single_caller(tmp_path): + """Variable defined in one new file, imported and used by exactly one other.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + pre_split = "_TIMEOUT = 30\n" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "core.py": "_TIMEOUT = 30\n", + "runner.py": "from .core import _TIMEOUT\nif _TIMEOUT > 0: pass\n", + }, + abort=False, + entity_to_target={}, + ) + result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) + # runner.py imports and uses _TIMEOUT → single caller + assert result["mypkg.big._TIMEOUT"] == "mypkg.runner._TIMEOUT" + + +def test_build_patch_map_assignment_forking_skipped(tmp_path): + """Variable imported and used by two new files → forking → skipped.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + pre_split = "_TIMEOUT = 30\n" + fl_result = FileLimiterResult( + original_source="", + new_files={ + "core.py": "_TIMEOUT = 30\n", + "a.py": "from .core import _TIMEOUT\nif _TIMEOUT: pass\n", + "b.py": "from .core import _TIMEOUT\nif _TIMEOUT: pass\n", + }, + abort=False, + entity_to_target={}, + ) + result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) + assert "mypkg.big._TIMEOUT" not in result + + +def test_build_patch_map_assignment_already_in_patch_map_skipped(tmp_path): + """Variable in patch_map from import-alias section → assignment section skips it.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + # pre_split both imports and assigns _TIMEOUT → import-alias section maps it first. + pre_split = "from ext import _TIMEOUT\n_TIMEOUT = 30\n" + fl_result = FileLimiterResult( + original_source="", + # core.py imports, assigns, and uses _TIMEOUT: alias + assignment. + new_files={ + "core.py": ( + "from ext import _TIMEOUT\n_TIMEOUT = 30\nif _TIMEOUT > 0: pass\n" + ) + }, + abort=False, + entity_to_target={}, + ) + result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) + # Import-alias section mapped it; assignment section hits old_path in patch_map. + assert result["mypkg.big._TIMEOUT"] == "mypkg.core._TIMEOUT" + # Verify mapped exactly once (assignment section did NOT add a duplicate). + assert list(result.values()).count("mypkg.core._TIMEOUT") == 1 + + +def test_build_patch_map_assignment_new_module_none_skipped(tmp_path): + """When _module_path_for_file returns None for the target, entry is skipped.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + pre_split = "_TIMEOUT = 30\n" + fl_result = FileLimiterResult( + original_source="", + # Target path cannot be resolved to a module (no pyproject.toml ancestor). + new_files={"/unresolvable/abs/path.py": "_TIMEOUT = 30\n"}, + abort=False, + entity_to_target={}, + ) + result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) + assert "mypkg.big._TIMEOUT" not in result diff --git a/tests/engine/helpers.py b/tests/engine/helpers.py new file mode 100644 index 0000000..74880b8 --- /dev/null +++ b/tests/engine/helpers.py @@ -0,0 +1,58 @@ +from crispen.config import CrispenConfig +from crispen.engine import run_engine +from crispen.errors import CrispenAPIError +from crispen.file_limiter.runner import FileLimiterResult +from crispen.refactors.base import Refactor + + +def _run(changed): + return list(run_engine(changed, config=CrispenConfig(min_tuple_size=3))) + + +class _RaisingTransformer(Refactor): + """A Refactor subclass that always raises during tree traversal.""" + + @classmethod + def name(cls): + return "RaisingRefactor" + + def leave_Module(self, original_node, updated_node): + raise RuntimeError("intentional transform error") + + +class _CrispenApiErrorRefactor(Refactor): + @classmethod + def name(cls): + return "ApiErrorRefactor" + + def leave_Module(self, original_node, updated_node): + raise CrispenAPIError("test api error") + + +def _make_pkg(root, name): + pkg = root / name + pkg.mkdir(exist_ok=True) + (pkg / "__init__.py").write_text("", encoding="utf-8") + return pkg + + +def _make_phase1_pkg(root): + """Helper: return a tmp_path containing a package for Phase 1 tests.""" + pkg = root / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + return pkg + + +_FL_PATCH = "crispen.engine.run_file_limiter" + + +def _make_fl_result_with_entities(source="# reduced\n"): + """Build a FileLimiterResult that moved MyClass → utils.py.""" + return FileLimiterResult( + original_source=source, + new_files={"utils.py": "class MyClass: pass\n"}, + messages=["big.py: FileLimiter: moved MyClass → utils.py"], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) diff --git a/tests/engine/patch_update.py b/tests/engine/patch_update.py new file mode 100644 index 0000000..eae9fba --- /dev/null +++ b/tests/engine/patch_update.py @@ -0,0 +1,4 @@ +_REWRITE_PATCH = "crispen.engine.apply_patch_rewrite" + + +_CG_PATCH = "crispen.engine.apply_patch_callgraph" diff --git a/tests/engine/test_file_limiter.py b/tests/engine/test_file_limiter.py new file mode 100644 index 0000000..631c836 --- /dev/null +++ b/tests/engine/test_file_limiter.py @@ -0,0 +1,526 @@ +from unittest.mock import patch +from crispen.config import CrispenConfig +from crispen.engine import run_engine +from crispen.errors import CrispenAPIError +from crispen.file_limiter.runner import FileLimiterResult +from crispen.stats import RunStats +import pytest +from .helpers import _FL_PATCH + + +def test_file_limiter_disabled_by_max_file_lines_zero(tmp_path): + """max_file_lines=0 disables FileLimiter entirely (branch: if > 0 is False).""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + with patch(_FL_PATCH) as mock_fl: + list(run_engine({str(f): [(1, 1)]}, config=CrispenConfig(max_file_lines=0))) + mock_fl.assert_not_called() + + +def test_file_limiter_skips_short_file(tmp_path): + """File under max_file_lines → FileLimiter is not called for that file.""" + f = tmp_path / "short.py" + f.write_text("x = 1\n", encoding="utf-8") + with patch(_FL_PATCH) as mock_fl: + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=100), + ) + ) + mock_fl.assert_not_called() + + +def test_file_limiter_abort_adds_skip_message(tmp_path): + """FileLimiter abort → SKIP message added; no new files written.""" + f = tmp_path / "big.py" + original = "".join(f"var_{i} = {i}\n" for i in range(10)) + f.write_text(original, encoding="utf-8") + abort_result = FileLimiterResult( + original_source=original, + new_files={}, + messages=[f"SKIP {f} (FileLimiter): file cannot be split"], + abort=True, + ) + with patch(_FL_PATCH, return_value=abort_result): + msgs = list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + assert any("SKIP" in m and "FileLimiter" in m for m in msgs) + assert not (tmp_path / "utils.py").exists() + + +def test_file_limiter_no_messages_no_new_files(tmp_path): + """FileLimiter returns empty messages + no new files → no output, no writes.""" + f = tmp_path / "big.py" + original = "".join(f"var_{i} = {i}\n" for i in range(10)) + f.write_text(original, encoding="utf-8") + no_op_result = FileLimiterResult( + original_source=original, + new_files={}, + messages=[], + abort=False, + ) + with patch(_FL_PATCH, return_value=no_op_result): + msgs = list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + assert not any("FileLimiter" in m for m in msgs) + + +def test_file_limiter_success_writes_new_file(tmp_path): + """FileLimiter success → new file written, original source updated.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + success_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"utils.py": "# new file\n"}, + messages=[f"{f}: FileLimiter: moved foo → utils.py"], + abort=False, + ) + with patch(_FL_PATCH, return_value=success_result): + msgs = list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + assert any("FileLimiter" in m for m in msgs) + new_file = tmp_path / "utils.py" + assert new_file.exists() + assert new_file.read_text(encoding="utf-8") == "# new file\n" + # Original file updated with reduced source. + assert f.read_text(encoding="utf-8") == "# reduced\n" + + +def test_file_limiter_creates_nested_directory(tmp_path): + """FileLimiter target in subdir → parent dirs and __init__.py created.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + success_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"helpers/utils.py": "# helpers\n"}, + messages=[f"{f}: FileLimiter: moved bar → helpers/utils.py"], + abort=False, + ) + with patch(_FL_PATCH, return_value=success_result): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + nested = tmp_path / "helpers" / "utils.py" + assert nested.exists() + assert nested.read_text(encoding="utf-8") == "# helpers\n" + # Subdirectory is initialised as a Python package. + assert (tmp_path / "helpers" / "__init__.py").exists() + + +def test_file_limiter_existing_init_not_overwritten(tmp_path): + """If the target subdir already has __init__.py, it is not overwritten.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + helpers = tmp_path / "helpers" + helpers.mkdir() + (helpers / "__init__.py").write_text("# existing\n", encoding="utf-8") + success_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"helpers/utils.py": "# utils\n"}, + messages=[], + abort=False, + ) + with patch(_FL_PATCH, return_value=success_result): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + assert (helpers / "__init__.py").read_text(encoding="utf-8") == "# existing\n" + + +def test_file_limiter_subdir_split_non_test_deletes_original(tmp_path): + """Non-test subdir split → original file deleted; __init__.py gets split content.""" + f = tmp_path / "service.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + success_result = FileLimiterResult( + original_source=f.read_text(encoding="utf-8"), # reset to original → no write + new_files={ + "service/__init__.py": "# init\n", + "service/utils.py": "# utils\n", + }, + messages=[f"{f}: FileLimiter: moved foo → service/utils.py"], + abort=False, + subdir_name="service", + ) + s = RunStats() + with patch(_FL_PATCH, return_value=success_result): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + stats=s, + ) + ) + # Original service.py must be deleted. + assert not f.exists() + # Package files must exist. + assert (tmp_path / "service" / "__init__.py").read_text( + encoding="utf-8" + ) == "# init\n" + assert (tmp_path / "service" / "utils.py").read_text( + encoding="utf-8" + ) == "# utils\n" + # All original lines must be counted as deleted so verified_lines ≤ lines_deleted. + assert s.lines_deleted == 10 + + +def test_file_limiter_subdir_split_test_keeps_original(tmp_path): + """Test subdir split → original test file kept (not deleted).""" + f = tmp_path / "test_service.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + re_export_src = "# re-exports\n" + success_result = FileLimiterResult( + original_source=re_export_src, + new_files={"service/test_utils.py": "# test utils\n"}, + messages=[], + abort=False, + subdir_name="service", + ) + with patch(_FL_PATCH, return_value=success_result): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + # Original test file must still exist (with re-export content written back). + assert f.exists() + assert f.read_text(encoding="utf-8") == re_export_src + + +def test_file_limiter_subdir_split_has_main_keeps_original(tmp_path): + """Non-test subdir split with has_main → original file kept and updated.""" + f = tmp_path / "service.py" + original_src = "".join(f"var_{i} = {i}\n" for i in range(10)) + f.write_text(original_src, encoding="utf-8") + re_export_src = ( + "from service_lib.utils import foo\n\nif __name__ == '__main__':\n foo()\n" + ) + success_result = FileLimiterResult( + original_source=re_export_src, + new_files={"service_lib/utils.py": "def foo():\n pass\n"}, + messages=[], + abort=False, + subdir_name="service_lib", + has_main=True, + ) + with patch(_FL_PATCH, return_value=success_result): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + # Original service.py must still exist (not deleted). + assert f.exists() + # It should be updated with the re-export stubs + __main__. + assert f.read_text(encoding="utf-8") == re_export_src + # New subdir file must exist. + assert (tmp_path / "service_lib" / "utils.py").read_text(encoding="utf-8") == ( + "def foo():\n pass\n" + ) + + +def test_file_limiter_api_error_propagates(tmp_path): + """CrispenAPIError from FileLimiter propagates out of run_engine.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + with patch(_FL_PATCH, side_effect=CrispenAPIError("rate limit")): + with pytest.raises(CrispenAPIError, match="rate limit"): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + + +def test_file_limiter_recursive_splits_new_file(tmp_path): + """When a new file from FileLimiter is over the limit, it is recursively split.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + # First call: original file → creates "chunk.py" which is still over the limit. + first_result = FileLimiterResult( + original_source="# reduced original\n", + new_files={"chunk.py": "".join(f"x_{i} = {i}\n" for i in range(10))}, + messages=[f"{f}: FileLimiter: moved vars → chunk.py"], + abort=False, + ) + # Second call (recursive): chunk.py → creates "chunk_a.py" and "chunk_b.py". + second_result = FileLimiterResult( + original_source="# reduced chunk\n", + new_files={"chunk_a.py": "# a\n", "chunk_b.py": "# b\n"}, + messages=[str(tmp_path / "chunk.py") + ": FileLimiter: moved → chunk_a/b"], + abort=False, + ) + + call_count = 0 + + def _fl_side_effect(**kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + return first_result + return second_result + + with patch(_FL_PATCH, side_effect=_fl_side_effect): + msgs = list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), + ) + ) + + assert call_count == 2 + # Messages from the recursive call are yielded. + assert any("chunk_a/b" in m for m in msgs) + # Recursive split wrote additional files. + assert (tmp_path / "chunk_a.py").exists() + assert (tmp_path / "chunk_b.py").exists() + # chunk.py was updated with the reduced source from the recursive split. + assert (tmp_path / "chunk.py").read_text(encoding="utf-8") == "# reduced chunk\n" + + +def test_file_limiter_recursive_disabled(tmp_path): + """file_limiter_recursive=False skips recursive processing of new files.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) + first_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"chunk.py": oversized}, + messages=[], + abort=False, + ) + + call_count = 0 + + def _fl_side_effect(**kwargs): + nonlocal call_count + call_count += 1 + return first_result + + with patch(_FL_PATCH, side_effect=_fl_side_effect): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=False), + ) + ) + + # Only one call: recursive processing was disabled. + assert call_count == 1 + + +def test_file_limiter_recursive_abort_stops_recursion(tmp_path): + """Recursive call that aborts does not enqueue further files.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) + first_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"chunk.py": oversized}, + messages=[], + abort=False, + ) + abort_result = FileLimiterResult( + original_source=oversized, + new_files={}, + messages=["SKIP chunk.py (FileLimiter): cannot be split"], + abort=True, + ) + + side_effects = [first_result, abort_result] + + with patch(_FL_PATCH, side_effect=side_effects): + msgs = list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), + ) + ) + + assert any("cannot be split" in m for m in msgs) + + +def test_file_limiter_recursive_api_error_propagates(tmp_path): + """CrispenAPIError during recursive FileLimiter call propagates out.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) + first_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"chunk.py": oversized}, + messages=[], + abort=False, + ) + + side_effects = [first_result, CrispenAPIError("rate limit")] + + with patch(_FL_PATCH, side_effect=side_effects): + with pytest.raises(CrispenAPIError, match="rate limit"): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), + ) + ) + + +def test_file_limiter_recursive_creates_nested_init(tmp_path): + """Recursive FileLimiter creating a file in a subdirectory creates __init__.py.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) + first_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"chunk.py": oversized}, + messages=[], + abort=False, + ) + # Recursive call creates a file in a subdirectory. + second_result = FileLimiterResult( + original_source="# reduced chunk\n", + new_files={"sub/part.py": "# part\n"}, + messages=[], + abort=False, + ) + + with patch(_FL_PATCH, side_effect=[first_result, second_result]): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), + ) + ) + + assert (tmp_path / "sub" / "part.py").exists() + assert (tmp_path / "sub" / "__init__.py").exists() + + +def test_file_limiter_recursive_chains(tmp_path): + """A file created by a recursive call that is still over the limit is re-queued.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) + first_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"chunk.py": oversized}, + messages=[], + abort=False, + ) + # chunk.py recursive call itself creates another oversized file. + second_result = FileLimiterResult( + original_source="# reduced chunk\n", + new_files={"chunk2.py": oversized}, + messages=[], + abort=False, + ) + third_result = FileLimiterResult( + original_source="# reduced chunk2\n", + new_files={}, + messages=[], + abort=True, + ) + + with patch(_FL_PATCH, side_effect=[first_result, second_result, third_result]): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), + ) + ) + + assert (tmp_path / "chunk.py").exists() + assert (tmp_path / "chunk2.py").exists() + + +def test_file_limiter_recursive_source_unchanged(tmp_path): + """Recursive result with same original_source does not rewrite the file.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) + first_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"chunk.py": oversized}, + messages=[], + abort=False, + ) + # Recursive call: original_source equals the input source → no rewrite. + second_result = FileLimiterResult( + original_source=oversized, # same as what was written + new_files={"part.py": "# part\n"}, + messages=[], + abort=False, + ) + + with patch(_FL_PATCH, side_effect=[first_result, second_result]): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), + ) + ) + + # chunk.py content is the oversized source (unchanged); the engine did not + # rewrite it because original_source == r_source. + assert (tmp_path / "chunk.py").read_text(encoding="utf-8") == oversized + + +def test_file_limiter_recursive_subdir_split_deletes_file(tmp_path): + """Recursive FileLimiter subdir split on a non-test file deletes the file.""" + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) + first_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"chunk.py": oversized}, + messages=[], + abort=False, + ) + # Recursive call triggers subdir split: chunk.py → chunk/ package. + second_result = FileLimiterResult( + original_source=oversized, + new_files={"chunk/__init__.py": "# init\n", "chunk/utils.py": "# utils\n"}, + messages=[], + abort=False, + subdir_name="chunk", + ) + + with patch(_FL_PATCH, side_effect=[first_result, second_result]): + list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), + ) + ) + + # chunk.py was deleted because subdir_name is set and it's not a test file. + assert not (tmp_path / "chunk.py").exists() + assert (tmp_path / "chunk" / "__init__.py").exists() diff --git a/tests/engine/test_fl_context.py b/tests/engine/test_fl_context.py new file mode 100644 index 0000000..21cd5f0 --- /dev/null +++ b/tests/engine/test_fl_context.py @@ -0,0 +1,250 @@ +from crispen.engine import _add_fl_context +from crispen.file_limiter.runner import FileLimiterResult + + +def test_add_fl_context_block_entity_uses_specific_names(tmp_path): + """When a _block_N entity was moved and all named entities are mapped, + the block-internal names (vars, imports) from the target file are used as + specific scan keys — NOT the broad module path — so already-updated strings + like ``old_module.sub.run_engine`` are not re-sent to the LLM.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + (tmp_path / "mypkg").mkdir() + fl_list = [] + fl_result = FileLimiterResult( + original_source="modified\n", + new_files={ + "core.py": "_REFACTORS = []\nimport libcst as cst\n\ndef X(): pass\n" + }, + abort=False, + entity_to_target={"_block_1": "core.py", "X": "core.py"}, + ) + filepath = str(tmp_path / "mypkg" / "big.py") + # Both entities are in the patch map → forking_old_paths would be empty. + # _block_1 is a TOP_LEVEL block → scan core.py for block-internal names. + # X is in entity_to_target so it's excluded; _REFACTORS and cst are not. + combined = { + "mypkg.big._block_1": "mypkg.core._block_1", + "mypkg.big.X": "mypkg.core.X", + } + _add_fl_context(fl_list, filepath, "original\n", fl_result, combined) + assert len(fl_list) == 1 + # Only _REFACTORS and cst are block-internal; X is excluded (named entity). + assert fl_list[0].forking_old_paths == {"mypkg.big._REFACTORS", "mypkg.big.cst"} + assert fl_list[0].old_module == "mypkg.big" + + +def test_add_fl_context_block_entity_no_new_names(tmp_path): + """When a _block_N entity was moved but the target file contains no names + beyond those already in entity_to_target, nothing is appended.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + (tmp_path / "mypkg").mkdir() + fl_list = [] + fl_result = FileLimiterResult( + original_source="modified\n", + # core.py only defines X, which is already in entity_to_target. + new_files={"core.py": "def X(): pass\n"}, + abort=False, + entity_to_target={"_block_1": "core.py", "X": "core.py"}, + ) + filepath = str(tmp_path / "mypkg" / "big.py") + combined = { + "mypkg.big._block_1": "mypkg.core._block_1", + "mypkg.big.X": "mypkg.core.X", + } + _add_fl_context(fl_list, filepath, "original\n", fl_result, combined) + assert fl_list == [] + + +def test_add_fl_context_forking_and_block_combined(tmp_path): + """Forking entities AND block-internal names are both added to forking_old_paths.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + (tmp_path / "mypkg").mkdir() + fl_list = [] + fl_result = FileLimiterResult( + original_source="modified\n", + new_files={"core.py": "_TIMEOUT = 30\ndef Y(): pass\n"}, + abort=False, + # Y is forking (not in combined_patch_map); _block_1 moved with _TIMEOUT inside. + entity_to_target={"_block_1": "core.py", "Y": "core.py"}, + ) + filepath = str(tmp_path / "mypkg" / "big.py") + # Only _block_1 is in combined_patch_map; Y is not (forking). + combined = {"mypkg.big._block_1": "mypkg.core._block_1"} + _add_fl_context(fl_list, filepath, "original\n", fl_result, combined) + assert len(fl_list) == 1 + # Y is a forking entity; _TIMEOUT is block-internal; Y in new file is excluded + # (it's in all_entity_names). + assert fl_list[0].forking_old_paths == {"mypkg.big.Y", "mypkg.big._TIMEOUT"} + + +def test_add_fl_context_normal(tmp_path): + """Forking entity not in combined_patch_map → appended to fl_all_contexts.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + (tmp_path / "mypkg").mkdir() + fl_list = [] + fl_result = FileLimiterResult( + original_source="modified\n", + new_files={"utils.py": "class X: pass\n"}, + abort=False, + entity_to_target={"X": "utils.py"}, + ) + filepath = str(tmp_path / "mypkg" / "big.py") + _add_fl_context(fl_list, filepath, "original\n", fl_result, {}) + assert len(fl_list) == 1 + assert fl_list[0].forking_old_paths == {"mypkg.big.X"} + assert fl_list[0].old_module == "mypkg.big" + assert fl_list[0].original_source == "original\n" + assert fl_list[0].modified_source == "modified\n" + + +def test_add_fl_context_forked_import_alias_added(tmp_path): + """Import aliases forked across multiple new files are added to forking_old_paths. + + When the original file imports ``call_with_tool`` and multiple new sub-files + also import it, basic mode skips it (forking). _add_fl_context must still + add ``old_module.call_with_tool`` to forking_old_paths so the LLM rewrite + step can detect and update @patch decorators that reference it. + """ + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + (tmp_path / "mypkg").mkdir() + fl_list = [] + # Original file imports call_with_tool; both new sub-files also import it + # (forking) so basic mode left it out of combined_patch_map. + pre_split = "from external import call_with_tool\ndef F(): pass\n" + fl_result = FileLimiterResult( + original_source="modified\n", + new_files={ + "a.py": ( + "from external import call_with_tool\ncall_with_tool()\ndef F(): pass\n" + ), + "b.py": "from external import call_with_tool\ncall_with_tool()\n", + }, + abort=False, + entity_to_target={"F": "a.py"}, + ) + filepath = str(tmp_path / "mypkg" / "big.py") + # F is already in combined_patch_map (non-forking entity); call_with_tool + # is NOT in combined_patch_map (forked, skipped by basic mode). + combined = {"mypkg.big.F": "mypkg.a.F"} + _add_fl_context(fl_list, filepath, pre_split, fl_result, combined) + assert len(fl_list) == 1 + # call_with_tool must be in forking_old_paths despite F being already mapped. + assert "mypkg.big.call_with_tool" in fl_list[0].forking_old_paths + + +def test_add_fl_context_forked_import_alias_entity_name_skipped(tmp_path): + """Import alias that is also an entity name is skipped by the alias loop's continue. + + Helper is in entity_to_target (not in combined_patch_map) so the entity + section already adds it to forking_old_paths. The alias loop hits the + ``continue`` branch and does not process it again. ``other`` (import alias + only, not an entity) is picked up by the alias loop instead. + """ + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + (tmp_path / "mypkg").mkdir() + fl_list = [] + pre_split = "from ext import Helper, other\ndef F(): pass\n" + fl_result = FileLimiterResult( + original_source="modified\n", + new_files={"a.py": "from ext import other\nother()\ndef F(): pass\n"}, + abort=False, + # Helper is both an imported alias and a named entity (forking entity). + entity_to_target={"Helper": "a.py", "F": "a.py"}, + ) + filepath = str(tmp_path / "mypkg" / "big.py") + # Neither entity is in combined_patch_map → both are forking. + _add_fl_context(fl_list, filepath, pre_split, fl_result, {}) + assert len(fl_list) == 1 + # Helper was added by the entity section; other was added by the alias loop. + assert "mypkg.big.Helper" in fl_list[0].forking_old_paths + assert "mypkg.big.other" in fl_list[0].forking_old_paths + + +def test_add_fl_context_forked_import_alias_already_mapped_skipped(tmp_path): + """Import alias already in combined_patch_map is not re-added.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + (tmp_path / "mypkg").mkdir() + fl_list = [] + # pre_split imports Helper (already mapped by basic) and call_with_tool (forked). + pre_split = "from ext import Helper, call_with_tool\ndef F(): pass\n" + fl_result = FileLimiterResult( + original_source="modified\n", + new_files={ + "a.py": "from ext import call_with_tool\ncall_with_tool()\ndef F(): pass\n", + "b.py": "from ext import call_with_tool\ncall_with_tool()\n", + }, + abort=False, + entity_to_target={"F": "a.py"}, + ) + filepath = str(tmp_path / "mypkg" / "big.py") + # Helper is already in combined_patch_map (basic mapped it); call_with_tool is not. + combined = { + "mypkg.big.F": "mypkg.a.F", + "mypkg.big.Helper": "mypkg.a.Helper", + } + _add_fl_context(fl_list, filepath, pre_split, fl_result, combined) + assert len(fl_list) == 1 + # Helper is already mapped → not added again; call_with_tool is forked → added. + assert "mypkg.big.Helper" not in fl_list[0].forking_old_paths + assert "mypkg.big.call_with_tool" in fl_list[0].forking_old_paths + + +def test_add_fl_context_subdir_split_uses_init_as_modified_source(tmp_path): + """Non-test subdir split: modified_source comes from new_files[subdir/__init__.py]. + + runner.py restores fl_result.original_source to the pre-split source for + non-test, non-has_main subdir splits and places the post-split __init__.py + content in new_files. _add_fl_context must use that __init__.py content as + modified_source so _build_rename_guard_sets and the BFS terminal builder see + the correct set of names still present in the module after the split. + """ + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + (tmp_path / "mypkg").mkdir() + fl_list = [] + pre_split = "from llm import call_with_tool\ndef F(): pass\ndef G(): pass\n" + # Post-split __init__.py re-exports F but call_with_tool is NOT re-exported. + init_src = "from .sub import F\ndef advise(): pass\n" + fl_result = FileLimiterResult( + # runner.py restored original_source to pre-split for non-test subdir. + original_source=pre_split, + new_files={ + "advisor/__init__.py": init_src, + "advisor/sub.py": "from llm import call_with_tool\ndef F(): pass\n", + }, + abort=False, + subdir_name="advisor", + entity_to_target={"F": "advisor/sub.py"}, + ) + filepath = str(tmp_path / "mypkg" / "big.py") + _add_fl_context(fl_list, filepath, pre_split, fl_result, {}) + assert len(fl_list) == 1 + # modified_source must be the __init__.py content, not original_source. + assert fl_list[0].modified_source == init_src + assert fl_list[0].original_source == pre_split + + +def test_add_fl_context_subdir_split_no_init_falls_back_to_original_source(tmp_path): + """Test/has_main subdir split: no __init__.py → falls back to original_source. + + For test files and has_main files with subdir_name set, runner.py does NOT + add a subdir/__init__.py to new_files. The modified_source should therefore + fall back to fl_result.original_source (the post-split original file). + """ + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + (tmp_path / "mypkg").mkdir() + fl_list = [] + pre_split = "from llm import call_with_tool\ndef F(): pass\n" + # No __init__.py in new_files; original_source is the post-split state. + post_split_original = "from llm import call_with_tool\ndef F(): pass\n# stubs\n" + fl_result = FileLimiterResult( + original_source=post_split_original, + new_files={"advisor/sub.py": "def G(): pass\n"}, + abort=False, + subdir_name="advisor", + entity_to_target={"G": "advisor/sub.py"}, + ) + filepath = str(tmp_path / "mypkg" / "big.py") + _add_fl_context(fl_list, filepath, pre_split, fl_result, {}) + assert len(fl_list) == 1 + # Falls back to fl_result.original_source since no __init__.py in new_files. + assert fl_list[0].modified_source == post_split_original diff --git a/tests/engine/test_helpers.py b/tests/engine/test_helpers.py new file mode 100644 index 0000000..bdb6d90 --- /dev/null +++ b/tests/engine/test_helpers.py @@ -0,0 +1,145 @@ +from unittest.mock import patch +from crispen.config import CrispenConfig +from crispen.engine import ( + _add_fl_context, + _build_patch_map, + _collect_code_referenced_names, + _patch_inline_imports_after_test_deletion, + _should_run, + run_engine, +) +from crispen.file_limiter.runner import FileLimiterResult +from .helpers import _FL_PATCH + + +def test_should_run_disabled_list_allows_unlisted(): + cfg = CrispenConfig(disabled_refactors=["function_splitter"]) + assert _should_run("if_not_else", cfg) is True + assert _should_run("tuple_dataclass", cfg) is True + + +def test_file_limiter_subdir_split_empty_source_file_already_deleted(tmp_path): + """Subdir split deletes the original file; empty original_source skips re-unlink.""" + f = tmp_path / "big.py" + original = "".join(f"var_{i} = {i}\n" for i in range(10)) + f.write_text(original, encoding="utf-8") + # subdir_name causes Phase 3 to delete the original file. original_source="" + # means the per_file loop sees an empty source for a file that no longer + # exists — exercising the elif-is-False branch (803→805). + # new_files content kept short (≤ max_file_lines) to avoid recursive queue. + subdir_result = FileLimiterResult( + original_source="", + new_files={"big/__init__.py": "# package\n"}, + messages=[f"{f}: FileLimiter: subdir split → big/"], + abort=False, + subdir_name="big", + ) + with patch(_FL_PATCH, return_value=subdir_result): + msgs = list( + run_engine( + {str(f): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5), + ) + ) + assert any("FileLimiter" in m for m in msgs) + # Original file was deleted by the subdir split; must not exist. + assert not f.exists() + # Package init was created. + assert (tmp_path / "big" / "__init__.py").exists() + + +def test_file_limiter_recursive_empty_init_py_preserved(tmp_path): + """__init__.py created during recursive split is kept even when drained empty.""" + orig = tmp_path / "big.py" + orig.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) + first_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"pkg/__init__.py": oversized}, + messages=[], + abort=False, + ) + # Recursive pass drains pkg/__init__.py; original_source is empty. + second_result = FileLimiterResult( + original_source="", + new_files={"pkg/utils.py": "# utils\n"}, + messages=[], + abort=False, + ) + with patch(_FL_PATCH, side_effect=[first_result, second_result]): + list( + run_engine( + {str(orig): [(1, 1)]}, + config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), + ) + ) + # pkg/__init__.py must survive as an empty file, not be deleted. + assert (tmp_path / "pkg" / "__init__.py").exists() + assert (tmp_path / "pkg" / "__init__.py").read_text(encoding="utf-8") == "" + + +def test_patch_inline_imports_after_test_deletion_empty_fl_entry_skipped(tmp_path): + # fl_new_file_final entry with empty/None content is skipped without error. + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + sub = tmp_path / "pkg" / "sub" + sub.mkdir(parents=True) + (sub / "test_new.py").write_text("class TestFoo:\n pass\n", encoding="utf-8") + deleted_path = str(tmp_path / "pkg" / "test_old.py") + deleted_dir = tmp_path / "pkg" + fl_new_file_final = {str(tmp_path / "empty.py"): ""} + _patch_inline_imports_after_test_deletion( + deleted_path, + deleted_dir, + {"sub/test_new.py": "class TestFoo:\n pass\n"}, + {}, + fl_new_file_final, + ) + # Empty entry was skipped; dict unchanged. + assert fl_new_file_final[str(tmp_path / "empty.py")] == "" + + +def test_collect_code_referenced_names_syntax_error(): + """Returns empty set on unparseable source.""" + assert _collect_code_referenced_names("def (broken:") == set() + + +def test_build_patch_map_import_alias_forking_skipped(tmp_path): + """Import alias used in zero or multiple new files is skipped.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + pre_split = ( + "from external import Forked\nfrom external import Nowhere\ndef F(): pass\n" + ) + fl_result = FileLimiterResult( + original_source="", + new_files={ + "sub.py": "from external import Forked\nForked()\ndef F(): pass\n", + "utils.py": "from external import Forked\nForked()\n", + }, + entity_to_target={"F": "sub.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg, pre_split) + # Forked used in 2 files → forking → skipped; Nowhere used in 0 files → skipped + assert "mypkg.module.Forked" not in result + assert "mypkg.module.Nowhere" not in result + + +def test_add_fl_context_no_forking(tmp_path): + """When all entities are already in combined_patch_map, nothing is appended.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + (tmp_path / "mypkg").mkdir() + fl_list = [] + fl_result = FileLimiterResult( + original_source="", + new_files={}, + abort=False, + entity_to_target={"X": "a.py"}, + ) + filepath = str(tmp_path / "mypkg" / "big.py") + # Entity already covered by combined_patch_map → forking_old_paths is empty. + # No _block_N entities → nothing appended. + _add_fl_context(fl_list, filepath, "", fl_result, {"mypkg.big.X": "mypkg.a.X"}) + assert fl_list == [] diff --git a/tests/engine/test_patch_update.py b/tests/engine/test_patch_update.py new file mode 100644 index 0000000..297ebd6 --- /dev/null +++ b/tests/engine/test_patch_update.py @@ -0,0 +1,365 @@ +from unittest.mock import patch +from crispen.config import CrispenConfig +from crispen.engine import _build_patch_map, run_engine +from crispen.file_limiter.runner import FileLimiterResult +from crispen.stats import RunStats +from .helpers import _FL_PATCH, _make_fl_result_with_entities +from .patch_update import _CG_PATCH, _REWRITE_PATCH + + +def test_build_patch_map_empty_entity_to_target(tmp_path): + """No entity_to_target → empty map.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + f = tmp_path / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={}, + entity_to_target={}, + ) + result = _build_patch_map(str(f), fl_result, tmp_path) + assert result == {} + + +def test_build_patch_map_no_old_module(tmp_path): + """When _module_path_for_file returns None for filepath → empty map.""" + # No pyproject.toml anywhere → cannot find project root + f = tmp_path / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={"utils.py": "class MyClass: pass\n"}, + entity_to_target={"MyClass": "utils.py"}, + ) + result = _build_patch_map(str(f), fl_result, tmp_path) + assert result == {} + + +def test_build_patch_map_no_callers_uses_definer(tmp_path): + """Entity with no callers maps to its definition file.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + f = pkg / "module.py" + fl_result = FileLimiterResult( + original_source="", + new_files={"utils.py": "class MyClass: pass\n"}, + entity_to_target={"MyClass": "utils.py"}, + ) + result = _build_patch_map(str(f), fl_result, pkg) + assert result == {"mypkg.module.MyClass": "mypkg.utils.MyClass"} + + +def test_patch_update_ignore_mode(tmp_path): + """Default 'ignore' mode → @patch strings are never updated.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + f = tmp_path / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + other = tmp_path / "test_other.py" + other.write_text( + '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" + ) + + fl_result = _make_fl_result_with_entities() + with patch(_FL_PATCH, return_value=fl_result): + list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="ignore", + ), + _repo_root=str(tmp_path), + ) + ) + # test_other.py should be unchanged + assert ( + other.read_text(encoding="utf-8") + == '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n' + ) + + +def test_patch_update_rewrite_mode_calls_apply_patch_rewrite(tmp_path): + """'rewrite' mode with forking entities calls apply_patch_rewrite in Phase 4.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + # Entity appears as a caller in two new files → forking → skipped by basic. + fl_result = FileLimiterResult( + original_source="# reduced\n", + new_files={ + "utils.py": "class MyClass: pass\n", + "caller_a.py": "from .big import MyClass\nMyClass()\n", + "caller_b.py": "from .big import MyClass\nMyClass()\n", + }, + messages=[], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) + with ( + patch(_FL_PATCH, return_value=fl_result), + patch(_REWRITE_PATCH, return_value=iter([])) as mock_rewrite, + ): + list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="rewrite", + ), + _repo_root=str(tmp_path), + ) + ) + mock_rewrite.assert_called_once() + contexts = mock_rewrite.call_args[0][0] + assert len(contexts) == 1 + assert "mypkg.big.MyClass" in contexts[0].forking_old_paths + + +def test_patch_update_rewrite_mode_records_llm_stats(tmp_path): + """Rewrite accumulator with non-zero elapsed/tokens triggers record_llm_call.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + fl_result = FileLimiterResult( + original_source="# reduced\n", + new_files={ + "utils.py": "class MyClass: pass\n", + "caller_a.py": "from .big import MyClass\nMyClass()\n", + "caller_b.py": "from .big import MyClass\nMyClass()\n", + }, + messages=[], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) + + def _rewrite_with_acc( + fl_contexts, per_file, repo_root, config, verbose=False, _acc=None, **_kwargs + ): + if _acc is not None: + _acc.calls = 2 + _acc.elapsed = 1.5 + _acc.input_tokens = 100 + _acc.output_tokens = 20 + _acc.files_updated = 1 + return iter([]) + + stats = RunStats() + with ( + patch(_FL_PATCH, return_value=fl_result), + patch(_REWRITE_PATCH, side_effect=_rewrite_with_acc), + ): + list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="rewrite", + ), + _repo_root=str(tmp_path), + stats=stats, + ) + ) + assert stats.patch_rewrite_llm_calls == 2 + assert stats.patch_update_edits == 1 + assert stats.llm_elapsed == 1.5 + assert stats.llm_input_tokens == 100 + assert "patch_rewriter" in stats.llm_elapsed_by_refactor + + +def test_patch_update_rewrite_mode_no_fl_contexts_skips_apply(tmp_path): + """'rewrite' mode but no forking entities → apply_patch_rewrite not called.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + # Entity has only ONE caller → non-forking → goes into combined_patch_map. + fl_result = FileLimiterResult( + original_source="# reduced\n", + new_files={"utils.py": "class MyClass: pass\n"}, + messages=[], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) + with ( + patch(_FL_PATCH, return_value=fl_result), + patch(_REWRITE_PATCH, return_value=iter([])) as mock_rewrite, + ): + list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="rewrite", + ), + _repo_root=str(tmp_path), + ) + ) + mock_rewrite.assert_not_called() + + +def test_patch_update_rewrite_mode_recursive_fl_context_added(tmp_path): + """'rewrite' mode: forking entity from recursive FL pass is collected.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + # Main FL result: produces medium.py with 6 lines (> max_file_lines=5), + # which triggers the recursive pass. No entity_to_target here so the + # main-loop rewrite branch is not entered. + medium_src = "".join(f"med_{i} = {i}\n" for i in range(6)) + main_fl_result = FileLimiterResult( + original_source="# big_reduced\n", + new_files={"medium.py": medium_src}, + messages=[], + abort=False, + entity_to_target={}, + ) + + # Recursive FL result: MyClass appears in two callers → forking → skipped + # by _build_patch_map → not in combined_patch_map → triggers _add_fl_context. + recursive_fl_result = FileLimiterResult( + original_source="# medium_reduced\n", + new_files={ + "small.py": "class MyClass: pass\n", + "caller_a.py": "from .medium import MyClass\nMyClass()\n", + "caller_b.py": "from .medium import MyClass\nMyClass()\n", + }, + messages=[], + abort=False, + entity_to_target={"MyClass": "small.py"}, + ) + + with ( + patch(_FL_PATCH, side_effect=[main_fl_result, recursive_fl_result]), + patch(_REWRITE_PATCH, return_value=iter([])) as mock_rewrite, + ): + list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_recursive=True, + file_limiter_patch_update="rewrite", + ), + _repo_root=str(tmp_path), + ) + ) + + mock_rewrite.assert_called_once() + contexts = mock_rewrite.call_args[0][0] + assert any("mypkg.medium.MyClass" in ctx.forking_old_paths for ctx in contexts) + + +def test_patch_update_callgraph_yields_message(tmp_path): + """apply_patch_callgraph message increments patch_update_edits and is yielded.""" + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + # Entity appears in two callers → forking → _fl_all_contexts is populated + fl_result = FileLimiterResult( + original_source="# reduced\n", + new_files={ + "utils.py": "class MyClass: pass\n", + "caller_a.py": "from .big import MyClass\nMyClass()\n", + "caller_b.py": "from .big import MyClass\nMyClass()\n", + }, + messages=[], + abort=False, + entity_to_target={"MyClass": "utils.py"}, + ) + + cg_msg = "test_other.py: patch_callgraph: resolved MyClass" + + stats = RunStats() + with ( + patch(_FL_PATCH, return_value=fl_result), + patch(_CG_PATCH, return_value=iter([cg_msg])) as mock_cg, + ): + msgs = list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_patch_update="basic", + ), + _repo_root=str(tmp_path), + stats=stats, + ) + ) + + mock_cg.assert_called_once() + assert cg_msg in msgs + assert stats.patch_update_edits >= 1 + + +def test_patch_update_ignore_mode_recursive_fl_entity_to_target(tmp_path): + """'ignore' mode: recursive FL result with entity_to_target skips _add_fl_context.""" # noqa: E501 + (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + f = pkg / "big.py" + f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") + + medium_src = "".join(f"med_{i} = {i}\n" for i in range(6)) + main_fl_result = FileLimiterResult( + original_source="# big_reduced\n", + new_files={"medium.py": medium_src}, + messages=[], + abort=False, + entity_to_target={}, # empty — no _add_fl_context for main result + ) + + # Recursive FL result has non-empty entity_to_target; with "ignore" mode the + # branch at engine.py line 1278 is False → _add_fl_context is not called. + recursive_fl_result = FileLimiterResult( + original_source="# medium_reduced\n", + new_files={"small.py": "class MyClass: pass\n"}, + messages=[], + abort=False, + entity_to_target={"MyClass": "small.py"}, + ) + + medium_path = pkg / "medium.py" + call_count = 0 + + def _fl_side_effect(**kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + medium_path.write_text(medium_src, encoding="utf-8") + return main_fl_result + return recursive_fl_result + + with patch(_FL_PATCH, side_effect=_fl_side_effect): + msgs = list( + run_engine( + {str(f): [(1, 10)]}, + config=CrispenConfig( + max_file_lines=5, + file_limiter_recursive=True, + file_limiter_patch_update="ignore", + ), + _repo_root=str(tmp_path), + ) + ) + + assert call_count == 2 # main pass + one recursive pass + assert not any("callgraph" in m for m in msgs) diff --git a/tests/function_splitter/__init__.py b/tests/function_splitter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/function_splitter/test_integration.py b/tests/function_splitter/test_integration.py new file mode 100644 index 0000000..02916e8 --- /dev/null +++ b/tests/function_splitter/test_integration.py @@ -0,0 +1,428 @@ +from __future__ import annotations +from unittest.mock import patch +from crispen.refactors.function_splitter import FunctionSplitter, _ApiTimeout +from .test_unit import _make_mock_response + + +def _make_long_func(n_stmts: int, func_name: str = "long_func") -> str: + """Build a function with n_stmts independent assignments.""" + lines = [f"def {func_name}():\n"] + for i in range(n_stmts): + lines.append(f" a{i} = {i}\n") + lines.append(" return 0\n") + return "".join(lines) + + +def test_function_splitter_under_limits_no_op(): + # A small function should not be split + src = "def small():\n x = 1\n return x\n" + splitter = FunctionSplitter([(1, 10)], source=src, verbose=False) + assert splitter.get_rewritten_source() is None + + +def test_function_splitter_parse_error_no_crash(): + # Invalid source should not crash + splitter = FunctionSplitter([(1, 10)], source="def f(\n !!invalid", verbose=False) + assert splitter.get_rewritten_source() is None + + +def test_function_splitter_out_of_range_no_op(): + # Function exists but is outside changed ranges + src = _make_long_func(80) + splitter = FunctionSplitter([(200, 300)], source=src, verbose=False, max_lines=10) + assert splitter.get_rewritten_source() is None + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_over_line_limit(mock_anthropic): + mock_anthropic.Anthropic.return_value.messages.create.return_value = ( + _make_mock_response(["process_tail"]) + ) + src = _make_long_func(80) + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], + source=src, + verbose=False, + max_lines=50, + ) + + result = splitter.get_rewritten_source() + assert result is not None + compile(result, "", "exec") + assert "_process_tail" in result + assert "return _process_tail(" in result + assert len(splitter.changes_made) >= 1 + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_nested_funcdef_not_split(mock_anthropic): + # A long function containing a nested funcdef should never be split, + # even if it far exceeds the line limit. Splitting across a closure + # boundary produces cascading re-splits and semantically fragile helpers. + lines = ["def func_with_closure():\n"] + for i in range(80): + lines.append(f" a{i} = {i}\n") + lines.append(" def inner():\n") + lines.append(" return 0\n") + lines.append(" return inner()\n") + src = "".join(lines) + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], source=src, verbose=False, max_lines=10 + ) + + assert splitter.get_rewritten_source() is None + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_async_skipped(mock_anthropic): + # Async functions should not be split + src = ( + "async def foo():\n" + + "".join(f" a{i} = {i}\n" for i in range(80)) + + " return 0\n" + ) + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], source=src, verbose=False, max_lines=10 + ) + + assert splitter.get_rewritten_source() is None + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_generator_skipped(mock_anthropic): + # Generator functions should not be split + src = ( + "def gen():\n" + + "".join(f" a{i} = {i}\n" for i in range(80)) + + " yield 0\n" + ) + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], source=src, verbose=False, max_lines=10 + ) + + assert splitter.get_rewritten_source() is None + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_llm_fallback_on_api_error(mock_anthropic): + # API key not set → get_api_key raises CrispenAPIError → fallback names used + mock_anthropic.Anthropic.return_value.messages.create.return_value = ( + _make_mock_response(["tail"]) + ) + src = _make_long_func(60, "my_func") + + # No ANTHROPIC_API_KEY → get_api_key raises → fallback to "my_func_helper" + with patch.dict("os.environ", {}, clear=True): + # Remove any existing API key + import os + + os.environ.pop("ANTHROPIC_API_KEY", None) + splitter = FunctionSplitter( + [(1, 1000)], source=src, verbose=False, max_lines=30 + ) + + result = splitter.get_rewritten_source() + assert result is not None + compile(result, "", "exec") + # Fallback name used: "my_func_helper" + assert "_my_func_helper" in result + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_recursive_split(mock_anthropic): + # With small max_lines and broad changed_ranges, triggers multiple iterations + # First call names helper for first function, second call for helper + mock_anthropic.Anthropic.return_value.messages.create.side_effect = [ + _make_mock_response(["part1"]), + _make_mock_response(["part2"]), + _make_mock_response(["part3"]), + ] + + # 13 body statements → with max_lines=5, needs multiple splits + src = _make_long_func(13, "func") + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], # broad range covers all helpers too + source=src, + verbose=False, + max_lines=5, + ) + + result = splitter.get_rewritten_source() + assert result is not None + compile(result, "", "exec") + # Multiple splits occurred + assert len(splitter.changes_made) >= 2 + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_syntax_error_in_output_is_skipped(mock_anthropic): + # If the assembled edit is invalid Python, the change is not applied + # We simulate this by making _generate_call return something invalid + # Instead, test the path via a function with 1-stmt body (no valid split) + src = "def foo():\n x = 1\n" # only 1 stmt → can't split + splitter = FunctionSplitter([(1, 10)], source=src, verbose=False, max_lines=0) + # body lines=1 > 0=max_lines → tries to split but len(body_stmts)=1 < 2 → skip + assert splitter.get_rewritten_source() is None + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_no_valid_split_skipped(mock_anthropic): + # max_lines=1 → even a head with 1 stmt (+return call=2) > max_lines=1 + # So no valid splits → no change + src = _make_long_func(5, "foo") + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter([(1, 1000)], source=src, verbose=False, max_lines=1) + + assert splitter.get_rewritten_source() is None + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_with_helper_docstrings(mock_anthropic): + mock_anthropic.Anthropic.return_value.messages.create.return_value = ( + _make_mock_response(["process"]) + ) + src = _make_long_func(80) + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], + source=src, + verbose=False, + max_lines=50, + helper_docstrings=True, + ) + + result = splitter.get_rewritten_source() + assert result is not None + assert '"""' in result + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_class_method(mock_anthropic): + mock_anthropic.Anthropic.return_value.messages.create.return_value = ( + _make_mock_response(["tail_work"]) + ) + lines = ["class Foo:\n", " def method(self):\n"] + for i in range(80): + lines.append(f" a{i} = {i}\n") + lines.append(" return 0\n") + src = "".join(lines) + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], + source=src, + verbose=False, + max_lines=50, + ) + + result = splitter.get_rewritten_source() + assert result is not None + compile(result, "", "exec") + # Class methods use staticmethod and ClassName._ call + assert "@staticmethod" in result + assert "Foo._tail_work(" in result + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_llm_timeout_fallback(mock_anthropic): + # LLM call times out → fallback names + + mock_anthropic.Anthropic.return_value.messages.create.side_effect = _ApiTimeout( + "timed out" + ) + src = _make_long_func(60, "slow_func") + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], + source=src, + verbose=False, + max_lines=30, + ) + + result = splitter.get_rewritten_source() + assert result is not None + compile(result, "", "exec") + # Fallback name "slow_func_helper" used + assert "_slow_func_helper" in result + + +def test_function_splitter_empty_source(): + """FunctionSplitter created with no source does nothing.""" + splitter = FunctionSplitter([(1, 10)]) + assert splitter.get_rewritten_source() is None + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_max_iterations_loop_exhausted(mock_anthropic): + """Loop runs to completion (no break) when max iterations reached.""" + mock_anthropic.Anthropic.return_value.messages.create.return_value = ( + _make_mock_response(["helper"]) + ) + src = _make_long_func(80, "foo") + + # Patch _MAX_SPLIT_ITERATIONS to 1 → loop runs exactly once without breaking + # (break only occurs at START of next iteration when tasks=[]) + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + with patch("crispen.refactors.function_splitter._MAX_SPLIT_ITERATIONS", 1): + splitter = FunctionSplitter( + [(1, 1000)], source=src, verbose=False, max_lines=50 + ) + + result = splitter.get_rewritten_source() + assert result is not None + assert len(splitter.changes_made) == 1 + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_syntax_error_in_generated_output(mock_anthropic): + """If assembled output fails compile(), the change is not applied.""" + mock_anthropic.Anthropic.return_value.messages.create.return_value = ( + _make_mock_response(["helper"]) + ) + src = _make_long_func(80, "foo") + + import builtins as _builtins + + orig_compile = _builtins.compile + + def _selective_compile(source, filename, mode, *args, **kwargs): + if filename == "": + raise SyntaxError("mocked error for test") + return orig_compile(source, filename, mode, *args, **kwargs) + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + with patch("builtins.compile", side_effect=_selective_compile): + splitter = FunctionSplitter( + [(1, 1000)], source=src, verbose=False, max_lines=50 + ) + + assert splitter.get_rewritten_source() is None + + +@patch( + "crispen.refactors.function_splitter._has_new_undefined_names", return_value=True +) +@patch("crispen.llm_client.anthropic") +def test_function_splitter_pyflakes_rejects_output(mock_anthropic, mock_has_undef): + """If pyflakes detects new undefined names in output, the split is not applied.""" + mock_anthropic.Anthropic.return_value.messages.create.return_value = ( + _make_mock_response(["helper"]) + ) + src = _make_long_func(80, "foo") + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], source=src, verbose=False, max_lines=50 + ) + + # Pyflakes check returned True → split not applied + assert splitter.get_rewritten_source() is None + + +def test_engine_includes_function_splitter_no_op(tmp_path): + """FunctionSplitter is in _REFACTORS and runs without error for simple files.""" + from crispen.engine import run_engine + from crispen.config import CrispenConfig + + py_file = tmp_path / "sample.py" + py_file.write_text("def foo():\n return 1\n") + config = CrispenConfig(max_function_length=75) + msgs = list(run_engine({str(py_file): [(1, 2)]}, verbose=False, config=config)) + # No split needed — no messages expected (or just no errors) + assert all("FunctionSplitter" not in m for m in msgs) + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_method_self_needed_uses_instance_method(mock_anthropic): + """When every tail needs self, split into a regular instance method helper.""" + mock_anthropic.Anthropic.return_value.messages.create.return_value = ( + _make_mock_response(["tail_work"]) + ) + lines = ["class Foo:\n", " def method(self):\n"] + for i in range(40): + lines.append(f" a{i} = self.val + {i}\n") + lines.append(" return 0\n") + src = "".join(lines) + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], source=src, verbose=False, max_lines=20 + ) + + result = splitter.get_rewritten_source() + assert result is not None + compile(result, "", "exec") + assert "@staticmethod" not in result + assert "return self._tail_work(" in result + assert "def _tail_work(self" in result + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_skips_name_collision(mock_anthropic): + """Helper name colliding with an existing function causes the task to be dropped.""" + mock_anthropic.Anthropic.return_value.messages.create.return_value = ( + _make_mock_response(["helper"]) # would produce _helper + ) + # _helper already exists; the LLM would name the extracted helper "helper" + existing = "def _helper():\n pass\n\n\n" + src = existing + _make_long_func(80) + + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], source=src, verbose=False, max_lines=50 + ) + + # collision detected → task dropped → no rewrite + assert splitter.get_rewritten_source() is None + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_timing_recorded(mock_anthropic): + """FunctionSplitter records LLM timing after a successful split.""" + mock_anthropic.Anthropic.return_value.messages.create.return_value = ( + _make_mock_response(["process_tail"]) + ) + mock_anthropic.APIError = Exception + + src = _make_long_func(80) + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + splitter = FunctionSplitter( + [(1, 1000)], source=src, verbose=False, max_lines=50 + ) + + # The timing branch was hit; record_llm_call ran for the edit call. + assert splitter.stats.llm_edit_calls >= 1 + # The elapsed time dict was populated. + assert "edit" in splitter.stats.llm_elapsed_by_category + + +@patch("crispen.llm_client.anthropic") +def test_function_splitter_detailed_timing_print(mock_anthropic, capsys): + """FunctionSplitter prints per-call timing in verbose + detailed mode.""" + mock_client = mock_anthropic.Anthropic.return_value + mock_client.messages.create.return_value = _make_mock_response(["process_tail"]) + mock_anthropic.APIError = Exception + + src = _make_long_func(80) + with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): + # Construct without source so _analyze is not called yet. + splitter = FunctionSplitter([(1, 1000)], source="", verbose=True, max_lines=50) + splitter.timing = "detailed" + # Now trigger _analyze with detailed timing in place. + splitter._analyze(src) + + err = capsys.readouterr().err + assert "→ naming [" in err diff --git a/tests/function_splitter/test_unit.py b/tests/function_splitter/test_unit.py new file mode 100644 index 0000000..9e88494 --- /dev/null +++ b/tests/function_splitter/test_unit.py @@ -0,0 +1,4 @@ +from __future__ import annotations + +# Re-exported for backwards compatibility with external callers. +from tests.function_splitter.unit.test_collection import _make_mock_response # fmt: skip # noqa: F401, E501 diff --git a/tests/function_splitter/unit/__init__.py b/tests/function_splitter/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/function_splitter/unit/test_analysis.py b/tests/function_splitter/unit/test_analysis.py new file mode 100644 index 0000000..0beffa5 --- /dev/null +++ b/tests/function_splitter/unit/test_analysis.py @@ -0,0 +1,588 @@ +from __future__ import annotations +from unittest.mock import patch +import textwrap +from libcst.metadata import MetadataWrapper, PositionProvider +from crispen.refactors.function_splitter import ( + _count_body_lines, + _find_free_vars, + _has_nested_funcdef, + _has_new_undefined_names, + _has_yield, + _head_effective_lines, + _is_docstring_stmt, + _stmts_source, +) +import libcst as cst + + +def _parse_func(source: str): + """Return (body_stmts, positions, source_lines) for the first function. + + Uses a CSTVisitor to capture body_stmts from the wrapper's internal copy, + ensuring they match the keys in the positions dict. + """ + tree = cst.parse_module(source) + wrapper = MetadataWrapper(tree) + positions = wrapper.resolve(PositionProvider) + + class _Getter(cst.CSTVisitor): + METADATA_DEPENDENCIES = (PositionProvider,) + + def __init__(self): + self.stmts: list = [] + + def visit_FunctionDef(self, node: cst.FunctionDef) -> None: + if not self.stmts: # first function only + self.stmts = list(node.body.body) + + getter = _Getter() + wrapper.visit(getter) + source_lines = source.splitlines(keepends=True) + return getter.stmts, positions, source_lines + + +def _parse_stmt(src: str) -> cst.BaseStatement: + return cst.parse_module(src).body[0] + + +def test_is_docstring_triple_quoted(): + stmt = _parse_stmt('def f():\n """doc"""\n').body.body[0] + assert _is_docstring_stmt(stmt) is True + + +def test_is_docstring_single_quoted(): + stmt = _parse_stmt("def f():\n 'doc'\n").body.body[0] + assert _is_docstring_stmt(stmt) is True + + +def test_is_docstring_concatenated(): + stmt = _parse_stmt('def f():\n "foo" "bar"\n').body.body[0] + assert _is_docstring_stmt(stmt) is True + + +def test_is_docstring_non_docstring_expr(): + # A numeric literal is not a docstring + stmt = _parse_stmt("def f():\n 42\n").body.body[0] + assert _is_docstring_stmt(stmt) is False + + +def test_is_docstring_import(): + stmt = _parse_stmt("import os\n") + assert _is_docstring_stmt(stmt) is False + + +def test_is_docstring_assignment(): + stmt = _parse_stmt("x = 1\n") + assert _is_docstring_stmt(stmt) is False + + +def test_is_docstring_two_stmts_on_line(): + # Two statements on one line — len(body) != 1 + stmt = _parse_stmt("x = 1; y = 2\n") + assert _is_docstring_stmt(stmt) is False + + +def test_is_docstring_compound_stmt(): + # A compound statement (If) is not a SimpleStatementLine + src = "def f():\n if True:\n pass\n" + stmt = cst.parse_module(src).body[0].body.body[0] + assert _is_docstring_stmt(stmt) is False + + +def test_count_body_lines_no_docstring(): + src = "def foo():\n x = 1\n y = 2\n z = 3\n" + assert _count_body_lines(src) == 3 + + +def test_count_body_lines_with_docstring(): + src = 'def foo():\n """doc"""\n x = 1\n y = 2\n' + # docstring skipped; body is lines 2 (x=1) and 3 (y=2) + assert _count_body_lines(src) == 2 + + +def test_count_body_lines_multiline_docstring(): + src = 'def foo():\n """line1\n line2\n """\n x = 1\n' + # docstring spans lines 2-4; body starts at x=1 (line 5) + result = _count_body_lines(src) + assert result == 1 + + +def test_count_body_lines_only_docstring(): + # Body has only a docstring → effectively empty + src = 'def foo():\n """doc"""\n' + assert _count_body_lines(src) == 0 + + +def test_count_body_lines_parse_error(): + assert _count_body_lines("def f(\n !!invalid") == 0 + + +def test_count_body_lines_no_funcdef(): + # Module-level code, no function + assert _count_body_lines("x = 1\n") == 0 + + +def test_find_free_vars_all_local(): + src = "x = 1\ny = x + 1\n" + assert _find_free_vars(src) == [] + + +def test_find_free_vars_one_free(): + src = "y = external_var + 1\n" + result = _find_free_vars(src) + assert "external_var" in result + assert "y" not in result + + +def test_find_free_vars_builtins_excluded(): + src = "print(len([1, 2, 3]))\n" + result = _find_free_vars(src) + assert "print" not in result + assert "len" not in result + + +def test_find_free_vars_nested_function_not_recursed(): + src = "def inner():\n return outer_var\n" + # outer_var is used inside nested function — not recursed into + assert _find_free_vars(src) == [] + + +def test_find_free_vars_nested_class_not_recursed(): + src = "class Inner:\n x = class_var\n" + # class_var inside nested class — not recursed + assert _find_free_vars(src) == [] + + +def test_find_free_vars_for_target_not_free(): + src = "for item in some_list:\n pass\n" + result = _find_free_vars(src) + # item is a store, some_list is a load + assert "item" not in result + assert "some_list" in result + + +def test_find_free_vars_import_not_free(): + src = "import os\npath = os.getcwd()\n" + result = _find_free_vars(src) + # os is imported (stored), path is stored + assert "os" not in result + assert "path" not in result + + +def test_find_free_vars_import_from_not_free(): + src = "from os import path\nresult = path.join('a', 'b')\n" + result = _find_free_vars(src) + assert "path" not in result + + +def test_find_free_vars_parse_error(): + assert _find_free_vars("def f(\n !!") == [] + + +def test_find_free_vars_del_is_store(): + src = "del some_name\n" + # some_name has Del context (not Load) — not treated as free + result = _find_free_vars(src) + assert "some_name" not in result + + +def test_find_free_vars_augassign_free(): + # weight += 1 reads weight before writing — weight must come from outside + src = "weight += 1\n" + result = _find_free_vars(src) + assert "weight" in result + + +def test_find_free_vars_augassign_already_defined(): + # weight is unconditionally assigned first, so AugAssign doesn't need it free + src = "weight = 0\nweight += 1\n" + result = _find_free_vars(src) + assert "weight" not in result + + +def test_find_free_vars_augassign_subscript(): + # data[0] += 1: target is a subscript, data is loaded + src = "data[0] += 1\n" + result = _find_free_vars(src) + assert "data" in result + + +def test_find_free_vars_for_orelse(): + # for-else: orelse runs when loop completes normally + src = "for item in data:\n pass\nelse:\n fallback()\n" + result = _find_free_vars(src) + assert "item" not in result # for target is locally scoped + assert "data" in result + assert "fallback" in result # used in orelse, not locally defined + + +def test_find_free_vars_with_target(): + # with-statement target is locally scoped inside the body + src = "with open(filename) as fp:\n content = fp.read()\n" + result = _find_free_vars(src) + assert "fp" not in result # with target, locally scoped + assert "filename" in result # context_expr is free + + +def test_find_free_vars_with_no_target(): + # with-statement without 'as' clause + src = "with ctx_mgr():\n do_work()\n" + result = _find_free_vars(src) + assert "ctx_mgr" in result + assert "do_work" in result + + +def test_find_free_vars_except_handler_name(): + # except-handler name is locally bound for the handler body + src = "try:\n risky()\nexcept ValueError as exc:\n handle(exc)\n" + result = _find_free_vars(src) + assert "exc" not in result # locally bound by except clause + assert "risky" in result + assert "handle" in result + + +def test_find_free_vars_except_no_name(): + # bare except without 'as' binding + src = "try:\n risky()\nexcept ValueError:\n pass\n" + result = _find_free_vars(src) + assert "risky" in result + + +def test_find_free_vars_listcomp(): + # list comprehension: loop var is locally scoped + src = "result = [x * 2 for x in data]\n" + result = _find_free_vars(src) + assert "x" not in result # comprehension target, locally scoped + assert "data" in result + + +def test_find_free_vars_listcomp_with_filter(): + # comprehension with 'if' guard: threshold must come from outside + src = "result = [x for x in data if x > threshold]\n" + result = _find_free_vars(src) + assert "x" not in result + assert "data" in result + assert "threshold" in result + + +def test_find_free_vars_dictcomp(): + # dict comprehension: both key and value expressions are walked + src = "result = {k: v for k, v in pairs}\n" + result = _find_free_vars(src) + assert "k" not in result # tuple target of comprehension + assert "v" not in result + assert "pairs" in result + + +def test_find_free_vars_tuple_for_target(): + # tuple-unpacking for target: both names locally scoped + src = "for a, b in pairs:\n use(a, b)\n" + result = _find_free_vars(src) + assert "a" not in result + assert "b" not in result + assert "pairs" in result + + +def test_find_free_vars_subscript_assign_target(): + # subscript assignment target (e.g. data[0] = 1): _target_names returns {} + # so nothing is added to definitely_defined, but data is loaded + src = "data[0] = 1\n" + result = _find_free_vars(src) + assert "data" in result # data is loaded as the subscript base + + +def test_find_free_vars_annassign_with_value(): + # annotated assignment with value: name is definitely defined afterwards + src = "x: int = 5\ny = x + 1\n" + result = _find_free_vars(src) + assert "x" not in result + assert "y" not in result + + +def test_find_free_vars_annassign_no_value(): + # annotation without assignment: x is NOT definitely defined + src = "x: int\ny = x + 1\n" + result = _find_free_vars(src) + assert "x" in result # not assigned, so it is free + + +def test_find_free_vars_annassign_non_name_target(): + # annotated assignment where target is not a plain Name + src = "obj.attr: int = 5\n" + result = _find_free_vars(src) + assert "obj" in result # obj is loaded to set the attribute + + +def test_find_free_vars_conditional_store_is_free(): + # variables only assigned inside a conditional block remain free + src = "for i in xs:\n result = f(i)\nprint(result)\n" + result = _find_free_vars(src) + assert "result" in result # conditionally assigned → still free after loop + + +def test_find_free_vars_for_body_sequential(): + # a variable assigned then used in the same for-body iteration is not free + src = "for alias in names:\n name = alias.asname\n result.add(name)\n" + result = _find_free_vars(src) + assert "name" not in result # assigned before used in same loop body + assert "names" in result + assert "result" in result + + +def test_find_free_vars_if_branch(): + # if-body assignments do not propagate to after the if block + src = "if cond:\n x = 1\nelse:\n y = 2\nz = x + y\n" + result = _find_free_vars(src) + assert "cond" in result + assert "x" in result # only conditionally defined in if body + assert "y" in result # only conditionally defined in else body + + +def test_find_free_vars_while_loop(): + # while condition is free; while-else is walked + src = "while running:\n do_work()\nelse:\n finalize()\n" + result = _find_free_vars(src) + assert "running" in result + assert "do_work" in result + assert "finalize" in result + + +def test_find_free_vars_try_propagates(): + # variables assigned in a try body propagate to code after the try block + src = textwrap.dedent( + """\ + try: + lineno = compute() + except ValueError: + return + use(lineno) + """ + ) + result = _find_free_vars(src) + assert "lineno" not in result # defined in try body, propagated outward + assert "compute" in result + assert "use" in result + + +def test_find_free_vars_try_orelse(): + # try-else clause is walked with the try-body scope (x is defined there) + src = textwrap.dedent( + """\ + try: + x = compute() + except ValueError: + return + else: + use(x) + """ + ) + result = _find_free_vars(src) + assert "x" not in result # defined in try body, visible in else clause + assert "use" in result + assert "compute" in result + + +def test_find_free_vars_try_finally(): + # try with finally and no handlers: handlers loop is empty + src = "try:\n x = compute()\nfinally:\n cleanup()\n" + result = _find_free_vars(src) + assert "compute" in result + assert "cleanup" in result + assert "x" not in result # defined in try body, propagated + + +def test_find_free_vars_bare_except(): + # bare 'except:' has node.type = None (covers the None branch) + src = "try:\n risky()\nexcept:\n pass\n" + result = _find_free_vars(src) + assert "risky" in result + + +def test_find_free_vars_lambda_param_not_free(): + # lambda parameter must not appear as a free variable + src = "result = sorted(tasks, key=lambda t: t.name)\n" + result = _find_free_vars(src) + assert "t" not in result + assert "tasks" in result + + +def test_find_free_vars_lambda_vararg_not_free(): + # *args in lambda body — args is the vararg, not free + src = "f = lambda *args: list(args)\n" + result = _find_free_vars(src) + assert "args" not in result + + +def test_find_free_vars_lambda_kwarg_not_free(): + # **kw in lambda body — kw is the kwarg, not free + src = "f = lambda **kw: kw\n" + result = _find_free_vars(src) + assert "kw" not in result + + +def test_find_free_vars_lambda_default_outer_scope(): + # Default values are evaluated in the enclosing scope, not the lambda scope. + src = "f = lambda x=outer_val: x\n" + result = _find_free_vars(src) + assert "outer_val" in result # evaluated in outer scope → free + assert "x" not in result # lambda param → not free + + +def test_find_free_vars_lambda_kw_default_none_entry(): + # keyword-only param without a default: kw_defaults has a None entry + # lambda *, x, y=outer_val: x+y → kw_defaults=[None, outer_val_node] + src = "f = lambda *, x, y=outer_val: x + y\n" + result = _find_free_vars(src) + assert "x" not in result # kwonly param → not free + assert "y" not in result # kwonly param → not free + assert "outer_val" in result # kw_default evaluated in outer scope → free + + +def test_stmts_source_basic(): + src = "def foo():\n x = 1\n y = 2\n z = 3\n" + stmts, positions, lines = _parse_func(src) + result = _stmts_source(stmts[:2], lines, positions) + assert "x = 1" in result + assert "y = 2" in result + assert "z = 3" not in result + + +def test_stmts_source_empty(): + src = "def foo():\n x = 1\n" + _, positions, lines = _parse_func(src) + assert _stmts_source([], lines, positions) == "" + + +def test_stmts_source_dedented(): + src = "def foo():\n x = 1\n y = 2\n" + stmts, positions, lines = _parse_func(src) + result = _stmts_source(stmts, lines, positions) + # Should be dedented (no leading 4-space indent) + assert result.startswith("x = 1") or result.startswith("x = 1\n") + + +def test_head_effective_lines_no_docstring(): + src = "def foo():\n x = 1\n y = 2\n z = 3\n" + stmts, positions, lines = _parse_func(src) + # split_idx=2: head=[x,y], last=y at line 3, first=x at line 2 → 3-2+2=3 + result = _head_effective_lines(stmts, 2, positions, False) + assert result == 3 + + +def test_head_effective_lines_with_docstring_normal(): + src = 'def foo():\n """doc"""\n x = 1\n y = 2\n z = 3\n' + stmts, positions, lines = _parse_func(src) + # split_idx=3: head=[doc, x, y], first_non_doc=x at line 3, last=y at line 4 + # 4-3+2=3 + result = _head_effective_lines(stmts, 3, positions, True) + assert result == 3 + + +def test_head_effective_lines_only_docstring_in_head(): + # split_idx=1 with docstring: first_non_doc_idx=1 >= split_idx=1 → returns 1 + src = 'def foo():\n """doc"""\n x = 1\n y = 2\n' + stmts, positions, lines = _parse_func(src) + result = _head_effective_lines(stmts, 1, positions, True) + assert result == 1 + + +def test_has_yield_simple(): + src = "def gen():\n yield 1\n" + func = cst.parse_module(src).body[0] + assert _has_yield(func) is True + + +def test_has_yield_from(): + src = "def gen():\n yield from [1, 2]\n" + func = cst.parse_module(src).body[0] + assert _has_yield(func) is True + + +def test_has_yield_none(): + src = "def foo():\n return 1\n" + func = cst.parse_module(src).body[0] + assert _has_yield(func) is False + + +def test_has_yield_nested_not_counted(): + src = textwrap.dedent( + """\ + def foo(): + def inner(): + yield 1 + return inner + """ + ) + func = cst.parse_module(src).body[0] + # yield is inside nested function, should not count + assert _has_yield(func) is False + + +def test_has_nested_funcdef_with_nested(): + src = textwrap.dedent( + """\ + def outer(): + x = 1 + def inner(): + return x + return inner + """ + ) + func = cst.parse_module(src).body[0] + assert _has_nested_funcdef(func) is True + + +def test_has_nested_funcdef_without_nested(): + src = "def foo():\n x = 1\n return x\n" + func = cst.parse_module(src).body[0] + assert _has_nested_funcdef(func) is False + + +def test_has_nested_funcdef_first_stmt(): + # Nested funcdef is the very first statement in the body + src = textwrap.dedent( + """\ + def outer(): + def inner(): + pass + return inner() + """ + ) + func = cst.parse_module(src).body[0] + assert _has_nested_funcdef(func) is True + + +def test_find_free_vars_del_context(): + """del statement adds name to stores (else branch for non-Load contexts).""" + src = "del my_var\n" + result = _find_free_vars(src) + assert "my_var" not in result + + +def test_has_new_undefined_names_no_new(): + """No new undefined names → returns False.""" + before = "x = 1\ny = x + 1\n" + after = "x = 1\ny = x + 1\nz = y + 1\n" + assert _has_new_undefined_names(before, after) is False + + +def test_has_new_undefined_names_introduced(): + """After introduces an undefined name that before didn't have → returns True.""" + before = "x = 1\n" + after = "x = undefined_var\n" + assert _has_new_undefined_names(before, after) is True + + +def test_has_new_undefined_names_non_undefined_warning(): + """Non-UndefinedName pyflakes warning (e.g. UnusedImport) → returns False.""" + # An unused import produces an UnusedImport warning, not UndefinedName. + # This exercises the isinstance() False branch inside _Collector.flake. + before = "" + after = "import os\n" + assert _has_new_undefined_names(before, after) is False + + +def test_has_new_undefined_names_exception(): + """If pyflakes raises an unexpected exception, returns False (safe default).""" + with patch("pyflakes.api.check", side_effect=RuntimeError("boom")): + assert _has_new_undefined_names("x = 1\n", "y = 1\n") is False diff --git a/tests/function_splitter/unit/test_collection.py b/tests/function_splitter/unit/test_collection.py new file mode 100644 index 0000000..e5e558c --- /dev/null +++ b/tests/function_splitter/unit/test_collection.py @@ -0,0 +1,296 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch +import textwrap +from libcst.metadata import MetadataWrapper +from crispen.refactors.function_splitter import ( + _ApiTimeout, + _FuncInfo, + _FunctionCollector, + _SplitTask, + _extract_func_source, + _func_in_changed_range, + _llm_name_helpers, + _run_with_timeout, +) +import libcst as cst +import pytest + + +def _make_mock_response(names_list): + """Build a mock Anthropic message response for the name_helper_functions tool.""" + mock_block = MagicMock() + mock_block.type = "tool_use" + mock_block.name = "name_helper_functions" + mock_block.input = { + "names": [{"id": str(i), "name": n} for i, n in enumerate(names_list)] + } + mock_response = MagicMock() + mock_response.content = [mock_block] + return mock_response + + +def test_run_with_timeout_success(): + result = _run_with_timeout(lambda x: x * 2, 5, 21) + assert result == 42 + + +def test_run_with_timeout_exceeds(): + import time + + with pytest.raises(_ApiTimeout): + _run_with_timeout(lambda: time.sleep(10), timeout=0.05) + + +def test_run_with_timeout_propagates_exception(): + def _raise(): + raise ValueError("test error") + + with pytest.raises(ValueError, match="test error"): + _run_with_timeout(_raise, 5) + + +def _make_func_info(start, end): + """Create a minimal _FuncInfo for range tests.""" + mock_node = MagicMock() + return _FuncInfo( + node=mock_node, + start_line=start, + end_line=end, + class_name=None, + indent="", + original_params=[], + ) + + +def test_func_in_changed_range_overlaps(): + fi = _make_func_info(5, 15) + assert _func_in_changed_range(fi, [(1, 10)]) is True + + +def test_func_in_changed_range_no_overlap(): + fi = _make_func_info(5, 10) + assert _func_in_changed_range(fi, [(20, 30)]) is False + + +def test_func_in_changed_range_adjacent(): + fi = _make_func_info(5, 10) + assert _func_in_changed_range(fi, [(10, 20)]) is True + + +def test_extract_func_source(): + lines = ["line1\n", "line2\n", "line3\n", "line4\n"] + fi = _make_func_info(2, 3) + result = _extract_func_source(fi, lines) + assert result == "line2\nline3\n" + + +def test_function_collector_module_level(): + src = "def foo():\n x = 1\n" + tree = cst.parse_module(src) + wrapper = MetadataWrapper(tree) + collector = _FunctionCollector() + wrapper.visit(collector) + assert len(collector.functions) == 1 + assert collector.functions[0].node.name.value == "foo" + assert collector.functions[0].class_name is None + assert collector.functions[0].indent == "" + + +def test_function_collector_class_method(): + src = textwrap.dedent( + """\ + class Foo: + def bar(self): + pass + """ + ) + tree = cst.parse_module(src) + wrapper = MetadataWrapper(tree) + collector = _FunctionCollector() + wrapper.visit(collector) + assert len(collector.functions) == 1 + assert collector.functions[0].class_name == "Foo" + assert collector.functions[0].indent == " " + + +def test_function_collector_skips_async(): + src = "async def foo():\n pass\n" + tree = cst.parse_module(src) + wrapper = MetadataWrapper(tree) + collector = _FunctionCollector() + wrapper.visit(collector) + assert len(collector.functions) == 0 + + +def test_function_collector_skips_generator(): + src = "def gen():\n yield 1\n" + tree = cst.parse_module(src) + wrapper = MetadataWrapper(tree) + collector = _FunctionCollector() + wrapper.visit(collector) + assert len(collector.functions) == 0 + + +def test_function_collector_skips_nested_functions(): + # Functions with nested funcdefs are skipped entirely; inner functions + # (inside a function scope) are also skipped by the scope-kind guard. + src = textwrap.dedent( + """\ + def outer(): + def inner(): + pass + return inner + """ + ) + tree = cst.parse_module(src) + wrapper = MetadataWrapper(tree) + collector = _FunctionCollector() + wrapper.visit(collector) + # outer has a nested funcdef → skipped; inner is in a function scope → skipped + assert len(collector.functions) == 0 + + +def test_function_collector_captures_params(): + src = "def foo(a, b, c):\n pass\n" + tree = cst.parse_module(src) + wrapper = MetadataWrapper(tree) + collector = _FunctionCollector() + wrapper.visit(collector) + assert collector.functions[0].original_params == ["a", "b", "c"] + + +def _make_task(func_name, params=None, tail_source="return 0\n"): + """Create a minimal _SplitTask for testing _llm_name_helpers.""" + mock_node = MagicMock() + mock_node.name.value = func_name + fi = _FuncInfo( + node=mock_node, + start_line=1, + end_line=5, + class_name=None, + indent="", + original_params=[], + ) + return _SplitTask(fi, 1, params or [], tail_source=tail_source) + + +@patch("crispen.llm_client.anthropic") +def test_llm_name_helpers_success(mock_anthropic): + mock_response = _make_mock_response(["process_tail"]) + mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response + + tasks = [_make_task("my_func")] + client = mock_anthropic.Anthropic.return_value + result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) + assert result == ["process_tail"] + + +@patch("crispen.llm_client.anthropic") +def test_llm_name_helpers_result_none(mock_anthropic): + # LLM returns no tool use block + mock_response = MagicMock() + mock_response.content = [] + mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response + + tasks = [_make_task("my_func")] + client = mock_anthropic.Anthropic.return_value + result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) + # Falls back to "my_func_helper" + assert result == ["my_func_helper"] + + +@patch("crispen.llm_client.anthropic") +def test_llm_name_helpers_no_names_key(mock_anthropic): + # LLM returns tool use but without "names" key + mock_block = MagicMock() + mock_block.type = "tool_use" + mock_block.name = "name_helper_functions" + mock_block.input = {"something_else": []} + mock_response = MagicMock() + mock_response.content = [mock_block] + mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response + + tasks = [_make_task("my_func")] + client = mock_anthropic.Anthropic.return_value + result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) + assert result == ["my_func_helper"] + + +@patch("crispen.llm_client.anthropic") +def test_llm_name_helpers_strips_leading_underscore(mock_anthropic): + mock_response = _make_mock_response(["__private_name"]) + mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response + + tasks = [_make_task("foo")] + client = mock_anthropic.Anthropic.return_value + result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) + assert result == ["private_name"] + + +@patch("crispen.llm_client.anthropic") +def test_llm_name_helpers_all_underscores_uses_helper(mock_anthropic): + mock_response = _make_mock_response(["___"]) + mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response + + tasks = [_make_task("foo")] + client = mock_anthropic.Anthropic.return_value + result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) + assert result == ["helper"] + + +@patch("crispen.llm_client.anthropic") +def test_llm_name_helpers_bad_item_skipped(mock_anthropic): + # One item has a TypeError (e.g. name is not a string) + mock_block = MagicMock() + mock_block.type = "tool_use" + mock_block.name = "name_helper_functions" + mock_block.input = { + "names": [{"id": "0", "name": None}] # None.lstrip() raises AttributeError + } + mock_response = MagicMock() + mock_response.content = [mock_block] + mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response + + tasks = [_make_task("foo")] + client = mock_anthropic.Anthropic.return_value + result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) + # Falls back to "foo_helper" because item had AttributeError + assert result == ["foo_helper"] + + +@patch("crispen.llm_client.anthropic") +def test_llm_name_helpers_with_class_name(mock_anthropic): + mock_response = _make_mock_response(["process"]) + mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response + + mock_node = MagicMock() + mock_node.name.value = "method" + fi = _FuncInfo( + node=mock_node, + start_line=1, + end_line=5, + class_name="MyClass", + indent=" ", + original_params=[], + ) + task = _SplitTask(fi, 1, [], tail_source="return 0\n") + client = mock_anthropic.Anthropic.return_value + result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", [task]) + assert result == ["process"] + + +@patch("crispen.llm_client.anthropic") +def test_llm_name_helpers_with_timing_out(mock_anthropic): + """_llm_name_helpers appends result to _timing_out when provided.""" + mock_response = _make_mock_response(["process_tail"]) + mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response + mock_anthropic.APIError = Exception + + tasks = [_make_task("my_func")] + client = mock_anthropic.Anthropic.return_value + timing: list = [] + result = _llm_name_helpers( + client, "claude-sonnet-4-6", "anthropic", tasks, _timing_out=timing + ) + assert result == ["process_tail"] + assert len(timing) == 1 diff --git a/tests/function_splitter/unit/test_splitting.py b/tests/function_splitter/unit/test_splitting.py new file mode 100644 index 0000000..fd429b9 --- /dev/null +++ b/tests/function_splitter/unit/test_splitting.py @@ -0,0 +1,328 @@ +from __future__ import annotations +import textwrap +from crispen.refactors.function_splitter import ( + _choose_best_split, + _find_valid_splits, + _generate_call, + _generate_helper_source, + _module_global_names, +) +from .test_analysis import _parse_func + + +def test_find_valid_splits_all_valid(): + src = "def foo():\n a = 1\n b = 2\n c = 3\n d = 4\n" + stmts, positions, lines = _parse_func(src) + # With a very loose limit, all splits should be valid + result = _find_valid_splits(stmts, positions, max_lines=1000) + assert len(result) > 0 + # Ordered latest first + assert result == sorted(result, reverse=True) + + +def test_find_valid_splits_none_valid(): + # max_lines=1 means even a 1-stmt head (+ return call = 2 lines) is invalid + src = "def foo():\n a = 1\n b = 2\n c = 3\n" + stmts, positions, lines = _parse_func(src) + result = _find_valid_splits(stmts, positions, max_lines=1) + assert result == [] + + +def test_find_valid_splits_stops_at_max_candidates(): + # 7 statements → iterates from 6 down, stops after 5 valid candidates + src = "def foo():\n" + "".join(f" a{i} = {i}\n" for i in range(7)) + stmts, positions, lines = _parse_func(src) + result = _find_valid_splits(stmts, positions, max_lines=1000) + assert len(result) == 5 + + +def test_find_valid_splits_fewer_than_max(): + # 4 statements → at most 3 valid splits (indices 3, 2, 1) + src = "def foo():\n a = 1\n b = 2\n c = 3\n d = 4\n" + stmts, positions, lines = _parse_func(src) + result = _find_valid_splits(stmts, positions, max_lines=1000) + assert 1 <= len(result) <= 3 + + +def test_find_valid_splits_empty_body(): + # Should not crash with an empty list (though normally not called) + result = _find_valid_splits([], {}, max_lines=1000) + assert result == [] + + +def test_find_valid_splits_nested_funcdef_restricts_upper(): + # First nested funcdef at index 2 → valid splits only at indices ≤ 2. + src = textwrap.dedent( + """\ + def outer(): + a = 1 + b = 2 + def inner(): + pass + c = 3 + d = 4 + """ + ) + stmts, positions, lines = _parse_func(src) + # body_stmts: [a=1, b=2, def inner, c=3, d=4] + # First nested funcdef at index 2 → upper=2 → range(2, 0, -1) = [2, 1] + result = _find_valid_splits(stmts, positions, max_lines=1000) + assert all(i <= 2 for i in result) + assert 3 not in result + assert 4 not in result + + +def test_choose_best_split_fewest_params(): + # Two splits: one has free vars, one doesn't + src = textwrap.dedent( + """\ + def foo(external): + a = 1 + b = external + 1 + """ + ) + stmts, positions, lines = _parse_func(src) + # split_idx=1: tail=[b=external+1] → free vars: [external] + # split_idx=2: tail=[] → but we need at least 1 stmt in tail, + # so valid splits are [1] only for 2-stmt function + # Let's use 3 stmts with different free var counts + src2 = textwrap.dedent( + """\ + def foo(ext): + a = 1 + b = ext + 1 + c = a + b + """ + ) + stmts2, positions2, lines2 = _parse_func(src2) + # split_idx=1: tail=[b=ext+1, c=a+b] → free vars: [a, ext] (a from head) + # Actually 'a' is assigned in head (split_idx=1 → head=[a=1]) and used in tail + # So tail [b=ext+1, c=a+b] has free vars: [a, ext] + # split_idx=2: tail=[c=a+b] → free vars: [a, b] (assigned in head) + # Wait no, head=[a=1, b=ext+1] so tail=[c=a+b] has free vars: [a, b] + # split_idx=3: not valid (needs at least 1 in tail) + # So split_idx=1 has 2 free vars [a, ext], split_idx=2 has 2 free vars [a, b] + # Tie → choose earliest in list = latest split = 2 + valid_splits = [2, 1] # latest first + split_idx, params, _ = _choose_best_split( + stmts2, valid_splits, lines2, positions2, ["ext"] + ) + # Both have 2 free vars, tie broken by latest (first in list) = 2 + assert split_idx == 2 + + +def test_choose_best_split_fewer_params_wins(): + # Use a source where one split clearly has fewer params + src = textwrap.dedent( + """\ + def foo(): + a = 1 + b = 2 + c = a + b + """ + ) + stmts, positions, lines = _parse_func(src) + # split_idx=1: tail=[b=2, c=a+b] → free vars: [a] (1 free var) + # split_idx=2: tail=[c=a+b] → free vars: [a, b] (2 free vars) + valid_splits = [2, 1] + split_idx, params, _ = _choose_best_split(stmts, valid_splits, lines, positions, []) + # split_idx=1 has 1 free var (a) vs split_idx=2 has 2 free vars (a, b) + assert split_idx == 1 + assert params == ["a"] + + +def test_choose_best_split_single_candidate(): + src = "def foo():\n x = 1\n y = 2\n" + stmts, positions, lines = _parse_func(src) + split_idx, params, _ = _choose_best_split(stmts, [1], lines, positions, []) + assert split_idx == 1 + + +def test_choose_best_split_self_in_tail_returns_instance_method(): + # Tail requires self → extracted as instance method, not static + src = textwrap.dedent( + """\ + class Foo: + def method(self, x): + a = 1 + b = self.value + a + """ + ) + stmts, positions, lines = _parse_func(src) + # split_idx=1: tail=[b = self.value + a] → free: [a, self] → instance method + result = _choose_best_split(stmts, [1], lines, positions, ["self", "x"]) + assert result is not None + split_idx, params, is_instance_method = result + assert split_idx == 1 + assert is_instance_method is True + assert "self" not in params # self is implicit, not in params list + assert "a" in params # a is still a real param + + +def test_choose_best_split_empty_splits_returns_none(): + # No valid split candidates → None returned + src = "def foo():\n x = 1\n y = 2\n" + stmts, positions, lines = _parse_func(src) + result = _choose_best_split(stmts, [], lines, positions, []) + assert result is None + + +def test_choose_best_split_filters_module_globals(): + # Tail references a module-level import; it must not appear in params. + src = textwrap.dedent( + """\ + def foo(): + x = 1 + y = os.path.join("a", "b") + """ + ) + stmts, positions, lines = _parse_func(src) + # Without filtering: "os" would be a free var of the tail. + # With module_globals={"os"}: "os" is filtered out → params = [] + result = _choose_best_split(stmts, [1], lines, positions, [], module_globals={"os"}) + assert result is not None + _, params, _ = result + assert "os" not in params + + +def test_module_global_names_imports(): + source = "import ast\nfrom pathlib import Path\nimport libcst as cst\n" + result = _module_global_names(source) + assert "ast" in result + assert "Path" in result + assert "cst" in result + + +def test_module_global_names_functions_and_classes(): + source = "def foo():\n pass\n\nclass Bar:\n pass\n" + result = _module_global_names(source) + assert "foo" in result + assert "Bar" in result + + +def test_module_global_names_assignments(): + source = "_CONST = frozenset()\nVALUE: int = 42\n" + result = _module_global_names(source) + assert "_CONST" in result + assert "VALUE" in result + + +def test_module_global_names_syntax_error(): + result = _module_global_names("def foo(") + assert result == set() + + +def test_module_global_names_tuple_assign_target_not_collected(): + # Tuple-unpacking: Assign target is a Tuple node, not a Name → skipped + source = "a, b = 1, 2\n" + result = _module_global_names(source) + assert "a" not in result + assert "b" not in result + + +def test_module_global_names_ann_assign_non_name_target_skipped(): + # AnnAssign where target is an Attribute, not a Name → skipped + source = "Foo.x: int\n" + result = _module_global_names(source) + assert "x" not in result + + +def test_generate_helper_source_with_staticmethod(): + result = _generate_helper_source( + name="process", + params=["x", "y"], + tail_source="return x + y\n", + func_indent=" ", + is_static=True, + add_docstring=False, + ) + assert "@staticmethod" in result + assert "def _process(x, y):" in result + assert "return x + y" in result + assert result.startswith(" @staticmethod") + + +def test_generate_helper_source_without_staticmethod(): + result = _generate_helper_source( + name="process", + params=["x"], + tail_source="return x * 2\n", + func_indent="", + is_static=False, + add_docstring=False, + ) + assert "@staticmethod" not in result + assert "def _process(x):" in result + assert "return x * 2" in result + + +def test_generate_helper_source_with_docstring(): + result = _generate_helper_source( + name="process", + params=[], + tail_source="return 42\n", + func_indent="", + is_static=False, + add_docstring=True, + ) + assert '"""' in result + assert "return 42" in result + + +def test_generate_helper_source_instance_method(): + result = _generate_helper_source( + name="process", + params=["a"], + tail_source="return self.x + a\n", + func_indent=" ", + is_static=False, + add_docstring=False, + is_instance_method=True, + ) + assert "@staticmethod" not in result + assert "def _process(self, a):" in result + assert "return self.x + a" in result + + +def test_generate_helper_source_indentation_correct(): + result = _generate_helper_source( + name="helper", + params=[], + tail_source="x = 1\ny = 2\n", + func_indent=" ", + is_static=False, + add_docstring=False, + ) + # Body should be indented by 8 spaces (func_indent=4 + body_indent=4) + assert " x = 1" in result + assert " y = 2" in result + + +def test_generate_call_with_class(): + result = _generate_call("helper", ["x", "y"], "MyClass", " ") + assert result == " return MyClass._helper(x, y)" + + +def test_generate_call_module_level(): + result = _generate_call("helper", ["a"], None, " ") + assert result == " return _helper(a)" + + +def test_generate_call_no_params(): + result = _generate_call("do_work", [], None, " ") + assert result == " return _do_work()" + + +def test_generate_call_class_no_params(): + result = _generate_call("do_work", [], "Foo", " ") + assert result == " return Foo._do_work()" + + +def test_generate_call_instance_method(): + result = _generate_call("process", ["a", "b"], "MyClass", " ", True) + assert result == " return self._process(a, b)" + + +def test_generate_call_instance_method_no_params(): + result = _generate_call("process", [], "MyClass", " ", True) + assert result == " return self._process()" diff --git a/tests/patch_rewriter/__init__.py b/tests/patch_rewriter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/patch_rewriter/const_refs/__init__.py b/tests/patch_rewriter/const_refs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/patch_rewriter/const_refs/test_callgraph_voting.py b/tests/patch_rewriter/const_refs/test_callgraph_voting.py new file mode 100644 index 0000000..37fc798 --- /dev/null +++ b/tests/patch_rewriter/const_refs/test_callgraph_voting.py @@ -0,0 +1,289 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch as mock_patch +from crispen.llm_client import LLMCallResult +from crispen.patch_rewriter import ( + _FLContext, + _callgraph_update_file, + _get_const_votes_from_rewrite, + _process_file_source, +) +from ..helpers import ( + _CFG, + _PATCH_CALL_TOOL, + _VERIFY_OK, + _make_cuf_index, + _make_ref, + _ok, +) + + +def test_get_const_votes_empty_refs(): + """No const_refs → empty dict, no parsing needed.""" + assert _get_const_votes_from_rewrite("def test_f(): pass\n", []) == {} + + +def test_get_const_votes_syntax_error(): + """Unparseable func_text → empty dict (SyntaxError branch).""" + refs = [_make_ref("TARGET", "pkg.old.X")] + assert _get_const_votes_from_rewrite("def f(:\n", refs) == {} + + +def test_get_const_votes_no_function_in_body(): + """Valid Python but no FunctionDef/AsyncFunctionDef → empty dict.""" + refs = [_make_ref("TARGET", "pkg.old.X")] + result = _get_const_votes_from_rewrite("x = 1\n", refs) + assert result == {} + + +def test_get_const_votes_non_call_decorator_skipped(): + """A bare-name decorator (not a Call node) is skipped without error.""" + code = "@pytest.mark.slow\n@patch(TARGET)\ndef test_f(m): pass\n" + refs = [_make_ref("TARGET", "pkg.old.X")] + # TARGET still present as Name → no vote entry (const unchanged). + result = _get_const_votes_from_rewrite(code, refs) + assert result == {} + + +def test_get_const_votes_non_patch_call_skipped(): + """A Call decorator whose func is not 'patch' is skipped.""" + code = "@other_decorator('pkg.old.X')\ndef test_f(m): pass\n" + refs = [_make_ref("TARGET", "pkg.old.X")] + result = _get_const_votes_from_rewrite(code, refs) + assert result == {} + + +def test_get_const_votes_no_args_decorator_skipped(): + """@patch() with no args → skipped (no args branch).""" + code = "@patch()\ndef test_f(): pass\n" + refs = [_make_ref("TARGET", "pkg.old.X")] + result = _get_const_votes_from_rewrite(code, refs) + assert result == {} + + +def test_get_const_votes_module_attr_const_name(): + """@patch(module.CONST) style (Attribute node) → const name recorded correctly.""" + # Attribute form used when const is module-aliased after _restore_const_refs. + code = "@patch(module.TARGET)\ndef test_f(m): pass\n" + refs = [_make_ref("module.TARGET", "pkg.old.X")] + result = _get_const_votes_from_rewrite(code, refs) + # const still present as module.TARGET → no vote entry. + assert result == {} + + +def test_get_const_votes_successful_vote(): + """LLM updated the path → new literal collected, vote returned.""" + refs = [_make_ref("TARGET", "pkg.mod.X")] + code = '@patch("pkg.mod.sub.X")\ndef test_f(m): pass\n' + result = _get_const_votes_from_rewrite(code, refs) + assert result == {"pkg.mod.X": "pkg.mod.sub.X"} + + +def test_get_const_votes_deeply_nested_attr_skipped(): + """@patch(module.sub.CONST) where arg0 is Attribute(Attribute) — falls through + all elif branches (663->647 coverage: the third elif is False for this form).""" + # module.sub.CONST: arg0.value is Attribute, not Name → elif at 661 is False; + # arg0 is not Constant → elif at 663 is False → no match, loop continues. + code = "@patch(module.sub.CONST)\ndef test_f(m): pass\n" + refs = [_make_ref("TARGET", "pkg.mod.X")] + result = _get_const_votes_from_rewrite(code, refs) + # No string literal collected, TARGET still absent → no vote. + assert result == {} + + +@mock_patch(_PATCH_CALL_TOOL) +def test_rewrite_non_participant_casts_keep_old_vote(mock_call, tmp_path): + """A function that fails the rewrite (edit_failure) still casts a keep-old + vote, preventing a const from being updated when only one of two users + successfully renamed it. + + Scenario: + test_a: classify → rename X → after.X; verify OK → string_swap_results. + test_b: classify → needs_rewrite → rewrite → LLM returns None (failure) + → edit_failure → NOT in string_swap_results. + + Without the keep-old fix: X proposals = {"after.X"} (single) → const updated. + With the keep-old fix: X proposals = {"after.X", "old.X"} → conflicting → + test_a inlined, const definition unchanged. + """ + src = ( + 'TARGET = "crispen.before.X"\n' + "\n" + "@patch(TARGET)\n" + "def test_a(mock_x):\n" + " pass\n" + "\n" + "@patch(TARGET)\n" + "def test_b(mock_x):\n" + " pass\n" + ) + scan = str(tmp_path / "test_foo.py") + # test_a: classify → rename → verify OK. + # test_b: classify → needs_rewrite → rewrite attempt → None response (failure). + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.after.X"}, + } + ), + _ok(_VERIFY_OK), + _ok({"needs_rewrite": True}), + LLMCallResult(tool_input=None, elapsed=0.0, input_tokens=0, output_tokens=0), + ] + result, changed, cross = _process_file_source( + src, + {"crispen.before.X"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + ) + # test_b failed → keep-old vote → conflicting → const NOT updated. + assert 'TARGET = "crispen.before.X"' in result + # test_a's decorator inlined individually. + assert '@patch("crispen.after.X")' in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_rewrite_non_participant_cross_file_ref_skipped(mock_call, tmp_path): + """Non-participant with a cross-file const ref: the ref.source_file != scan_file_abs + branch is False, so no same-file keep-old vote is cast (3171->3170 branch). + + test_a: succeeds (in string_swap_results). + test_b: fails (not in string_swap_results). test_b's const is defined in + helpers.py (cross-file) so the keep-old loop skips it — no + same_file_proposals entry for that ref. + """ + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") + src = ( + "from .helpers import TARGET\n" + "\n" + "@patch(TARGET)\n" + "def test_a(mock_x):\n" + " pass\n" + "\n" + "@patch(TARGET)\n" + "def test_b(mock_x):\n" + " pass\n" + ) + scan = str(tmp_path / "test_foo.py") + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.after.X"}, + } + ), + _ok(_VERIFY_OK), + _ok({"needs_rewrite": True}), + LLMCallResult(tool_input=None, elapsed=0.0, input_tokens=0, output_tokens=0), + ] + result, changed, cross = _process_file_source( + src, + {"crispen.before.X"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + repo_root=str(tmp_path), + ) + # Cross-file ref → no same-file conflict → cross updated (test_a's rename wins). + helpers_abs = str(helpers.resolve()) + assert helpers_abs in cross + assert cross[helpers_abs] == {"crispen.before.X": "crispen.after.X"} + + +def test_callgraph_const_ref_no_scan_file_skips_keep_old(tmp_path): + """scan_file=None → scan_file_abs="" (falsy) → the keep-old block is skipped + entirely (3420->3426 branch). BFS ambiguous functions don't cast any vote. + The string literal path is still updated normally. + """ + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", + "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", + }, + new_module_paths={ + "placement.py": "pkg.placement", + "conflict.py": "pkg.conflict", + }, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = ( + "from pkg.placement import helper\n" + "from pkg.conflict import resolve\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(m):\n" + " helper()\n" + " resolve()\n" + ) + # scan_file=None → scan_file_abs="" → keep-old block skipped; string literal + # unchanged because BFS is ambiguous (no resolved result). + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + [ctx], + scan_file=None, + index=None, + ) + assert not changed + + +def test_callgraph_const_ref_ambiguous_casts_keep_old_vote(tmp_path): + """When BFS finds multiple candidates for a const-backed path (ambiguous), + the function casts a keep-old vote so a shared constant isn't updated to a + value that is wrong for the ambiguous function. + + test_a: calls helper() → placement (single BFS candidate) → vote "placement". + test_b: calls helper() + resolve() → placement AND conflict (2 BFS candidates + for use_fn) → ambiguous → keep-old vote. + Proposals for _PATCH_USE: {"pkg.placement.use_fn", "pkg.orig.use_fn"} → conflict + → constant NOT updated; test_a gets its decorator inlined individually. + """ + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", + "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", + }, + new_module_paths={ + "placement.py": "pkg.placement", + "conflict.py": "pkg.conflict", + }, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = ( + "from pkg.placement import helper\n" + "from pkg.conflict import resolve\n" + '_PATCH_USE = "pkg.orig.use_fn"\n' + "@patch(_PATCH_USE)\n" + "def test_a(m):\n" + " helper()\n" + "\n" + "@patch(_PATCH_USE)\n" + "def test_b(m):\n" + " helper()\n" + " resolve()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, {"pkg.orig.use_fn"}, [ctx], scan_file=scan, index=index + ) + # test_b is ambiguous → keep-old vote → conflict with test_a's rename vote. + # Constant definition must NOT be updated. + assert '_PATCH_USE = "pkg.orig.use_fn"' in result + # test_a's decorator IS inlined (it had a resolved rename). + assert '@patch("pkg.placement.use_fn")' in result diff --git a/tests/patch_rewriter/const_refs/test_find_const_refs.py b/tests/patch_rewriter/const_refs/test_find_const_refs.py new file mode 100644 index 0000000..fa086f7 --- /dev/null +++ b/tests/patch_rewriter/const_refs/test_find_const_refs.py @@ -0,0 +1,445 @@ +from __future__ import annotations +from crispen.patch_rewriter import ( + _build_attr_const_map, + _build_const_map, + _build_local_const_map, + _find_test_functions_to_update, + _resolve_import_to_file, +) + + +def test_local_const_map_string_assignment(): + src = 'TARGET = "myapp.service.MyClass"\n' + result = _build_local_const_map(src) + assert result == {"TARGET": "myapp.service.MyClass"} + + +def test_local_const_map_non_string_excluded(): + src = "TARGET = 42\n" + assert _build_local_const_map(src) == {} + + +def test_local_const_map_multi_target_excluded(): + # a = b = "value" has two targets → not included. + src = 'a = b = "value"\n' + assert _build_local_const_map(src) == {} + + +def test_local_const_map_syntax_error(): + assert _build_local_const_map("def f(:\n") == {} + + +def test_local_const_map_empty_source(): + assert _build_local_const_map("") == {} + + +def test_local_const_map_last_wins(): + src = 'X = "first"\nX = "second"\n' + assert _build_local_const_map(src)["X"] == "second" + + +def test_local_const_map_annotated_assignment(): + src = 'TARGET: str = "myapp.service.MyClass"\n' + assert _build_local_const_map(src) == {"TARGET": "myapp.service.MyClass"} + + +def test_local_const_map_annotated_non_string_excluded(): + src = "TARGET: int = 42\n" + assert _build_local_const_map(src) == {} + + +def test_local_const_map_annotated_no_value_excluded(): + # Bare annotation with no value: ``TARGET: str`` — ast.AnnAssign with value=None + src = "TARGET: str\n" + assert _build_local_const_map(src) == {} + + +def test_resolve_relative_level1_py(tmp_path): + # from .sub import NAME — sub.py exists + (tmp_path / "sub.py").write_text("X = 1\n", encoding="utf-8") + scan = str(tmp_path / "test_foo.py") + result = _resolve_import_to_file("sub", 1, scan, None) + assert result == str(tmp_path / "sub.py") + + +def test_resolve_relative_level1_init(tmp_path): + # from .pkg import NAME — pkg/__init__.py exists + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("", encoding="utf-8") + scan = str(tmp_path / "test_foo.py") + result = _resolve_import_to_file("pkg", 1, scan, None) + assert result == str(pkg / "__init__.py") + + +def test_resolve_relative_level1_no_module(tmp_path): + # from . import NAME — finds __init__.py in same dir + (tmp_path / "__init__.py").write_text("", encoding="utf-8") + scan = str(tmp_path / "test_foo.py") + result = _resolve_import_to_file(None, 1, scan, None) + assert result == str(tmp_path / "__init__.py") + + +def test_resolve_relative_level2(tmp_path): + # from ..sub import NAME — goes up one level + parent = tmp_path / "parent" + parent.mkdir() + child = parent / "child" + child.mkdir() + (parent / "sub.py").write_text("X = 1\n", encoding="utf-8") + scan = str(child / "test_foo.py") + result = _resolve_import_to_file("sub", 2, scan, None) + assert result == str(parent / "sub.py") + + +def test_resolve_relative_not_found(tmp_path): + scan = str(tmp_path / "test_foo.py") + assert _resolve_import_to_file("missing", 1, scan, None) is None + + +def test_resolve_relative_no_module_no_init(tmp_path): + scan = str(tmp_path / "test_foo.py") + assert _resolve_import_to_file(None, 1, scan, None) is None + + +def test_resolve_absolute_found(tmp_path): + pkg = tmp_path / "mypkg" + pkg.mkdir() + (pkg / "helpers.py").write_text("X = 1\n", encoding="utf-8") + scan = str(tmp_path / "tests" / "test_foo.py") + result = _resolve_import_to_file("mypkg.helpers", 0, scan, str(tmp_path)) + assert result == str(pkg / "helpers.py") + + +def test_resolve_absolute_no_repo_root(tmp_path): + scan = str(tmp_path / "test_foo.py") + assert _resolve_import_to_file("mypkg.helpers", 0, scan, None) is None + + +def test_resolve_absolute_no_module(tmp_path): + scan = str(tmp_path / "test_foo.py") + assert _resolve_import_to_file(None, 0, scan, str(tmp_path)) is None + + +def test_resolve_absolute_not_found(tmp_path): + scan = str(tmp_path / "test_foo.py") + assert _resolve_import_to_file("no.such.module", 0, scan, str(tmp_path)) is None + + +def test_build_const_map_same_file(tmp_path): + src = 'TARGET = "myapp.service.MyClass"\n' + scan = str(tmp_path / "test_foo.py") + result = _build_const_map(src, scan, None) + val, def_file = result["TARGET"] + assert val == "myapp.service.MyClass" + assert def_file == str((tmp_path / "test_foo.py").resolve()) + + +def test_build_const_map_cross_file(tmp_path): + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "myapp.service.MyClass"\n', encoding="utf-8") + src = "from .helpers import TARGET\n" + scan = str(tmp_path / "test_foo.py") + result = _build_const_map(src, scan, None) + val, def_file = result["TARGET"] + assert val == "myapp.service.MyClass" + assert def_file == str(helpers.resolve()) + + +def test_build_const_map_alias(tmp_path): + helpers = tmp_path / "helpers.py" + helpers.write_text('X = "myapp.service.MyClass"\n', encoding="utf-8") + src = "from .helpers import X as MY_TARGET\n" + scan = str(tmp_path / "test_foo.py") + result = _build_const_map(src, scan, None) + assert "MY_TARGET" in result + assert result["MY_TARGET"][0] == "myapp.service.MyClass" + + +def test_build_const_map_local_priority(tmp_path): + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "imported.value"\n', encoding="utf-8") + src = 'TARGET = "local.value"\nfrom .helpers import TARGET\n' + scan = str(tmp_path / "test_foo.py") + result = _build_const_map(src, scan, None) + assert result["TARGET"][0] == "local.value" + + +def test_build_const_map_star_import_skipped(tmp_path): + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "myapp.service.MyClass"\n', encoding="utf-8") + src = "from .helpers import *\n" + scan = str(tmp_path / "test_foo.py") + result = _build_const_map(src, scan, None) + assert result == {} + + +def test_build_const_map_import_file_not_found(tmp_path): + src = "from .missing import TARGET\n" + scan = str(tmp_path / "test_foo.py") + result = _build_const_map(src, scan, None) + assert result == {} + + +def test_build_const_map_import_oserror(tmp_path): + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "val"\n', encoding="utf-8") + helpers.chmod(0o000) + try: + src = "from .helpers import TARGET\n" + scan = str(tmp_path / "test_foo.py") + result = _build_const_map(src, scan, None) + assert result == {} + finally: + helpers.chmod(0o644) + + +def test_build_const_map_syntax_error(): + result = _build_const_map("def f(:\n", "/some/file.py", None) + assert result == {} + + +def test_build_const_map_no_const_in_import(tmp_path): + helpers = tmp_path / "helpers.py" + helpers.write_text("def some_func(): pass\n", encoding="utf-8") + src = "from .helpers import some_func\n" + scan = str(tmp_path / "test_foo.py") + result = _build_const_map(src, scan, None) + assert result == {} + + +def test_build_attr_const_map_basic(tmp_path): + """``import constants`` resolves string constants from the module file.""" + constants_file = tmp_path / "constants.py" + constants_file.write_text('TARGET = "myapp.service.MyClass"\n', encoding="utf-8") + src = "import constants\n" + scan = str(tmp_path / "test_foo.py") + result = _build_attr_const_map(src, scan, str(tmp_path)) + assert "constants" in result + val, def_file = result["constants"]["TARGET"] + assert val == "myapp.service.MyClass" + assert def_file == str(constants_file.resolve()) + + +def test_build_attr_const_map_with_alias(tmp_path): + """``import pkg.constants as C`` maps alias ``C`` to module constants.""" + pkg = tmp_path / "pkg" + pkg.mkdir() + constants_file = pkg / "constants.py" + constants_file.write_text('TARGET = "myapp.svc.MyClass"\n', encoding="utf-8") + src = "import pkg.constants as C\n" + scan = str(tmp_path / "test_foo.py") + result = _build_attr_const_map(src, scan, str(tmp_path)) + assert "C" in result + assert result["C"]["TARGET"][0] == "myapp.svc.MyClass" + + +def test_build_attr_const_map_no_file(tmp_path): + """Import that doesn't resolve to a file → skipped, empty result.""" + src = "import missing_module\n" + scan = str(tmp_path / "test_foo.py") + result = _build_attr_const_map(src, scan, str(tmp_path)) + assert result == {} + + +def test_build_attr_const_map_oserror(tmp_path): + """Module file exists but is unreadable → skipped.""" + constants_file = tmp_path / "constants.py" + constants_file.write_text('TARGET = "val"\n', encoding="utf-8") + constants_file.chmod(0o000) + try: + src = "import constants\n" + scan = str(tmp_path / "test_foo.py") + result = _build_attr_const_map(src, scan, str(tmp_path)) + assert result == {} + finally: + constants_file.chmod(0o644) + + +def test_build_attr_const_map_syntax_error(): + """SyntaxError in source → empty result.""" + assert _build_attr_const_map("def f(:\n", "/some/file.py", None) == {} + + +def test_build_attr_const_map_non_import_skipped(tmp_path): + """Non-``import`` statements (from-imports, assignments) are skipped.""" + constants_file = tmp_path / "constants.py" + constants_file.write_text('TARGET = "val"\n', encoding="utf-8") + # Only a from-import and an assignment; no plain ``import`` → empty. + src = 'from .constants import TARGET\nX = "y"\n' + scan = str(tmp_path / "test_foo.py") + result = _build_attr_const_map(src, scan, str(tmp_path)) + assert result == {} + + +def test_find_const_ref_same_file(tmp_path): + """@patch(CONST) where CONST is in the same file → collected, substituted.""" + src = ( + 'TARGET = "crispen.before.X"\n\n@patch(TARGET)\ndef test_f(mock_x):\n pass\n' + ) + scan = str(tmp_path / "test_foo.py") + result = _find_test_functions_to_update(src, {"crispen.before.X"}, scan_file=scan) + assert len(result) == 1 + assert result[0].function_name == "test_f" + # full_text sent to LLM has the value inlined + assert '"crispen.before.X"' in result[0].full_text + assert "TARGET" not in result[0].full_text + # const_ref recorded + assert len(result[0].const_refs) == 1 + assert result[0].const_refs[0].const_name == "TARGET" + assert result[0].const_refs[0].resolved_value == "crispen.before.X" + assert result[0].const_refs[0].patch_dec_idx == 0 + + +def test_find_const_ref_not_in_map_not_collected(tmp_path): + """@patch(UNRESOLVED) where name not in const_map → not collected.""" + src = "@patch(UNRESOLVED)\ndef test_f(mock): pass\n" + scan = str(tmp_path / "test_foo.py") + result = _find_test_functions_to_update(src, {"crispen.before.X"}, scan_file=scan) + assert result == [] + + +def test_find_const_ref_value_no_match(tmp_path): + """@patch(CONST) where const value doesn't match old_paths → not collected.""" + src = 'TARGET = "other.mod.Y"\n\n@patch(TARGET)\ndef test_f(mock): pass\n' + scan = str(tmp_path / "test_foo.py") + result = _find_test_functions_to_update(src, {"crispen.before.X"}, scan_file=scan) + assert result == [] + + +def test_find_mix_literal_and_const(tmp_path): + """Function with both a literal @patch and a const @patch → both collected.""" + src = ( + 'TARGET = "crispen.before.X"\n\n' + '@patch("crispen.before.X")\n' + "@patch(TARGET)\n" + "def test_f(m1, m2):\n pass\n" + ) + scan = str(tmp_path / "test_foo.py") + result = _find_test_functions_to_update(src, {"crispen.before.X"}, scan_file=scan) + assert len(result) == 1 + assert len(result[0].old_patch_paths) == 2 + assert len(result[0].const_refs) == 1 + # patch_dec_idx of the const ref is 1 (second @patch decorator) + assert result[0].const_refs[0].patch_dec_idx == 1 + + +def test_find_non_matching_decorator_split_into_stable(tmp_path): + """Non-matching decorators go to stable_patch_paths, not old_patch_paths. + + A test that patches get_api_key (already correct) and call_with_tool + (forking, needs rewrite) should have only call_with_tool in old_patch_paths + and get_api_key in stable_patch_paths so the LLM is not asked to evaluate + the already-correct path. + """ + src = ( + 'KEY = "crispen.mod.get_api_key"\n' + 'CALL = "crispen.mod.call_with_tool"\n\n' + "@patch(KEY)\n" + "@patch(CALL)\n" + "def test_f(mock_call, mock_key):\n pass\n" + ) + scan = str(tmp_path / "test_foo.py") + # Only CALL's value is in old_paths; KEY's value is already correct. + result = _find_test_functions_to_update( + src, {"crispen.mod.call_with_tool"}, scan_file=scan + ) + assert len(result) == 1 + # Forking path goes to old_patch_paths only. + assert result[0].old_patch_paths == ["crispen.mod.call_with_tool"] + # Already-correct path goes to stable_patch_paths. + assert result[0].stable_patch_paths == ["crispen.mod.get_api_key"] + # Both const refs must be recorded so their definitions can be updated. + assert len(result[0].const_refs) == 2 + + +def test_find_patch_no_args_increments_idx(tmp_path): + """@patch() with no args increments patch_dec_idx before the const @patch.""" + src = ( + 'TARGET = "crispen.before.X"\n\n' + "@patch()\n" + "@patch(TARGET)\n" + "def test_f(m):\n pass\n" + ) + scan = str(tmp_path / "test_foo.py") + result = _find_test_functions_to_update(src, {"crispen.before.X"}, scan_file=scan) + assert len(result) == 1 + assert result[0].const_refs[0].patch_dec_idx == 1 + + +def test_find_cross_file_const(tmp_path): + """@patch(CONST) where CONST comes from a relative import → collected.""" + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") + src = "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(mock):\n pass\n" + scan = str(tmp_path / "test_foo.py") + result = _find_test_functions_to_update( + src, {"crispen.before.X"}, scan_file=scan, repo_root=str(tmp_path) + ) + assert len(result) == 1 + assert result[0].const_refs[0].source_file == str(helpers.resolve()) + + +def test_find_attr_const_ref_collected(tmp_path): + """@patch(constants.TARGET) where ``import constants`` resolves → collected.""" + constants_file = tmp_path / "constants.py" + constants_file.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") + src = "import constants\n\n@patch(constants.TARGET)\ndef test_f(mock):\n pass\n" + scan = str(tmp_path / "test_foo.py") + result = _find_test_functions_to_update( + src, {"crispen.before.X"}, scan_file=scan, repo_root=str(tmp_path) + ) + assert len(result) == 1 + assert result[0].function_name == "test_f" + assert result[0].const_refs[0].const_name == "constants.TARGET" + assert result[0].const_refs[0].resolved_value == "crispen.before.X" + assert result[0].const_refs[0].patch_dec_idx == 0 + assert result[0].const_refs[0].source_file == str(constants_file.resolve()) + # LLM sees inlined value, not the attribute access form. + assert '"crispen.before.X"' in result[0].full_text + assert "constants.TARGET" not in result[0].full_text + + +def test_find_attr_const_module_not_in_map(tmp_path): + """@patch(unknown.TARGET) where module not in attr_const_map → not collected.""" + src = "@patch(unknown.TARGET)\ndef test_f(mock):\n pass\n" + scan = str(tmp_path / "test_foo.py") + # No ``import unknown`` in source → attr_const_map empty → no match. + result = _find_test_functions_to_update( + src, {"crispen.before.X"}, scan_file=scan, repo_root=str(tmp_path) + ) + assert result == [] + + +def test_find_attr_const_attr_not_in_module(tmp_path): + """@patch(constants.UNKNOWN) where attr not in module constants → not collected.""" + constants_file = tmp_path / "constants.py" + constants_file.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") + src = "import constants\n\n@patch(constants.UNKNOWN)\ndef test_f(mock):\n pass\n" + scan = str(tmp_path / "test_foo.py") + result = _find_test_functions_to_update( + src, {"crispen.before.X"}, scan_file=scan, repo_root=str(tmp_path) + ) + assert result == [] + + +def test_find_attr_const_value_no_match(tmp_path): + """@patch(constants.OTHER) where value doesn't match old_paths → not collected.""" + constants_file = tmp_path / "constants.py" + constants_file.write_text('OTHER = "unrelated.path.Class"\n', encoding="utf-8") + src = "import constants\n\n@patch(constants.OTHER)\ndef test_f(mock):\n pass\n" + scan = str(tmp_path / "test_foo.py") + result = _find_test_functions_to_update( + src, {"crispen.before.X"}, scan_file=scan, repo_root=str(tmp_path) + ) + assert result == [] + + +def test_find_attr_multi_level_not_handled(tmp_path): + """@patch(a.b.c) multi-level attribute (base not Name) → not collected.""" + src = "@patch(a.b.c)\ndef test_f(mock):\n pass\n" + scan = str(tmp_path / "test_foo.py") + result = _find_test_functions_to_update( + src, {"a.b.c"}, scan_file=scan, repo_root=str(tmp_path) + ) + assert result == [] diff --git a/tests/patch_rewriter/const_refs/test_process_const_refs.py b/tests/patch_rewriter/const_refs/test_process_const_refs.py new file mode 100644 index 0000000..ca6c492 --- /dev/null +++ b/tests/patch_rewriter/const_refs/test_process_const_refs.py @@ -0,0 +1,427 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch as mock_patch +from crispen.patch_rewriter import ( + _apply_cross_file_const_updates, + _process_file_source, + _restore_const_refs, + _substitute_consts_in_func_text, +) +from ..helpers import ( + _CFG, + _PATCH_CALL_TOOL, + _SRC_WITH_CONST, + _VERIFY_OK, + _make_ref, + _ok, +) + + +def test_substitute_replaces_const(): + code = "@patch(TARGET)\ndef test_f(mock): pass\n" + result = _substitute_consts_in_func_text(code, {"TARGET": "myapp.svc.MyClass"}) + assert '@patch("myapp.svc.MyClass")' in result + assert "TARGET" not in result + + +def test_substitute_no_subs_unchanged(): + code = "@patch(TARGET)\ndef test_f(mock): pass\n" + assert _substitute_consts_in_func_text(code, {}) == code + + +def test_substitute_parse_error_returns_original(): + code = "def f(:\n" + assert _substitute_consts_in_func_text(code, {"X": "val"}) == code + + +def test_substitute_non_patch_call_unchanged(): + # other_func(TARGET) inside the body is not a patch call → left as-is. + code = "@patch(TARGET)\ndef test_f(mock):\n other_func(TARGET)\n" + result = _substitute_consts_in_func_text(code, {"TARGET": "myapp.svc.MyClass"}) + assert '@patch("myapp.svc.MyClass")' in result + assert "other_func(TARGET)" in result + + +def test_substitute_name_not_in_subs_unchanged(): + # @patch(OTHER) where OTHER is not in substitutions → left as-is (line 311). + code = "@patch(TARGET)\n@patch(OTHER)\ndef test_f(m1, m2):\n pass\n" + result = _substitute_consts_in_func_text(code, {"TARGET": "myapp.svc.MyClass"}) + assert '@patch("myapp.svc.MyClass")' in result + assert "@patch(OTHER)" in result + + +def test_substitute_attr_in_subs(): + """@patch(module.CONSTANT) with dotted key in subs → substituted.""" + code = "@patch(constants.TARGET)\ndef test_f(mock):\n pass\n" + result = _substitute_consts_in_func_text( + code, {"constants.TARGET": "myapp.svc.MyClass"} + ) + assert '@patch("myapp.svc.MyClass")' in result + assert "constants.TARGET" not in result + + +def test_substitute_attr_not_in_subs(): + """@patch(constants.OTHER) where dotted key not in subs → unchanged.""" + code = ( + "@patch(constants.TARGET)\n" + "@patch(constants.OTHER)\n" + "def test_f(m1, m2):\n pass\n" + ) + result = _substitute_consts_in_func_text( + code, {"constants.TARGET": "myapp.svc.MyClass"} + ) + assert '@patch("myapp.svc.MyClass")' in result + assert "@patch(constants.OTHER)" in result + + +def test_substitute_attr_non_name_base(): + """@patch(a.b.c) where base is Attribute (not Name) → else branch, unchanged.""" + code = "@patch(a.b.c)\ndef test_f(mock):\n pass\n" + result = _substitute_consts_in_func_text(code, {"a.b.c": "should.not.replace"}) + assert "@patch(a.b.c)" in result + + +def test_restore_reverts_unchanged_plain_name(): + """@patch("value") whose value matches a const_ref → reverted to @patch(NAME).""" + code = '@patch("myapp.svc.MyClass")\ndef test_f(mock): pass\n' + refs = [_make_ref("TARGET", "myapp.svc.MyClass")] + result = _restore_const_refs(code, refs) + assert "@patch(TARGET)" in result + assert '"myapp.svc.MyClass"' not in result + + +def test_restore_reverts_unchanged_attr_form(): + """@patch("value") matching module.CONST ref → reverted to @patch(module.CONST).""" + code = '@patch("myapp.svc.MyClass")\ndef test_f(mock): pass\n' + refs = [_make_ref("constants.TARGET", "myapp.svc.MyClass")] + result = _restore_const_refs(code, refs) + assert "@patch(constants.TARGET)" in result + assert '"myapp.svc.MyClass"' not in result + + +def test_restore_leaves_changed_value_as_literal(): + """@patch("new.value") where new.value is not in const_refs → kept as literal.""" + code = '@patch("myapp.new.MyClass")\ndef test_f(mock): pass\n' + refs = [_make_ref("TARGET", "myapp.old.MyClass")] + result = _restore_const_refs(code, refs) + assert '@patch("myapp.new.MyClass")' in result + + +def test_restore_empty_refs_unchanged(): + """No const_refs → text returned as-is.""" + code = '@patch("myapp.svc.MyClass")\ndef test_f(mock): pass\n' + assert _restore_const_refs(code, []) == code + + +def test_restore_parse_error_returns_original(): + """Unparseable text → original returned unchanged.""" + code = "def f(:\n" + refs = [_make_ref("TARGET", "myapp.svc.X")] + assert _restore_const_refs(code, refs) == code + + +def test_restore_empty_args_patch_unchanged(): + """@patch() with no args → left as-is.""" + code = "@patch()\ndef test_f(): pass\n" + refs = [_make_ref("TARGET", "myapp.svc.MyClass")] + assert _restore_const_refs(code, refs) == code + + +def test_restore_non_string_arg_unchanged(): + """@patch(NAME) where arg is a Name node (not SimpleString) → left as-is.""" + code = "@patch(OTHER_NAME)\ndef test_f(mock): pass\n" + refs = [_make_ref("TARGET", "myapp.svc.MyClass")] + result = _restore_const_refs(code, refs) + assert "@patch(OTHER_NAME)" in result + + +def test_restore_non_patch_call_untouched(): + """other_func("value") is not a patch call → left as-is.""" + code = ( + '@patch("myapp.svc.MyClass")\n' + "def test_f(mock):\n" + ' other_func("myapp.svc.OtherClass")\n' + ) + refs = [ + _make_ref("TARGET", "myapp.svc.MyClass"), + _make_ref("OTHER", "myapp.svc.OtherClass"), + ] + result = _restore_const_refs(code, refs) + assert "@patch(TARGET)" in result + assert 'other_func("myapp.svc.OtherClass")' in result + + +def test_restore_single_quote_string(): + """SimpleString with single quotes → still reverted.""" + code = "@patch('myapp.svc.MyClass')\ndef test_f(mock): pass\n" + refs = [_make_ref("TARGET", "myapp.svc.MyClass")] + result = _restore_const_refs(code, refs) + assert "@patch(TARGET)" in result + + +def test_restore_partial_revert_mixed(): + """One decorator changed, one unchanged → only unchanged one is reverted.""" + code = ( + '@patch("myapp.svc.MyClass")\n' + '@patch("myapp.new.Y")\n' + "def test_f(m1, m2): pass\n" + ) + # MyClass unchanged (should revert), Y was updated by LLM (keep literal) + refs = [ + _make_ref("TARGET", "myapp.svc.MyClass"), + _make_ref("Y_CONST", "myapp.old.Y"), # old value; new value won't match + ] + result = _restore_const_refs(code, refs) + assert "@patch(TARGET)" in result + assert '@patch("myapp.new.Y")' in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_const_same_file_update(mock_call, tmp_path): + """Same-file const ref → same_file_const_map updates the const definition.""" + scan = str(tmp_path / "test_foo.py") + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.after.X"}, + } + ), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_CONST, + {"crispen.before.X"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + ) + assert changed is True + # apply_patch_strings updates the const definition. + assert '"crispen.after.X"' in result + assert '"crispen.before.X"' not in result + # No cross-file updates for same-file const. + assert cross == {} + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_const_cross_file_update(mock_call, tmp_path): + """Const ref from imported file → cross_file_patch_maps returned.""" + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") + src = "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(mock):\n pass\n" + scan = str(tmp_path / "test_foo.py") + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.after.X"}, + } + ), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + src, + {"crispen.before.X"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + repo_root=str(tmp_path), + ) + helpers_abs = str(helpers.resolve()) + assert helpers_abs in cross + assert cross[helpers_abs] == {"crispen.before.X": "crispen.after.X"} + + +@mock_patch( + _PATCH_CALL_TOOL, return_value=_ok({"needs_rewrite": False, "patch_renames": {}}) +) +def test_process_const_no_change_no_cross(mock_call, tmp_path): + """LLM returns no renames → no change, cross is empty.""" + scan = str(tmp_path / "test_foo.py") + result, changed, cross = _process_file_source( + _SRC_WITH_CONST, + {"crispen.before.X"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + ) + assert changed is False + assert cross == {} + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_cross_file_const_ref_not_in_renames(mock_call, tmp_path): + """Cross-file const whose patch path is not in accepted renames → skipped. + + Scenario: function has two @patch decorators with different old paths. One + is a cross-file const ref (path A) and the other is a literal (path B). + Classify returns rename only for B; A is not in accepted renames. + The const ref for A should be skipped. + """ + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET_A = "crispen.before.A"\n', encoding="utf-8") + src = ( + "from .helpers import TARGET_A\n\n" + '@patch(TARGET_A)\n@patch("crispen.before.B")\n' + "def test_f(m1, m2):\n pass\n" + ) + scan = str(tmp_path / "test_foo.py") + # Classify: only rename crispen.before.B → crispen.after.B; + # crispen.before.A unchanged. + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.B": "crispen.after.B"}, + } + ), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + src, + {"crispen.before.A", "crispen.before.B"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + repo_root=str(tmp_path), + ) + assert changed is True + assert "crispen.after.B" in result + # crispen.before.A not in accepted renames → no cross-file update for helpers.py. + helpers_abs = str(helpers.resolve()) + assert helpers_abs not in cross + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_scan_file_no_const_processing(mock_call): + """scan_file="" → const_map is empty, const post-processing skipped.""" + # Even with a const-ref style source, no scan_file means no const resolution. + src = 'TARGET = "crispen.before.X"\n\n@patch(TARGET)\ndef test_f(m):\n pass\n' + # With scan_file="", const_map is empty, @patch(TARGET) is not collected. + result, changed, cross = _process_file_source( + src, {"crispen.before.X"}, "ctx", MagicMock(), _CFG, 1 + ) + assert result == src + assert changed is False + assert cross == {} + mock_call.assert_not_called() + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_attr_const_cross_file_update(mock_call, tmp_path): + """@patch(constants.TARGET) resolved via import → cross-file proposal returned.""" + constants_file = tmp_path / "constants.py" + constants_file.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") + src = ( + "import constants\n\n" + "@patch(constants.TARGET)\n" + "def test_f(mock_x):\n" + " pass\n" + ) + scan = str(tmp_path / "test_foo.py") + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.after.X"}, + } + ), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + src, + {"crispen.before.X"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + repo_root=str(tmp_path), + ) + # Cross-file proposal recorded for constants.py. + constants_abs = str(constants_file.resolve()) + assert constants_abs in cross + assert cross[constants_abs] == {"crispen.before.X": "crispen.after.X"} + + +def test_cross_file_empty_proposals(): + msgs = list(_apply_cross_file_const_updates({}, {})) + assert msgs == [] + + +def test_cross_file_conflicting_proposals(tmp_path): + """Multiple new values for the same constant → resolved is empty → skip.""" + f = tmp_path / "helpers.py" + f.write_text('TARGET = "old.val"\n', encoding="utf-8") + proposals = {str(f.resolve()): {"old.val": {"new.val1", "new.val2"}}} + msgs = list(_apply_cross_file_const_updates(proposals, {})) + assert msgs == [] + # File unchanged. + assert f.read_text(encoding="utf-8") == 'TARGET = "old.val"\n' + + +def test_cross_file_per_file_entry_updated(tmp_path): + """Const source file is in per_file → updates in-memory source, no disk write.""" + f = tmp_path / "helpers.py" + f.write_text('TARGET = "old.val"\n', encoding="utf-8") + per_file = {str(f): {"source": 'TARGET = "old.val"\n', "msgs": []}} + proposals = {str(f.resolve()): {"old.val": {"new.val"}}} + msgs = list(_apply_cross_file_const_updates(proposals, per_file)) + assert msgs == [] + assert '"new.val"' in per_file[str(f)]["source"] + assert any("constant definition" in m for m in per_file[str(f)]["msgs"]) + # Disk file unchanged. + assert f.read_text(encoding="utf-8") == 'TARGET = "old.val"\n' + + +def test_cross_file_per_file_entry_no_change(tmp_path): + """Resolved new value equals old → apply_patch_strings makes no change → no msg.""" + f = tmp_path / "helpers.py" + src = 'TARGET = "new.val"\n' # already has new value + per_file = {str(f): {"source": src, "msgs": []}} + proposals = {str(f.resolve()): {"old.val": {"new.val"}}} + # apply_patch_strings("TARGET = "new.val"\n", {"old.val": "new.val"}) → unchanged + msgs = list(_apply_cross_file_const_updates(proposals, per_file)) + assert msgs == [] + assert per_file[str(f)]["msgs"] == [] + + +def test_cross_file_disk_file_updated(tmp_path): + """Const source file is a disk file → written, message yielded.""" + f = tmp_path / "helpers.py" + f.write_text('TARGET = "old.val"\n', encoding="utf-8") + proposals = {str(f.resolve()): {"old.val": {"new.val"}}} + msgs = list(_apply_cross_file_const_updates(proposals, {})) + assert len(msgs) == 1 + assert "constant definition" in msgs[0] + assert '"new.val"' in f.read_text(encoding="utf-8") + + +def test_cross_file_disk_file_no_change(tmp_path): + """Disk file already has the new value → no write, no message.""" + f = tmp_path / "helpers.py" + f.write_text('TARGET = "new.val"\n', encoding="utf-8") + proposals = {str(f.resolve()): {"old.val": {"new.val"}}} + msgs = list(_apply_cross_file_const_updates(proposals, {})) + assert msgs == [] + + +def test_cross_file_disk_oserror(tmp_path): + """OSError reading disk file → skipped silently.""" + f = tmp_path / "helpers.py" + f.write_text('TARGET = "old.val"\n', encoding="utf-8") + f.chmod(0o000) + try: + proposals = {str(f.resolve()): {"old.val": {"new.val"}}} + msgs = list(_apply_cross_file_const_updates(proposals, {})) + assert msgs == [] + finally: + f.chmod(0o644) diff --git a/tests/patch_rewriter/helpers.py b/tests/patch_rewriter/helpers.py new file mode 100644 index 0000000..35472ed --- /dev/null +++ b/tests/patch_rewriter/helpers.py @@ -0,0 +1,184 @@ +from __future__ import annotations +from crispen.config import CrispenConfig +from crispen.llm_client import LLMCallResult +from crispen.patch_rewriter import ( + _CgIndex, + _ConstRef, + _FLContext, + _build_context_message, +) + + +def _ok(tool_input=None) -> LLMCallResult: + return LLMCallResult( + tool_input=tool_input, elapsed=0.0, input_tokens=0, output_tokens=0 + ) + + +def _truncated_ok() -> LLMCallResult: + """Simulate a truncated verify response (tool_input=None, truncated=True).""" + return LLMCallResult( + tool_input=None, elapsed=0.0, input_tokens=0, output_tokens=0, truncated=True + ) + + +def _make_fl_ctx(**kwargs) -> _FLContext: + defaults = dict( + filepath="/proj/pkg/big.py", + old_module="pkg.big", + original_source="class A: pass\nclass B: pass\n", + modified_source="from .sub_a import A\nfrom .sub_b import B\n", + new_files={"sub_a.py": "class A: pass\n", "sub_b.py": "class B: pass\n"}, + new_module_paths={"sub_a.py": "pkg.sub_a", "sub_b.py": "pkg.sub_b"}, + entity_to_target={"A": "sub_a.py", "B": "sub_b.py"}, + forking_old_paths={"pkg.big.A", "pkg.big.B"}, + ) + defaults.update(kwargs) + return _FLContext(**defaults) + + +_CFG = CrispenConfig(patch_update_retries=1) +_CFG_NO_LLM_VERIFY = CrispenConfig(patch_update_retries=1, llm_verify_retries=0) +_FORKING_PATHS = {"crispen.before.X"} +_SRC_WITH_PATCH = '@patch("crispen.before.X")\ndef test_f(mock_x):\n pass\n' + +_PATCH_GET_KEY = "crispen.patch_rewriter.get_api_key" +_PATCH_MAKE_CLIENT = "crispen.patch_rewriter.make_client" +_PATCH_CALL_TOOL = "crispen.patch_rewriter.call_with_tool" + +# Shorthand classify tool_inputs. +_CLASSIFY_RENAME = { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.after.X"}, +} +_CLASSIFY_NO_CHANGE = {"needs_rewrite": False, "patch_renames": {}} +_CLASSIFY_REWRITE = {"needs_rewrite": True} +_VERIFY_OK = {"correct": True, "issue": ""} +_VERIFY_REJECT = {"correct": False, "issue": "wrong path"} +_VERIFY_REJECT_WITH_CORRECTIONS = { + "correct": False, + "issue": "wrong path", + "corrections": {"crispen.before.X": "crispen.after.X"}, +} +_REWRITE_VERIFY_OK = {"correct": True, "issue": ""} +_REWRITE_VERIFY_REJECT = {"correct": False, "issue": "wrong mock setup"} + + +def _ctx_msg() -> str: + return _build_context_message([_make_fl_ctx()]) + + +_VALID_REWRITE = ( + '@patch("crispen.after.X")\n' + '@patch("crispen.after.Y")\n' + "def test_f(mock_x, mock_y):\n" + " pass\n" +) + + +_PATCH_MAKE_CLIENT = "crispen.patch_rewriter.make_client" +_PATCH_GET_KEY_PR = "crispen.patch_rewriter.get_api_key" +_PATCH_CALL_PR = "crispen.patch_rewriter.call_with_tool" + + +def _make_ref(const_name: str, resolved_value: str) -> _ConstRef: + return _ConstRef( + const_name=const_name, + source_file="/proj/tests/helpers.py", + resolved_value=resolved_value, + patch_dec_idx=0, + ) + + +_SRC_WITH_CONST = ( + 'TARGET = "crispen.before.X"\n\n' + "@patch(TARGET)\n" + "def test_f(mock_x):\n" + " pass\n" +) + + +def _make_bfs_ctx() -> _FLContext: + """Context with placement.py (helper) and conflict.py (resolve) using use_fn.""" + return _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="from .placement import helper\n", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", + "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", + }, + new_module_paths={ + "placement.py": "pkg.placement", + "conflict.py": "pkg.conflict", + }, + entity_to_target={"helper": "placement.py"}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + + +def _make_bfs_index(test_src: str, calling_module: str = "pkg.test_mod") -> _CgIndex: + """Minimal index: only the calling module's source (for import resolution).""" + parts = calling_module.split(".") + pkg = ".".join(parts[:-1]) if len(parts) > 1 else "" + return _CgIndex( + module_to_source={calling_module: test_src}, + module_to_package={calling_module: pkg}, + module_to_defs={calling_module: set()}, + file_to_module={}, + ) + + +def _make_cuf_contexts() -> list: + """FL context with placement (helper) and conflict (resolve) using use_fn.""" + return [ + _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\ndef helper(): use_fn()\n", + modified_source="from .placement import helper\n", + new_files={ + "placement.py": ( + "from external import use_fn\ndef helper(): use_fn()\n" + ), + "conflict.py": ( + "from external import use_fn\ndef resolve(): use_fn()\n" + ), + }, + new_module_paths={ + "placement.py": "pkg.placement", + "conflict.py": "pkg.conflict", + }, + entity_to_target={"helper": "placement.py"}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + ] + + +def _make_cuf_index(scan_abs: str, test_src: str) -> _CgIndex: + """Minimal index for _callgraph_update_file: maps scan_abs → 'pkg.test_mod'.""" + return _CgIndex( + module_to_source={"pkg.test_mod": test_src}, + module_to_package={"pkg.test_mod": "pkg"}, + module_to_defs={"pkg.test_mod": set()}, + file_to_module={scan_abs: "pkg.test_mod"}, + ) + + +def _make_fl_ctx_simple(): + """Minimal FLContext for prompt builder tests.""" + return _FLContext( + filepath="/repo/pkg/big.py", + old_module="pkg.big", + original_source="from external import A\ndef f(): A()\n", + modified_source="from .sub_a import f\n", + new_files={"sub_a.py": "from external import A\ndef f(): A()\n"}, + new_module_paths={"sub_a.py": "pkg.sub_a"}, + entity_to_target={"f": "sub_a.py"}, + forking_old_paths={"pkg.big.A"}, + ) + + +def _make_process_cfg(): + return CrispenConfig(patch_update_retries=1, llm_verify_retries=0) diff --git a/tests/patch_rewriter/process_basic/__init__.py b/tests/patch_rewriter/process_basic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/patch_rewriter/process_basic/test_no_change.py b/tests/patch_rewriter/process_basic/test_no_change.py new file mode 100644 index 0000000..2b26c64 --- /dev/null +++ b/tests/patch_rewriter/process_basic/test_no_change.py @@ -0,0 +1,693 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch as mock_patch +from crispen.config import CrispenConfig +from crispen.llm_client import LLMCallResult +from crispen.patch_rewriter import RewriteAccumulator, _process_file_source +from ..helpers import ( + _CFG, + _CFG_NO_LLM_VERIFY, + _CLASSIFY_NO_CHANGE, + _CLASSIFY_RENAME, + _FORKING_PATHS, + _PATCH_CALL_TOOL, + _REWRITE_VERIFY_OK, + _SRC_WITH_PATCH, + _VALID_REWRITE, + _VERIFY_OK, + _VERIFY_REJECT, + _VERIFY_REJECT_WITH_CORRECTIONS, + _ok, + _truncated_ok, +) + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_functions(mock_call): + src = "def test_f(): pass\n" + result, changed, cross = _process_file_source( + src, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert result == src + assert changed is False + mock_call.assert_not_called() + + +@mock_patch(_PATCH_CALL_TOOL, return_value=_ok(None)) +def test_process_classify_tool_none(mock_call): + # Classify returns tool_input=None → break, no update. + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert result == _SRC_WITH_PATCH + assert changed is False + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_needed(mock_call): + # Classify returns empty renames → verify confirms no-change → no update. + mock_call.side_effect = [_ok(_CLASSIFY_NO_CHANGE), _ok(_VERIFY_OK)] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert result == _SRC_WITH_PATCH + assert changed is False + assert mock_call.call_count == 2 # classify + verify + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_verify_none_accept(mock_call): + # Classify says no change; verify returns None → accept no-change. + mock_call.side_effect = [_ok(_CLASSIFY_NO_CHANGE), _ok(None)] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is False + assert mock_call.call_count == 2 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_verify_truncated_reject(mock_call): + # No-change verify truncated → treated as rejection, not accepted as no-change. + mock_call.side_effect = [_ok(_CLASSIFY_NO_CHANGE), _truncated_ok()] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 1 + ) + assert result == _SRC_WITH_PATCH + assert changed is False + assert mock_call.call_count == 2 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_verify_rejects_then_accepts(mock_call): + # No-change verify rejects with corrections; corrections-verify accepts. + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok(_VERIFY_REJECT_WITH_CORRECTIONS), + _ok(_VERIFY_OK), # corrections-verify accepts + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 + ) + assert changed is True + assert "crispen.after.X" in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_verify_retries_exhausted(mock_call): + # llm_verify_retries=0: no escalation, accept no-change immediately. + + cfg = CrispenConfig(patch_update_retries=1, llm_verify_retries=0) + mock_call.side_effect = [_ok(_CLASSIFY_NO_CHANGE), _ok(_VERIFY_REJECT)] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), cfg, 1 + ) + assert changed is False + assert mock_call.call_count == 2 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_exhausted_escalates_to_rewrite(mock_call): + # When llm_verify_retries>0 and no-change retries are exhausted, escalate + # to the full rewrite path seeded with the verifier's explanation. + + cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) + # Corrections that rename the function itself (X→Y) are filtered out by the + # name-invariant guard, so corrections_renames ends up empty, causing the + # retry to exhaust and escalate to rewrite (covers lines 2897-2901). + name_change_correction = { + "correct": False, + "issue": "wrong path", + "corrections": {"crispen.before.X": "crispen.before.Y"}, + } + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), # classify → no change + _ok(name_change_correction), # verify → reject (filtered corrections) + _ok(_CLASSIFY_NO_CHANGE), # classify (retry) → no change again + _ok(name_change_correction), # verify → reject (retries exhausted → escalate) + _ok({"rewritten_function": _VALID_REWRITE}), # rewrite (escalated) + _ok(_REWRITE_VERIFY_OK), # verify rewrite → accept + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), cfg, 3 + ) + assert changed is True + assert mock_call.call_count == 6 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_exhausted_escalate_verbose(mock_call, capsys): + # verbose=True prints 'escalating to rewrite' when escalation is triggered. + + cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) + name_change_correction = { + "correct": False, + "issue": "wrong path", + "corrections": {"crispen.before.X": "crispen.before.Y"}, + } + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok(name_change_correction), + _ok(_CLASSIFY_NO_CHANGE), + _ok(name_change_correction), + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(_REWRITE_VERIFY_OK), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + cfg, + 3, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "escalating to rewrite" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_applied(mock_call): + # No-change verify returns corrections → corrections-verify accepts → apply. + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok(_VERIFY_REJECT_WITH_CORRECTIONS), + _ok(_VERIFY_OK), # corrections-verify accepts + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 + ) + assert changed is True + assert "crispen.after.X" in result + assert mock_call.call_count == 3 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_verify_none_accept(mock_call): + # Corrections-verify returns tool_input=None → accept. + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok(_VERIFY_REJECT_WITH_CORRECTIONS), + _ok(None), # corrections-verify returns None → accept + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 + ) + assert changed is True + assert "crispen.after.X" in result + assert mock_call.call_count == 3 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_verify_truncated_reject(mock_call): + # Corrections-verify truncated → treated as rejection, corrections not applied. + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok(_VERIFY_REJECT_WITH_CORRECTIONS), + _truncated_ok(), # corrections-verify truncated → reject + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 2 + ) + assert result == _SRC_WITH_PATCH + assert changed is False + assert mock_call.call_count == 3 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_verify_fails_retry(mock_call): + # Corrections-verify rejects → retries left → retry classify which succeeds. + + cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), # classify → no change + _ok(_VERIFY_REJECT_WITH_CORRECTIONS), # verify → reject + corrections + _ok(_VERIFY_REJECT), # corrections-verify → rejected + _ok(_CLASSIFY_RENAME), # classify (retry) → rename + _ok(_VERIFY_OK), # rename verify → accept + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), cfg, 3 + ) + assert changed is True + assert "crispen.after.X" in result + assert mock_call.call_count == 5 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_verbose(mock_call, capsys): + # verbose=True prints 'verifying corrections for' and 'ACCEPTED'. + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok(_VERIFY_REJECT_WITH_CORRECTIONS), + _ok(_VERIFY_OK), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + _CFG, + 2, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "verifying corrections for" in err + assert "ACCEPTED" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_verbose_reject(mock_call, capsys): + # verbose=True prints 'REJECTED' and issue when corrections-verify rejects. + + cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok(_VERIFY_REJECT_WITH_CORRECTIONS), + _ok({"correct": False, "issue": "correction still wrong", "corrections": {}}), + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_OK), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + cfg, + 3, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "corrections verify REJECTED" in err + assert "correction still wrong" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_timing_detailed(mock_call, capsys): + # timing='detailed' prints elapsed/token info after corrections-verify call. + + cfg = CrispenConfig(patch_update_retries=2, timing="detailed") + mock_call.side_effect = [ + LLMCallResult( + tool_input=_CLASSIFY_NO_CHANGE, + elapsed=0.5, + input_tokens=100, + output_tokens=10, + ), + LLMCallResult( + tool_input=_VERIFY_REJECT_WITH_CORRECTIONS, + elapsed=0.4, + input_tokens=90, + output_tokens=20, + ), + LLMCallResult( + tool_input=_VERIFY_OK, + elapsed=0.3, + input_tokens=80, + output_tokens=5, + ), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + cfg, + 2, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "0.30s" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_acc(mock_call): + # _acc accumulates calls from classify, no-change verify, and corrections-verify. + mock_call.side_effect = [ + LLMCallResult( + tool_input=_CLASSIFY_NO_CHANGE, + elapsed=0.5, + input_tokens=100, + output_tokens=10, + ), + LLMCallResult( + tool_input=_VERIFY_REJECT_WITH_CORRECTIONS, + elapsed=0.4, + input_tokens=90, + output_tokens=20, + ), + LLMCallResult( + tool_input=_VERIFY_OK, + elapsed=0.3, + input_tokens=80, + output_tokens=5, + ), + ] + acc = RewriteAccumulator() + _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2, _acc=acc + ) + assert acc.calls == 3 + assert abs(acc.elapsed - 1.2) < 1e-9 + assert acc.input_tokens == 270 + assert acc.output_tokens == 35 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_no_splice(mock_call, tmp_path): + # Corrections-verify accepts; function uses const ref → no splice; const updated. + src = ( + 'TARGET = "crispen.before.X"\n\n@patch(TARGET)\ndef test_f(mock_x):\n pass\n' + ) + scan = str(tmp_path / "test_foo.py") + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok(_VERIFY_REJECT_WITH_CORRECTIONS), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + src, {"crispen.before.X"}, "ctx", MagicMock(), _CFG, 2, scan_file=scan + ) + assert changed is True + assert "crispen.after.X" in result + assert mock_call.call_count == 3 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_name_invariant_filtered(mock_call): + # Verifier proposes corrections that rename the patched name itself + # (e.g. X → Y). These must be filtered out; with an empty corrections set + # the no-change result falls through to retry logic — here retries=1 so + # the second classify call is made and returns no-change confirmed by verify. + verify_name_change_correction = { + "correct": False, + "issue": "module moved", + "corrections": {"crispen.before.X": "crispen.before.Y"}, # name changed! + } + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok(verify_name_change_correction), + _ok(_CLASSIFY_NO_CHANGE), + _ok(_VERIFY_OK), + ] + + cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), cfg, 3 + ) + # Correction was filtered (name changed X→Y) — no change applied. + assert "crispen.before.Y" not in result + assert mock_call.call_count == 4 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_corrections_still_imported_guard(mock_call): + # Verifier proposes corrections that move a name listed as still-imported in + # the context message to a non-submodule path. The second still-imported + # filter drops the correction; with empty corrections the retry loop resumes + # and accepts no-change on verify. + still_imported_ctx = ( + "Names still externally imported in the modified original (check):\n" "- `X`\n" + ) + verify_still_imported_correction = { + "correct": False, + "issue": "hallucinated move", + "corrections": {"crispen.before.X": "crispen.sub.X"}, # X is still in orig + } + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok(verify_still_imported_correction), + _ok(_CLASSIFY_NO_CHANGE), + _ok(_VERIFY_OK), + ] + + cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + still_imported_ctx, + MagicMock(), + cfg, + 3, + still_imported={"X"}, + ) + # Correction was filtered (X still imported, non-submodule target) — + # no change applied; retry loop accepted no-change on subsequent verify. + assert "crispen.sub.X" not in result + assert mock_call.call_count == 4 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_verify_verbose(mock_call, capsys): + # verbose=True prints 'verifying no-change' and 'ACCEPTED'. + mock_call.side_effect = [_ok(_CLASSIFY_NO_CHANGE), _ok(_VERIFY_OK)] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "verifying no-change" in err + assert "ACCEPTED" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_verify_verbose_reject(mock_call, capsys): + # verbose=True prints 'REJECTED' and the issue when no-change verify rejects. + mock_call.side_effect = [ + _ok(_CLASSIFY_NO_CHANGE), + _ok({"correct": False, "issue": "patch still points to old module"}), + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_OK), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + _CFG, + 2, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "REJECTED" in err + assert "patch still points to old module" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_verify_timing_detailed(mock_call, capsys): + # timing='detailed' appends elapsed/token info after the no-change verify call. + + cfg = CrispenConfig(patch_update_retries=1, timing="detailed") + mock_call.side_effect = [ + LLMCallResult( + tool_input=_CLASSIFY_NO_CHANGE, + elapsed=0.5, + input_tokens=100, + output_tokens=10, + ), + LLMCallResult( + tool_input=_VERIFY_OK, + elapsed=0.3, + input_tokens=80, + output_tokens=5, + ), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + cfg, + 1, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "→ done" in err + assert "0.30s" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_no_change_acc_accumulates(mock_call): + # _acc accumulates calls from both classify and no-change verify; no_change counted. + mock_call.side_effect = [ + LLMCallResult( + tool_input=_CLASSIFY_NO_CHANGE, + elapsed=0.5, + input_tokens=100, + output_tokens=10, + ), + LLMCallResult( + tool_input=_VERIFY_OK, + elapsed=0.3, + input_tokens=80, + output_tokens=5, + ), + ] + acc = RewriteAccumulator() + _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1, _acc=acc + ) + assert acc.calls == 2 + assert abs(acc.elapsed - 0.8) < 1e-9 + assert acc.input_tokens == 180 + assert acc.output_tokens == 15 + assert acc.no_change == 1 + assert acc.rename == 0 + assert acc.rewrite == 0 + assert acc.edit_failures == 0 + + +@mock_patch(_PATCH_CALL_TOOL, return_value=_ok(None)) +def test_process_acc_edit_failure_on_classify_none(mock_call): + # Classify returns tool_input=None → edit_failures incremented. + acc = RewriteAccumulator() + _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1, _acc=acc + ) + assert acc.edit_failures == 1 + assert acc.no_change == 0 + assert acc.rename == 0 + assert acc.rewrite == 0 + + +@mock_patch( + _PATCH_CALL_TOOL, + return_value=_ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.before.X"}, + } + ), +) +def test_process_same_path_filtered_out(mock_call): + # Rename where old == new → filtered to empty → triggers no-change verify. + # return_value repeats for both calls; verify gets wrong type → rejects; retries + # exhaust → accept no-change. + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is False + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_string_swap_verify_accepts(mock_call): + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is True + assert "crispen.after.X" in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verify_none_accept(mock_call): + # Verify call returns tool_input=None → accept proposed renames. + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _ok(None), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is True + assert "crispen.after.X" in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verify_truncated_reject(mock_call): + # Verify call truncated → treated as rejection, renames not applied. + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _truncated_ok(), # verify truncated → reject + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 1 + ) + assert result == _SRC_WITH_PATCH + assert changed is False + assert mock_call.call_count == 2 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verify_none_accept_no_splice(mock_call, tmp_path): + # Verify returns None; function uses const ref → new_text == orig_text → no splice. + src = ( + 'TARGET = "crispen.before.X"\n\n@patch(TARGET)\ndef test_f(mock_x):\n pass\n' + ) + scan = str(tmp_path / "test_foo.py") + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _ok(None), + ] + result, changed, cross = _process_file_source( + src, {"crispen.before.X"}, "ctx", MagicMock(), _CFG, 1, scan_file=scan + ) + # No splice but const should be updated via same_file_const_map. + assert changed is True + assert "crispen.after.X" in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verify_rejected_then_accept(mock_call): + # First verify rejects; second classify+verify is accepted. + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_REJECT_WITH_CORRECTIONS), + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 + ) + assert changed is True + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verify_rejected_exhausted(mock_call): + # Verify rejects with llm_verify_retries=0 → function skipped. + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_REJECT), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 1 + ) + assert result == _SRC_WITH_PATCH + assert changed is False + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verify_rejected_exhausted_escalates_to_rewrite(mock_call): + # When llm_verify_retries>0 and rename verify retries are exhausted, + # escalate to the full rewrite path seeded with the verifier's explanation. + + cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), # classify → rename + _ok(_VERIFY_REJECT_WITH_CORRECTIONS), # verify → reject (retries left) + _ok(_CLASSIFY_RENAME), # classify (retry) → rename again + _ok( + _VERIFY_REJECT_WITH_CORRECTIONS + ), # verify → reject (retries exhausted → escalate) + _ok({"rewritten_function": _VALID_REWRITE}), # rewrite (escalated) + _ok(_REWRITE_VERIFY_OK), # verify rewrite → accept + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), cfg, 3 + ) + assert changed is True + assert mock_call.call_count == 6 diff --git a/tests/patch_rewriter/process_basic/test_renames_and_misc.py b/tests/patch_rewriter/process_basic/test_renames_and_misc.py new file mode 100644 index 0000000..94682d8 --- /dev/null +++ b/tests/patch_rewriter/process_basic/test_renames_and_misc.py @@ -0,0 +1,479 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch as mock_patch +from crispen.config import CrispenConfig +from crispen.llm_client import LLMCallResult +from crispen.patch_rewriter import ( + RewriteAccumulator, + _FLContext, + _build_context_message, + _process_file_source, +) +from ..helpers import ( + _CFG, + _CLASSIFY_RENAME, + _FORKING_PATHS, + _PATCH_CALL_PR, + _PATCH_CALL_TOOL, + _PATCH_GET_KEY_PR, + _SRC_WITH_PATCH, + _VERIFY_OK, + _VERIFY_REJECT, + _make_process_cfg, + _ok, +) +from .. import helpers + + +@mock_patch( + _PATCH_CALL_TOOL, + return_value=_ok({"needs_rewrite": False, "patch_renames": "not-a-dict"}), +) +def test_process_patch_renames_not_dict(mock_call): + # patch_renames is not a dict → treated as empty, no change. + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is False + + +@mock_patch( + _PATCH_CALL_TOOL, + return_value=_ok( + {"needs_rewrite": False, "patch_renames": {42: "crispen.after.X"}} + ), +) +def test_process_patch_renames_non_string_key(mock_call): + # Non-string key in patch_renames → filtered out. + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is False + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_patch_renames_name_invariant_filtered(mock_call): + # LLM proposes renaming crispen.before.X → crispen.before.Y (name changed from + # X to Y). A file split never renames an entity — only its module path changes. + # The rename must be filtered out, leaving no renames → triggers no-change verify. + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.before.Y"}, + } + ), + _ok(_VERIFY_OK), # no-change verify confirms + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is False + assert mock_call.call_count == 2 # classify + no-change verify + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_per_function_different_renames(mock_call): + """Two functions with the same @patch string can receive different renames. + + This is the forking case: test_a tests an entity that moved to mod1, + test_b tests an entity that moved to mod2. Each gets classified and + renamed independently. + """ + src = ( + '@patch("crispen.before.X")\ndef test_a(m):\n call_a()\n\n' + '@patch("crispen.before.X")\ndef test_b(m):\n call_b()\n' + ) + mock_call.side_effect = [ + # test_a: classify → rename to mod1 + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.mod1.X"}, + } + ), + _ok(_VERIFY_OK), + # test_b: classify → rename to mod2 + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.mod2.X"}, + } + ), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + src, {"crispen.before.X"}, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is True + assert "crispen.mod1.X" in result + assert "crispen.mod2.X" in result + assert mock_call.call_count == 4 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_per_function_both_updated(mock_call): + """Two functions with the same @patch string both get the same rename.""" + src = ( + '@patch("crispen.before.X")\ndef test_a(m):\n pass\n\n' + '@patch("crispen.before.X")\ndef test_b(m):\n pass\n' + ) + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_OK), + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + src, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is True + assert result.count("crispen.after.X") == 2 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_acc_accumulates(mock_call): + """_process_file_source accumulates calls, elapsed, and tokens into _acc.""" + mock_call.side_effect = [ + LLMCallResult( + tool_input=_CLASSIFY_RENAME, + elapsed=1.2, + input_tokens=200, + output_tokens=40, + ), + LLMCallResult( + tool_input=_VERIFY_OK, + elapsed=0.3, + input_tokens=150, + output_tokens=5, + ), + ] + acc = RewriteAccumulator() + _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1, _acc=acc + ) + assert acc.calls == 2 + assert abs(acc.elapsed - 1.5) < 1e-9 + assert acc.input_tokens == 350 + assert acc.output_tokens == 45 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verbose_prints_to_stderr(mock_call, capsys): + """verbose=True emits per-call messages to stderr.""" + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_OK), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "patch_rewriter" in err + assert "classifying" in err + assert "verifying renames" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verbose_detailed_timing(mock_call, capsys): + """timing='detailed' appends elapsed/token info after each call.""" + mock_call.side_effect = [ + LLMCallResult( + tool_input=_CLASSIFY_RENAME, + elapsed=1.23, + input_tokens=100, + output_tokens=20, + ), + LLMCallResult( + tool_input=_VERIFY_OK, + elapsed=0.45, + input_tokens=80, + output_tokens=5, + ), + ] + + cfg = CrispenConfig(patch_update_retries=1, timing="detailed") + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + cfg, + 1, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "→ done" in err + assert "1.23s" in err + assert "0.45s" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verbose_retry_label(mock_call, capsys): + """Retry attempts include '(retry)' in the verbose message.""" + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_REJECT), + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_OK), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + _CFG, + 2, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "(retry)" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verbose_verify_accepted(mock_call, capsys): + """verbose=True prints 'ACCEPTED' when verify succeeds.""" + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_OK), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "ACCEPTED" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verbose_verify_rejected_prints_issue(mock_call, capsys): + """verbose=True prints 'REJECTED' and the issue when verify rejects.""" + mock_call.side_effect = [ + _ok(_CLASSIFY_RENAME), + _ok( + { + "correct": False, + "issue": "wrong module path", + "corrections": {"crispen.before.X": "crispen.after.X"}, + } + ), + _ok(_CLASSIFY_RENAME), + _ok(_VERIFY_OK), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + _CFG, + 2, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "REJECTED" in err + assert "wrong module path" in err + assert "ACCEPTED" in err + + +@mock_patch(_PATCH_CALL_PR) +@mock_patch(helpers._PATCH_MAKE_CLIENT) +@mock_patch(_PATCH_GET_KEY_PR, return_value="key") +def test_process_file_source_candidates_reject_no_change( + mock_key, mock_client, mock_call +): + # LLM proposes no change but candidates exist → reject and retry. + # First classify: no rename → rejected by candidates check. + # Second classify: correct rename in candidates → verify → accepted. + src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' + ctx = _FLContext( + filepath="/repo/pkg/big.py", + old_module="pkg.big", + original_source="from external import A\ndef f(): A()\n", + modified_source="from .sub_a import f\n", + new_files={"sub_a.py": "from external import A\ndef f(): A()\n"}, + new_module_paths={"sub_a.py": "pkg.sub_a"}, + entity_to_target={"f": "sub_a.py"}, + forking_old_paths={"pkg.big.A"}, + ) + context_msg = _build_context_message([ctx]) + mock_call.side_effect = [ + # First classify: no rename (LLM says no change needed) + LLMCallResult( + tool_input={"needs_rewrite": False, "patch_renames": {}}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # Second classify (after candidates rejection): correct rename + LLMCallResult( + tool_input={ + "needs_rewrite": False, + "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}, + }, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # Verify rename + LLMCallResult( + tool_input={"correct": True, "corrections": {}, "issue": ""}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + ] + cg_candidates = {"test_f": {"pkg.big.A": ["pkg.sub_a.A"]}} + new_src, changed, _ = _process_file_source( + src, + {"pkg.big.A"}, + context_msg, + mock_client.return_value, + _make_process_cfg(), + max_attempts=2, + cg_candidates=cg_candidates, + ) + assert changed + assert "pkg.sub_a.A" in new_src + # Two classify calls + one verify call = 3 + assert mock_call.call_count == 3 + + +@mock_patch(_PATCH_CALL_PR) +@mock_patch(helpers._PATCH_MAKE_CLIENT) +@mock_patch(_PATCH_GET_KEY_PR, return_value="key") +def test_process_file_source_candidates_reject_verbose( + mock_key, mock_client, mock_call, capsys +): + # verbose=True prints 'candidates check rejected' when cand_issue fires. + src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' + ctx = _FLContext( + filepath="/repo/pkg/big.py", + old_module="pkg.big", + original_source="from external import A\ndef f(): A()\n", + modified_source="from .sub_a import f\n", + new_files={"sub_a.py": "from external import A\ndef f(): A()\n"}, + new_module_paths={"sub_a.py": "pkg.sub_a"}, + entity_to_target={"f": "sub_a.py"}, + forking_old_paths={"pkg.big.A"}, + ) + context_msg = _build_context_message([ctx]) + mock_call.side_effect = [ + # First classify: no rename → rejected by candidates check. + LLMCallResult( + tool_input={"needs_rewrite": False, "patch_renames": {}}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # Second classify: correct rename + LLMCallResult( + tool_input={ + "needs_rewrite": False, + "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}, + }, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # Verify rename + LLMCallResult( + tool_input={"correct": True, "corrections": {}, "issue": ""}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + ] + cg_candidates = {"test_f": {"pkg.big.A": ["pkg.sub_a.A"]}} + _process_file_source( + src, + {"pkg.big.A"}, + context_msg, + mock_client.return_value, + _make_process_cfg(), + max_attempts=2, + cg_candidates=cg_candidates, + verbose=True, + ) + err = capsys.readouterr().err + assert "candidates check rejected" in err + + +@mock_patch(_PATCH_CALL_PR) +@mock_patch(helpers._PATCH_MAKE_CLIENT) +@mock_patch(_PATCH_GET_KEY_PR, return_value="key") +def test_process_file_source_candidates_reject_bad_rename( + mock_key, mock_client, mock_call +): + # LLM proposes a rename not in candidates → rejected → retry with correct answer. + src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' + ctx = _FLContext( + filepath="/repo/pkg/big.py", + old_module="pkg.big", + original_source="from external import A\ndef f(): A()\n", + modified_source="from .sub_a import f\n", + new_files={ + "sub_a.py": "from external import A\ndef f(): A()\n", + "sub_b.py": "from external import A\ndef g(): A()\n", + }, + new_module_paths={"sub_a.py": "pkg.sub_a", "sub_b.py": "pkg.sub_b"}, + entity_to_target={"f": "sub_a.py"}, + forking_old_paths={"pkg.big.A"}, + ) + context_msg = _build_context_message([ctx]) + mock_call.side_effect = [ + # First classify: wrong rename (not in candidates) + LLMCallResult( + tool_input={ + "needs_rewrite": False, + "patch_renames": {"pkg.big.A": "pkg.sub_b.A"}, + }, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # Second classify: correct rename + LLMCallResult( + tool_input={ + "needs_rewrite": False, + "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}, + }, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # Verify + LLMCallResult( + tool_input={"correct": True, "corrections": {}, "issue": ""}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + ] + cg_candidates = {"test_f": {"pkg.big.A": ["pkg.sub_a.A"]}} + new_src, changed, _ = _process_file_source( + src, + {"pkg.big.A"}, + context_msg, + mock_client.return_value, + _make_process_cfg(), + max_attempts=2, + cg_candidates=cg_candidates, + ) + assert changed + assert "pkg.sub_a.A" in new_src + assert mock_call.call_count == 3 diff --git a/tests/patch_rewriter/test_apply_rewrite.py b/tests/patch_rewriter/test_apply_rewrite.py new file mode 100644 index 0000000..9cdd50f --- /dev/null +++ b/tests/patch_rewriter/test_apply_rewrite.py @@ -0,0 +1,498 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch as mock_patch +from crispen.llm_client import LLMCallResult +from crispen.patch_rewriter import ( + RewriteAccumulator, + _FLContext, + apply_patch_callgraph, + apply_patch_rewrite, +) +from .helpers import ( + _CFG, + _PATCH_CALL_TOOL, + _PATCH_GET_KEY, + _VERIFY_OK, + _make_fl_ctx, + _ok, +) +from . import helpers + + +def test_rewrite_empty_contexts(): + msgs = list(apply_patch_rewrite([], {}, "/repo", _CFG)) + assert msgs == [] + + +def test_rewrite_no_forking_paths(): + ctx = _make_fl_ctx(forking_old_paths=set()) + msgs = list(apply_patch_rewrite([ctx], {}, "/repo", _CFG)) + assert msgs == [] + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_per_file_update(mock_key, mock_client, mock_call): + mock_call.side_effect = [ + _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), + _ok(_VERIFY_OK), + ] + src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' + per_file = {"/repo/tests/test_big.py": {"source": src, "msgs": []}} + ctx = _make_fl_ctx() + msgs = list(apply_patch_rewrite([ctx], per_file, None, _CFG)) + updated = per_file["/repo/tests/test_big.py"]["source"] + assert "pkg.sub_a.A" in updated + assert any("patch_update" in m for m in per_file["/repo/tests/test_big.py"]["msgs"]) + assert msgs == [] # no disk messages since repo_root=None + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_no_repo_root_no_disk_scan(mock_key, mock_client, mock_call): + # repo_root=None → exits after per_file; empty per_file → no LLM calls. + msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, None, _CFG)) + assert msgs == [] + mock_call.assert_not_called() + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_disk_file_update(mock_key, mock_client, mock_call, tmp_path): + test_file = tmp_path / "test_big.py" + src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' + test_file.write_text(src, encoding="utf-8") + mock_call.side_effect = [ + _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), + _ok(_VERIFY_OK), + ] + msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) + assert "pkg.sub_a.A" in test_file.read_text(encoding="utf-8") + assert len(msgs) == 1 + assert "patch_update" in msgs[0] + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_skip_excluded_dir(mock_key, mock_client, mock_call, tmp_path): + venv = tmp_path / "venv" + venv.mkdir() + f = venv / "test_big.py" + f.write_text('@patch("pkg.big.A")\ndef test_f(): pass\n', encoding="utf-8") + msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) + assert msgs == [] + mock_call.assert_not_called() + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_skip_per_file_abs(mock_key, mock_client, mock_call, tmp_path): + # A file already in per_file should NOT be re-processed from disk. + test_file = tmp_path / "test_big.py" + test_file.write_text('@patch("pkg.big.A")\ndef test_f(): pass\n', encoding="utf-8") + original_disk = test_file.read_text(encoding="utf-8") + # per_file entry uses a source without matching patches (no LLM call needed). + per_file = {str(test_file): {"source": "# no patches\n", "msgs": []}} + list(apply_patch_rewrite([_make_fl_ctx()], per_file, str(tmp_path), _CFG)) + # Disk file untouched since it was in per_file_abs. + assert test_file.read_text(encoding="utf-8") == original_disk + mock_call.assert_not_called() + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_oserror_skipped(mock_key, mock_client, mock_call, tmp_path): + test_file = tmp_path / "test_big.py" + test_file.write_text('@patch("pkg.big.A")\ndef test_f(): pass\n', encoding="utf-8") + test_file.chmod(0o000) + try: + msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) + assert msgs == [] + mock_call.assert_not_called() + finally: + test_file.chmod(0o644) + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_disk_file_no_match_not_updated( + mock_key, mock_client, mock_call, tmp_path +): + # Disk file exists but has no matching @patch decorators → changed=False, + # file is not written, no yield message (covers the `if changed: False` branch). + test_file = tmp_path / "no_patches.py" + test_file.write_text("def test_unrelated(): pass\n", encoding="utf-8") + msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) + assert msgs == [] + assert test_file.read_text(encoding="utf-8") == "def test_unrelated(): pass\n" + mock_call.assert_not_called() + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_no_py_files_in_repo(mock_key, mock_client, mock_call, tmp_path): + # tmp_path has no .py files → disk scan loop body never executes. + msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) + assert msgs == [] + mock_call.assert_not_called() + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_acc_tracks_calls_and_files(mock_key, mock_client, mock_call, tmp_path): + """RewriteAccumulator is populated with call counts and files_updated.""" + test_file = tmp_path / "test_big.py" + src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' + test_file.write_text(src, encoding="utf-8") + mock_call.side_effect = [ + LLMCallResult( + tool_input={ + "needs_rewrite": False, + "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}, + }, + elapsed=1.5, + input_tokens=100, + output_tokens=50, + ), + LLMCallResult( + tool_input=_VERIFY_OK, + elapsed=0.5, + input_tokens=80, + output_tokens=10, + ), + ] + acc = RewriteAccumulator() + list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG, _acc=acc)) + assert acc.calls == 2 + assert acc.elapsed == 2.0 + assert acc.input_tokens == 180 + assert acc.output_tokens == 60 + assert acc.files_updated == 1 + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_acc_per_file_files_updated(mock_key, mock_client, mock_call): + """files_updated is incremented for in-memory per_file changes.""" + src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' + per_file = {"/repo/tests/test_big.py": {"source": src, "msgs": []}} + mock_call.side_effect = [ + _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), + _ok(_VERIFY_OK), + ] + acc = RewriteAccumulator() + list(apply_patch_rewrite([_make_fl_ctx()], per_file, None, _CFG, _acc=acc)) + assert acc.files_updated == 1 + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_cross_file_const_per_file(mock_key, mock_client, mock_call, tmp_path): + """Cross-file const whose source is in per_file gets updated in-memory.""" + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "pkg.big.A"\n', encoding="utf-8") + # test_foo.py imports TARGET from helpers and uses it in @patch. + test_src = ( + "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(m):\n pass\n" + ) + helpers_state = {"source": 'TARGET = "pkg.big.A"\n', "msgs": []} + per_file = { + str(tmp_path / "test_foo.py"): {"source": test_src, "msgs": []}, + str(helpers): helpers_state, + } + mock_call.side_effect = [ + _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), + _ok(_VERIFY_OK), + ] + list(apply_patch_rewrite([_make_fl_ctx()], per_file, None, _CFG)) + # The constant definition in helpers.py (per_file entry) should be updated. + assert '"pkg.sub_a.A"' in helpers_state["source"] + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_cross_file_const_disk(mock_key, mock_client, mock_call, tmp_path): + """Cross-file const on disk (not in per_file) gets written directly.""" + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "pkg.big.A"\n', encoding="utf-8") + test_src = ( + "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(m):\n pass\n" + ) + test_file = tmp_path / "test_foo.py" + test_file.write_text(test_src, encoding="utf-8") + mock_call.side_effect = [ + _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), + _ok(_VERIFY_OK), + ] + list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) + # The constant definition on disk should be updated. + assert '"pkg.sub_a.A"' in helpers.read_text(encoding="utf-8") + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_cross_file_const_per_file_acc( + mock_key, mock_client, mock_call, tmp_path +): + """_acc.files_updated is incremented when a cross-file const in per_file changes.""" + (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "pkg.big.A"\n', encoding="utf-8") + test_src = ( + "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(m):\n pass\n" + ) + helpers_state = {"source": 'TARGET = "pkg.big.A"\n', "msgs": []} + per_file = { + str(tmp_path / "test_foo.py"): {"source": test_src, "msgs": []}, + str(helpers): helpers_state, + } + mock_call.side_effect = [ + _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), + _ok(_VERIFY_OK), + ] + acc = RewriteAccumulator() + list(apply_patch_rewrite([_make_fl_ctx()], per_file, None, _CFG, _acc=acc)) + # One file_updated for the test_foo.py source change, one for helpers const. + assert acc.files_updated >= 1 + assert '"pkg.sub_a.A"' in helpers_state["source"] + + +@mock_patch(_PATCH_CALL_TOOL) +@mock_patch(helpers._PATCH_MAKE_CLIENT, return_value=MagicMock()) +@mock_patch(_PATCH_GET_KEY, return_value="fake_key") +def test_rewrite_cross_file_const_disk_acc(mock_key, mock_client, mock_call, tmp_path): + """_acc.files_updated is incremented when a cross-file const on disk changes.""" + helpers = tmp_path / "helpers.py" + helpers.write_text('TARGET = "pkg.big.A"\n', encoding="utf-8") + test_src = ( + "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(m):\n pass\n" + ) + test_file = tmp_path / "test_foo.py" + test_file.write_text(test_src, encoding="utf-8") + mock_call.side_effect = [ + _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), + _ok(_VERIFY_OK), + ] + acc = RewriteAccumulator() + list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG, _acc=acc)) + assert acc.files_updated >= 1 + + +def test_apply_patch_callgraph_empty_contexts(): + result = list(apply_patch_callgraph([], {}, "/repo")) + assert result == [] + + +def test_apply_patch_callgraph_no_forking_paths(): + ctx = _make_fl_ctx(forking_old_paths=set()) + result = list(apply_patch_callgraph([ctx], {}, "/repo")) + assert result == [] + + +def test_apply_patch_callgraph_per_file_update(tmp_path): + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + conflict_src = "from external import use_fn\ndef resolve(): use_fn()\n" + ctx = _FLContext( + filepath=str(tmp_path / "pkg" / "orig.py"), + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src, "conflict.py": conflict_src}, + new_module_paths={ + "placement.py": "pkg.placement", + "conflict.py": "pkg.conflict", + }, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + file_src = ( + "from pkg.placement import helper\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(mock_use_fn):\n" + " helper()\n" + ) + test_file = tmp_path / "test_orig.py" + test_file.write_text(file_src, encoding="utf-8") + per_file = {str(test_file): {"source": file_src, "msgs": []}} + list(apply_patch_callgraph([ctx], per_file, str(tmp_path))) + assert '@patch("pkg.placement.use_fn")' in per_file[str(test_file)]["source"] + + +def test_apply_patch_callgraph_repo_scan(tmp_path): + test_file = tmp_path / "test_something.py" + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + test_file.write_text( + "from pkg.placement import helper\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(mock_use_fn):\n" + " helper()\n", + encoding="utf-8", + ) + ctx = _FLContext( + filepath=str(tmp_path / "pkg" / "orig.py"), + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + msgs = list(apply_patch_callgraph([ctx], {}, str(tmp_path))) + updated = test_file.read_text(encoding="utf-8") + assert '@patch("pkg.placement.use_fn")' in updated + assert any("call-graph" in m for m in msgs) + + +def test_apply_patch_callgraph_repo_scan_no_change(tmp_path): + test_file = tmp_path / "test_something.py" + test_file.write_text("def test_f(): pass\n", encoding="utf-8") + ctx = _FLContext( + filepath=str(tmp_path / "pkg" / "orig.py"), + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": "from external import use_fn\ndef f(): use_fn()\n"}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + msgs = list(apply_patch_callgraph([ctx], {}, str(tmp_path))) + assert msgs == [] + + +def test_apply_patch_callgraph_repo_root_none(): + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={}, + new_module_paths={}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + result = list(apply_patch_callgraph([ctx], {}, None)) + assert result == [] + + +def test_apply_patch_callgraph_per_file_no_change(tmp_path): + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + ctx = _FLContext( + filepath=str(tmp_path / "pkg" / "orig.py"), + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + file_src = "# pkg.orig.use_fn mentioned here but no test functions.\nx = 1\n" + key = str(tmp_path / "module.py") + per_file = {key: {"source": file_src, "msgs": []}} + list(apply_patch_callgraph([ctx], per_file, None)) + assert per_file[key]["source"] == file_src + + +def test_apply_patch_callgraph_per_file_no_match(tmp_path): + """per_file entry whose source contains no forking path string → continue.""" + ctx = _FLContext( + filepath=str(tmp_path / "pkg" / "orig.py"), + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n" + }, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + file_src = "x = 1\n" # no mention of forking path + key = str(tmp_path / "module.py") + per_file = {key: {"source": file_src, "msgs": []}} + list(apply_patch_callgraph([ctx], per_file, None)) + assert per_file[key]["source"] == file_src + + +def test_apply_patch_callgraph_repo_scan_oserror(tmp_path): + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + ctx = _FLContext( + filepath=str(tmp_path / "orig.py"), + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + bad_file = tmp_path / "test_bad.py" + bad_file.write_text( + '@patch("pkg.orig.use_fn")\ndef test_f(): helper()\n', encoding="utf-8" + ) + bad_file.chmod(0o000) + try: + msgs = list(apply_patch_callgraph([ctx], {}, str(tmp_path))) + assert msgs == [] + finally: + bad_file.chmod(0o644) + + +def test_apply_patch_callgraph_repo_scan_file_no_change(tmp_path): + test_file = tmp_path / "helper.py" + test_file.write_text( + "# references pkg.orig.use_fn in a comment\nx = 1\n", + encoding="utf-8", + ) + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + ctx = _FLContext( + filepath=str(tmp_path / "orig.py"), + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + msgs = list(apply_patch_callgraph([ctx], {}, str(tmp_path))) + assert msgs == [] + assert "x = 1" in test_file.read_text(encoding="utf-8") + + +def test_apply_patch_callgraph_excluded_dirs(tmp_path): + venv_dir = tmp_path / ".venv" + venv_dir.mkdir() + excluded_file = venv_dir / "test_something.py" + excluded_file.write_text( + '@patch("pkg.orig.use_fn")\ndef test_f(): helper()\n', encoding="utf-8" + ) + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + ctx = _FLContext( + filepath=str(tmp_path / "orig.py"), + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + msgs = list(apply_patch_callgraph([ctx], {}, str(tmp_path))) + assert '@patch("pkg.orig.use_fn")' in excluded_file.read_text(encoding="utf-8") + assert msgs == [] diff --git a/tests/patch_rewriter/test_callgraph_helpers.py b/tests/patch_rewriter/test_callgraph_helpers.py new file mode 100644 index 0000000..22cef6a --- /dev/null +++ b/tests/patch_rewriter/test_callgraph_helpers.py @@ -0,0 +1,451 @@ +from __future__ import annotations +from crispen.patch_rewriter import ( + _CgIndex, + _FLContext, + _cg_build_index, + _cg_collect_called_names, + _cg_collect_defined_names, + _cg_collect_func_body_calls, + _cg_file_to_module_and_package, + _cg_parse_imports, + _cg_resolve_call_to_import, +) + + +def test_cg_collect_called_names_name_and_attr(): + src = "foo()\nobj.bar()\n" + result = _cg_collect_called_names(src) + assert "foo" in result + assert "bar" in result + + +def test_cg_collect_called_names_complex_func(): + # f()() — outer call's func is a Call node (neither Name nor Attribute). + src = "f()()\n" + result = _cg_collect_called_names(src) + # Only the inner call's name is collected (f), the outer call is skipped. + assert "f" in result + + +def test_cg_collect_called_names_parse_error(): + assert _cg_collect_called_names("def f(:\n") == set() + + +def test_cg_collect_called_names_no_calls(): + assert _cg_collect_called_names("x = 1\n") == set() + + +def test_cg_collect_func_body_calls_found(): + src = "def helper(): foo()\ndef other(): bar()\n" + result = _cg_collect_func_body_calls(src, "helper") + assert "foo" in result + assert "bar" not in result + + +def test_cg_collect_func_body_calls_not_found(): + src = "def helper(): foo()\n" + assert _cg_collect_func_body_calls(src, "missing") == set() + + +def test_cg_collect_func_body_calls_parse_error(): + assert _cg_collect_func_body_calls("def f(:\n", "f") == set() + + +def test_cg_collect_func_body_calls_attribute_call(): + src = "def helper(): obj.method()\n" + result = _cg_collect_func_body_calls(src, "helper") + assert "method" in result + + +def test_cg_collect_func_body_calls_complex_func(): + # f()() inside a function body — outer call's func is a Call, not Name/Attribute. + src = "def helper(): f()()\n" + result = _cg_collect_func_body_calls(src, "helper") + assert "f" in result # inner call collected; outer (complex func) silently skipped + + +def test_cg_collect_func_body_calls_skips_non_function_nodes(): + # Module-level assignment before the function — should be skipped. + src = "X = 1\ndef helper(): foo()\n" + result = _cg_collect_func_body_calls(src, "helper") + assert "foo" in result + + +def test_cg_collect_called_names_alias_access_emits_pair(): + # ``m.func()`` should emit both ``"func"`` and ``"m.func"``. + src = "import mymod as m\nm.func()\n" + result = _cg_collect_called_names(src) + assert "func" in result + assert "m.func" in result + + +def test_cg_collect_called_names_nested_attr_no_alias_pair(): + # ``a.b.c()`` — the receiver of ``.c`` is itself an Attribute, not a Name; + # only the bare attr name is emitted (no alias pair for chained access). + src = "a.b.c()\n" + result = _cg_collect_called_names(src) + assert "c" in result + assert "b.c" not in result # receiver is Attribute, not Name + + +def test_cg_collect_func_body_calls_alias_access_emits_pair(): + src = "import mymod as m\ndef helper(): m.process()\n" + result = _cg_collect_func_body_calls(src, "helper") + assert "process" in result + assert "m.process" in result + + +def test_cg_collect_func_body_calls_nested_attr_no_alias_pair(): + # Chained access ``a.b.c()`` — receiver of attr c is not a Name. + src = "def helper(): a.b.c()\n" + result = _cg_collect_func_body_calls(src, "helper") + assert "c" in result + assert "b.c" not in result + + +def test_cg_resolve_call_plain_name(): + imports = {"foo": ("pkg.sub", "foo"), "bar": ("pkg.other", "bar")} + assert _cg_resolve_call_to_import("foo", imports) == ("pkg.sub", "foo") + + +def test_cg_resolve_call_alias_attr(): + # ``m.process()`` — alias ``m`` maps to module ``mymod``; resolves to + # ``(mymod, "process")``. + imports = {"m": ("mymod", "mymod")} + assert _cg_resolve_call_to_import("m.process", imports) == ("mymod", "process") + + +def test_cg_resolve_call_alias_attr_unknown_alias(): + # Alias not in imports → None. + assert _cg_resolve_call_to_import("unknown.func", {"m": ("mymod", "mymod")}) is None + + +def test_cg_resolve_call_plain_not_found(): + assert _cg_resolve_call_to_import("missing", {"foo": ("pkg", "foo")}) is None + + +def test_cg_collect_defined_names_functions_and_classes(): + src = "def foo(): pass\nclass Bar: pass\nasync def baz(): pass\n" + result = _cg_collect_defined_names(src) + assert result == {"foo", "Bar", "baz"} + + +def test_cg_collect_defined_names_parse_error(): + assert _cg_collect_defined_names("def f(:\n") == set() + + +def test_cg_collect_defined_names_empty(): + assert _cg_collect_defined_names("x = 1\n") == set() + + +def test_cg_file_to_module_regular(tmp_path): + pkg = tmp_path / "pkg" + pkg.mkdir() + f = pkg / "helpers.py" + f.touch() + mod, pkg_path = _cg_file_to_module_and_package(f, tmp_path) + assert mod == "pkg.helpers" + assert pkg_path == "pkg" + + +def test_cg_file_to_module_init(tmp_path): + d = tmp_path / "pkg" / "utils" + d.mkdir(parents=True) + f = d / "__init__.py" + f.touch() + mod, pkg_path = _cg_file_to_module_and_package(f, tmp_path) + assert mod == "pkg.utils" + assert pkg_path == "pkg.utils" + + +def test_cg_file_to_module_top_level(tmp_path): + f = tmp_path / "helpers.py" + f.touch() + mod, pkg_path = _cg_file_to_module_and_package(f, tmp_path) + assert mod == "helpers" + assert pkg_path == "" + + +def test_cg_file_to_module_nested(tmp_path): + d = tmp_path / "a" / "b" + d.mkdir(parents=True) + f = d / "c.py" + f.touch() + mod, pkg_path = _cg_file_to_module_and_package(f, tmp_path) + assert mod == "a.b.c" + assert pkg_path == "a.b" + + +def test_cg_parse_imports_from_import(): + assert _cg_parse_imports("from pkg.sub import foo\n", "pkg") == { + "foo": ("pkg.sub", "foo") + } + + +def test_cg_parse_imports_import_simple(): + result = _cg_parse_imports("import os\n", "pkg") + assert result["os"] == ("os", "os") + + +def test_cg_parse_imports_import_dotted(): + result = _cg_parse_imports("import pkg.sub\n", "") + assert result["pkg"] == ("pkg.sub", "pkg.sub") + + +def test_cg_parse_imports_import_as(): + result = _cg_parse_imports("import os as o\n", "pkg") + assert result["o"] == ("os", "os") + + +def test_cg_parse_imports_from_import_as(): + assert _cg_parse_imports("from pkg import foo as bar\n", "pkg") == { + "bar": ("pkg", "foo") + } + + +def test_cg_parse_imports_relative_level1(): + # `from . import helper` with package "pkg.sub" → mod = "pkg.sub" + assert _cg_parse_imports("from . import helper\n", "pkg.sub") == { + "helper": ("pkg.sub", "helper") + } + + +def test_cg_parse_imports_relative_level2(): + # `from .. import foo` with package "pkg.sub" → base = "pkg" + assert _cg_parse_imports("from .. import foo\n", "pkg.sub") == { + "foo": ("pkg", "foo") + } + + +def test_cg_parse_imports_relative_with_module(): + # `from .utils import helper` with package "pkg" → mod = "pkg.utils" + assert _cg_parse_imports("from .utils import helper\n", "pkg") == { + "helper": ("pkg.utils", "helper") + } + + +def test_cg_parse_imports_relative_with_empty_base(): + # `from .sub import foo` with empty package → base="" → mod = "sub" + assert _cg_parse_imports("from .sub import foo\n", "") == {"foo": ("sub", "foo")} + + +def test_cg_parse_imports_relative_no_module(): + # `from . import bar` with package "pkg.sub" → mod = "pkg.sub" + assert _cg_parse_imports("from . import bar\n", "pkg.sub") == { + "bar": ("pkg.sub", "bar") + } + + +def test_cg_parse_imports_star_skipped(): + assert _cg_parse_imports("from pkg import *\n", "pkg") == {} + + +def test_cg_parse_imports_syntax_error(): + assert _cg_parse_imports("def f(:\n", "pkg") == {} + + +def test_cg_parse_imports_too_deep_relative(): + # level=3 with package="pkg" → go_up=2 > len(["pkg"])=1 → skipped + assert _cg_parse_imports("from ... import foo\n", "pkg") == {} + + +def test_cg_parse_imports_level2_with_submodule(): + # `from ..utils import foo` with package "pkg.sub" → base="pkg" → "pkg.utils" + assert _cg_parse_imports("from ..utils import foo\n", "pkg.sub") == { + "foo": ("pkg.utils", "foo") + } + + +def test_cg_index_get_imports_cached(): + index = _CgIndex( + module_to_source={"pkg.mod": "from pkg.sub import foo\n"}, + module_to_package={"pkg.mod": "pkg"}, + module_to_defs={"pkg.mod": set()}, + file_to_module={}, + ) + r1 = index.get_imports("pkg.mod") + r2 = index.get_imports("pkg.mod") # second call — cached + assert r1 == r2 == {"foo": ("pkg.sub", "foo")} + assert "pkg.mod" in index._import_cache + + +def test_cg_index_get_imports_missing_module(): + index = _CgIndex( + module_to_source={}, + module_to_package={}, + module_to_defs={}, + file_to_module={}, + ) + assert index.get_imports("nonexistent") == {} + + +def test_cg_build_index_from_repo(tmp_path): + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "mod.py").write_text("def foo(): pass\n", encoding="utf-8") + index = _cg_build_index(str(tmp_path), {}, []) + assert "pkg.mod" in index.module_to_source + assert "foo" in index.module_to_defs["pkg.mod"] + assert index.module_to_package["pkg.mod"] == "pkg" + + +def test_cg_build_index_per_file_override(tmp_path): + pkg = tmp_path / "pkg" + pkg.mkdir() + f = pkg / "mod.py" + f.write_text("def old(): pass\n", encoding="utf-8") + abs_path = str(f.resolve()) + index = _cg_build_index(str(tmp_path), {abs_path: "def new(): pass\n"}, []) + assert "new" in index.module_to_defs["pkg.mod"] + assert "old" not in index.module_to_defs["pkg.mod"] + + +def test_cg_build_index_no_repo_root(): + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="", + modified_source="", + new_files={"placement.py": "def helper(): pass\n"}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths=set(), + ) + index = _cg_build_index(None, {}, [ctx]) + assert "pkg.placement" in index.module_to_source + assert index.file_to_module == {} + + +def test_cg_build_index_excluded_dirs(tmp_path): + venv = tmp_path / ".venv" + venv.mkdir() + (venv / "mod.py").write_text("def foo(): pass\n", encoding="utf-8") + index = _cg_build_index(str(tmp_path), {}, []) + assert "mod" not in index.module_to_source + + +def test_cg_build_index_new_files_from_context(): + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="", + modified_source="", + new_files={"placement.py": "def helper(): pass\n"}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths=set(), + ) + index = _cg_build_index(None, {}, [ctx]) + assert "helper" in index.module_to_defs["pkg.placement"] + assert index.module_to_package["pkg.placement"] == "pkg" + + +def test_cg_build_index_init_package(tmp_path): + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("from .sub import foo\n", encoding="utf-8") + (pkg / "sub.py").write_text("def foo(): pass\n", encoding="utf-8") + index = _cg_build_index(str(tmp_path), {}, []) + assert "pkg" in index.module_to_source + assert index.module_to_package["pkg"] == "pkg" + + +def test_cg_build_index_already_in_index(): + ctx1 = _FLContext( + filepath="/proj/orig.py", + old_module="orig", + original_source="", + modified_source="", + new_files={"placement.py": "def first(): pass\n"}, + new_module_paths={"placement.py": "pkg.shared"}, + entity_to_target={}, + forking_old_paths=set(), + ) + ctx2 = _FLContext( + filepath="/proj/orig2.py", + old_module="orig2", + original_source="", + modified_source="", + new_files={"placement.py": "def second(): pass\n"}, + new_module_paths={"placement.py": "pkg.shared"}, # same module path + entity_to_target={}, + forking_old_paths=set(), + ) + index = _cg_build_index(None, {}, [ctx1, ctx2]) + assert "first" in index.module_to_defs["pkg.shared"] + assert "second" not in index.module_to_defs["pkg.shared"] + + +def test_cg_build_index_oserror(tmp_path): + pkg = tmp_path / "pkg" + pkg.mkdir() + bad = pkg / "bad.py" + bad.write_text("def foo(): pass\n", encoding="utf-8") + bad.chmod(0o000) + try: + index = _cg_build_index(str(tmp_path), {}, []) + assert "pkg.bad" not in index.module_to_source + finally: + bad.chmod(0o644) + + +def test_cg_build_index_missing_module_path(): + ctx = _FLContext( + filepath="/proj/orig.py", + old_module="orig", + original_source="", + modified_source="", + new_files={"placement.py": "def helper(): pass\n"}, + new_module_paths={}, # rel_path missing → new_mod = None → skip + entity_to_target={}, + forking_old_paths=set(), + ) + index = _cg_build_index(None, {}, [ctx]) + assert "placement.py" not in index.module_to_source + + +def test_cg_build_index_empty_src(): + ctx = _FLContext( + filepath="/proj/orig.py", + old_module="orig", + original_source="", + modified_source="", + new_files={"placement.py": ""}, # empty src → skipped + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths=set(), + ) + index = _cg_build_index(None, {}, [ctx]) + assert "pkg.placement" not in index.module_to_source + + +def test_cg_build_index_init_package_new_file(): + # __init__.py as a new file: pkg = new_mod (not rsplit) + ctx = _FLContext( + filepath="/proj/orig.py", + old_module="orig", + original_source="", + modified_source="", + new_files={"__init__.py": "def init_fn(): pass\n"}, + new_module_paths={"__init__.py": "pkg.sub"}, + entity_to_target={}, + forking_old_paths=set(), + ) + index = _cg_build_index(None, {}, [ctx]) + assert index.module_to_package["pkg.sub"] == "pkg.sub" + + +def test_cg_build_index_top_level_new_file(): + # new_mod without a dot → package = "" + ctx = _FLContext( + filepath="/proj/orig.py", + old_module="orig", + original_source="", + modified_source="", + new_files={"placement.py": "def helper(): pass\n"}, + new_module_paths={"placement.py": "placement"}, # no dot + entity_to_target={}, + forking_old_paths=set(), + ) + index = _cg_build_index(None, {}, [ctx]) + assert index.module_to_package.get("placement") == "" diff --git a/tests/patch_rewriter/test_callgraph_resolution.py b/tests/patch_rewriter/test_callgraph_resolution.py new file mode 100644 index 0000000..0acd390 --- /dev/null +++ b/tests/patch_rewriter/test_callgraph_resolution.py @@ -0,0 +1,1005 @@ +from __future__ import annotations +from crispen.patch_rewriter import ( + _CG_MAX_DEPTH, + _CG_MAX_MODULES, + _CgIndex, + _FLContext, + _cg_collect_defined_names, + _expand_module_terminals, + _resolve_forking_path_candidates, + _resolve_forking_path_via_callgraph, +) +from .helpers import _make_bfs_ctx, _make_bfs_index + + +def test_resolve_callgraph_no_calling_module(): + ctx = _make_bfs_ctx() + index = _make_bfs_index("from pkg.placement import helper\n") + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): helper()\n", ctx, index, "" + ) + assert result is None + + +def test_resolve_callgraph_pre_check_fails(): + # original_source has no external import of 'use_fn' → pre-check fails + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="def helper(): use_fn()\n", # not imported externally + modified_source="", + new_files={"placement.py": "def helper(): use_fn()\n"}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + index = _make_bfs_index("from pkg.placement import helper\n") + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_no_terminal(): + # New files don't reference 'use_fn' → terminal empty → None + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": "def helper(): pass\n"}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + index = _make_bfs_index("from pkg.placement import helper\n") + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_direct_call(): + # Test directly calls 'helper'; helper in placement uses use_fn. + ctx = _make_bfs_ctx() + index = _make_bfs_index("from pkg.placement import helper\n") + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" + ) + assert result == "pkg.placement.use_fn" + + +def test_resolve_callgraph_multi_hop(): + # Test → intermediary → helper → terminal (placement.use_fn) + ctx = _make_bfs_ctx() + middle_src = "from pkg.placement import helper\ndef intermediary(): helper()\n" + test_src = "from pkg.middle import intermediary\n" + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, + module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, + module_to_defs={"pkg.test_mod": set(), "pkg.middle": {"intermediary"}}, + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): intermediary()\n", ctx, index, "pkg.test_mod" + ) + assert result == "pkg.placement.use_fn" + + +def test_resolve_callgraph_reexport(): + # Test imports helper from pkg.orig; pkg.orig re-exports helper from placement. + # Re-export is followed without incrementing depth. + ctx = _make_bfs_ctx() + orig_src = "from .placement import helper\n" # re-exports (fn not defined) + test_src = "from pkg.orig import helper\n" + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src, "pkg.orig": orig_src}, + module_to_package={"pkg.test_mod": "pkg", "pkg.orig": "pkg"}, + module_to_defs={"pkg.test_mod": set(), "pkg.orig": set()}, # helper not defined + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" + ) + assert result == "pkg.placement.use_fn" + + +def test_resolve_callgraph_multiple_candidates(): + # Both placement.helper and conflict.resolve are reachable → ambiguous → None. + ctx = _make_bfs_ctx() + test_src = "from pkg.placement import helper\n" "from pkg.conflict import resolve\n" + index = _make_bfs_index(test_src) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): helper(); resolve()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_not_reachable(): + # Test doesn't import anything relevant → BFS queue empty → None. + ctx = _make_bfs_ctx() + index = _make_bfs_index("") # no imports + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): unrelated()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_new_submodule_non_terminal(): + # Test imports 'other' from placement; 'other' doesn't use use_fn. + # 'other' is in a new sub-module but NOT in terminal → skipped. + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "placement.py": ( + "from external import use_fn\n" + "def helper(): use_fn()\n" + "def other(): pass\n" + ) + }, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = "from pkg.placement import other\n" + index = _make_bfs_index(test_src) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): other()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_init_reexport(): + # pkg/orig.py split into pkg/orig/__init__.py (re-exports helper) and + # pkg/orig/placement.py (defines helper, uses use_fn). + # Test imports helper from pkg.orig (the new __init__). + # __init__ is excluded from new_module_set so BFS traverses through it + # and follows the re-export to placement.py, finding the terminal. + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "orig/__init__.py": "from .placement import helper\n", + "orig/placement.py": ( + "from external import use_fn\ndef helper(): use_fn()\n" + ), + }, + new_module_paths={ + "orig/__init__.py": "pkg.orig", + "orig/placement.py": "pkg.orig.placement", + }, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + init_src = "from .placement import helper\n" + test_src = "from pkg.orig import helper\n" + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src, "pkg.orig": init_src}, + module_to_package={"pkg.test_mod": "pkg", "pkg.orig": "pkg.orig"}, + module_to_defs={"pkg.test_mod": set(), "pkg.orig": set()}, + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" + ) + assert result == "pkg.orig.placement.use_fn" + + +def test_resolve_callgraph_visited_dedup(): + # 'intermediary' and 'inter2' both map to same (module, func); processed once. + ctx = _make_bfs_ctx() + middle_src = "from pkg.placement import helper\ndef intermediary(): helper()\n" + test_src = ( + "from pkg.middle import intermediary\n" + "from pkg.middle import intermediary as inter2\n" + ) + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, + module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, + module_to_defs={"pkg.test_mod": set(), "pkg.middle": {"intermediary"}}, + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", + "def test_f(): intermediary(); inter2()\n", + ctx, + index, + "pkg.test_mod", + ) + assert result == "pkg.placement.use_fn" + + +def test_resolve_callgraph_empty_new_file(): + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "empty.py": "", + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", + }, + new_module_paths={"empty.py": "pkg.empty", "placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + index = _make_bfs_index("from pkg.placement import helper\n") + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" + ) + assert result == "pkg.placement.use_fn" + + +def test_resolve_callgraph_missing_module_path(): + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": "from external import use_fn\ndef f(): use_fn()\n"}, + new_module_paths={}, # missing → terminal empty → None + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + index = _make_bfs_index("from pkg.placement import f\n") + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): f()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_missing_source(): + # Called function's module is not in the index → src=None → continue + ctx = _make_bfs_ctx() + test_src = "from pkg.missing import something\n" + index = _make_bfs_index(test_src) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): something()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_func_defined_no_calls(): + # Function IS defined but has no imported calls → BFS dead-end → None + ctx = _make_bfs_ctx() + middle_src = "def standalone(): pass\n" + test_src = "from pkg.middle import standalone\n" + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, + module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, + module_to_defs={"pkg.test_mod": set(), "pkg.middle": {"standalone"}}, + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): standalone()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_body_call_not_importable(): + # Function's body calls something not in its import map → BFS dead-end → None + ctx = _make_bfs_ctx() + middle_src = "def fn(): bar()\n" # bar not imported + test_src = "from pkg.middle import fn\n" + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, + module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, + module_to_defs={"pkg.test_mod": set(), "pkg.middle": {"fn"}}, + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): fn()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_func_not_defined_not_reexported(): + # func_name not defined and not re-exported → BFS dead-end → None + ctx = _make_bfs_ctx() + middle_src = "def other(): pass\n" # 'fn' not defined, not re-exported + test_src = "from pkg.middle import fn\n" + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, + module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, + module_to_defs={"pkg.test_mod": set(), "pkg.middle": {"other"}}, + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): fn()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_body_call_already_visited(): + # fn_a → fn_b → fn_a (mutual recursion); fn_a already visited when fn_b adds it + ctx = _make_bfs_ctx() + m_a = "from pkg.m_b import fn_b\ndef fn_a(): fn_b()\n" + m_b = "from pkg.m_a import fn_a\ndef fn_b(): fn_a()\n" + test_src = "from pkg.m_a import fn_a\n" + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src, "pkg.m_a": m_a, "pkg.m_b": m_b}, + module_to_package={ + "pkg.test_mod": "pkg", + "pkg.m_a": "pkg", + "pkg.m_b": "pkg", + }, + module_to_defs={ + "pkg.test_mod": set(), + "pkg.m_a": {"fn_a"}, + "pkg.m_b": {"fn_b"}, + }, + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): fn_a()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_reexport_already_visited(): + # m_x re-exports fn_x from m_y; m_y re-exports fn_x from m_x (cycle). + # When m_y checks re-export, (m_x, fn_x) is already visited → skip. + ctx = _make_bfs_ctx() + m_x = "from pkg.m_y import fn_x\n" # re-exports fn_x from m_y + m_y = "from pkg.m_x import fn_x\n" # re-exports fn_x from m_x (cycle) + test_src = "from pkg.m_x import fn_x\n" + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src, "pkg.m_x": m_x, "pkg.m_y": m_y}, + module_to_package={ + "pkg.test_mod": "pkg", + "pkg.m_x": "pkg", + "pkg.m_y": "pkg", + }, + module_to_defs={"pkg.test_mod": set(), "pkg.m_x": set(), "pkg.m_y": set()}, + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): fn_x()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_depth_limit(): + # Chain of _CG_MAX_DEPTH + 1 hops; last function calls terminal but is cut off. + n = _CG_MAX_DEPTH + 1 # 13 intermediate functions + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"end.py": "from external import use_fn\ndef end_fn(): use_fn()\n"}, + new_module_paths={"end.py": "pkg.end"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + modules = {} + for i in range(n): + if i < n - 1: + src = f"from pkg.m{i + 1} import f{i + 1}\ndef f{i}(): f{i + 1}()\n" + else: + src = f"from pkg.end import end_fn\ndef f{i}(): end_fn()\n" + modules[f"pkg.m{i}"] = src + modules["pkg.test_mod"] = "from pkg.m0 import f0\n" + defs = {m: _cg_collect_defined_names(s) for m, s in modules.items()} + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs=defs, + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): f0()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_module_limit(): + # Re-export chain of _CG_MAX_MODULES + 1 unique modules; 51st is cut off. + n = _CG_MAX_MODULES # 50 re-export hops before the cut-off + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"end.py": "from external import use_fn\ndef final(): use_fn()\n"}, + new_module_paths={"end.py": "pkg.end"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + modules = {} + for i in range(n + 1): + if i < n: + modules[f"pkg.m{i}"] = f"from pkg.m{i + 1} import fn\n" + else: + modules[f"pkg.m{i}"] = "from pkg.end import final\ndef fn(): final()\n" + modules["pkg.test_mod"] = "from pkg.m0 import fn\n" + defs = {m: _cg_collect_defined_names(s) for m, s in modules.items()} + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs=defs, + file_to_module={}, + ) + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): fn()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_callgraph_same_module_twice(): + """Two called names both resolve to the same intermediate module. + + The second BFS entry hits the 'module already in modules_seen' fast path + (branch 1250->1255 in patch_rewriter.py). + """ + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"end.py": "from external import use_fn\ndef final(): use_fn()\n"}, + new_module_paths={"end.py": "pkg.end"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + # Both fn_a and fn_b live in pkg.middle; neither calls anything reachable. + middle_src = "def fn_a(): pass\ndef fn_b(): pass\n" + modules = { + "pkg.test_mod": "from pkg.middle import fn_a, fn_b\n", + "pkg.middle": middle_src, + } + defs = {m: _cg_collect_defined_names(s) for m, s in modules.items()} + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs=defs, + file_to_module={}, + ) + # BFS enqueues (pkg.middle, fn_a, 0) and (pkg.middle, fn_b, 0). + # First pop adds pkg.middle to modules_seen; second pop hits the fast path. + result = _resolve_forking_path_via_callgraph( + "use_fn", "def test_f(): fn_a(); fn_b()\n", ctx, index, "pkg.test_mod" + ) + assert result is None + + +def test_resolve_forking_path_candidates_single(): + # Single candidate: path returned, candidates=[path], truncated=False. + ctx = _make_bfs_ctx() + index = _make_bfs_index("from pkg.placement import helper\n") + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" + ) + assert path == "pkg.placement.use_fn" + assert cands == ["pkg.placement.use_fn"] + assert not truncated + + +def test_resolve_forking_path_candidates_multiple(): + # Multiple candidates → path=None, cands=[...], truncated=False. + ctx = _make_bfs_ctx() + test_src = "from pkg.placement import helper\nfrom pkg.conflict import resolve\n" + index = _make_bfs_index(test_src) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", + "def test_f(): helper(); resolve()\n", + ctx, + index, + "pkg.test_mod", + ) + assert path is None + assert sorted(cands) == ["pkg.conflict.use_fn", "pkg.placement.use_fn"] + assert not truncated + + +def test_resolve_forking_path_candidates_no_calling_module(): + ctx = _make_bfs_ctx() + index = _make_bfs_index("from pkg.placement import helper\n") + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", "def test_f(): helper()\n", ctx, index, "" + ) + assert path is None + assert cands == [] + assert not truncated + + +def test_resolve_forking_path_candidates_truncated_depth(): + # Chain of exactly _CG_MAX_DEPTH + 1 hops; the last hop is cut off → truncated=True. + # Chain: test_mod -[f0]-> mid0 -> mid1 -> ... -> mid{n-1} -[helper]-> placement + # n = _CG_MAX_DEPTH + 1 intermediate modules; helper is at depth n-1 = 13, + # but the depth limit cuts off at depth 12 before enqueuing helper. + n = _CG_MAX_DEPTH + 1 # 13 hops from test_mod to placement + ctx = _make_bfs_ctx() + all_src: dict = {} + all_src["pkg.test_mod"] = "from pkg.mid0 import f0\n" + for i in range(n): + caller = f"f{i}" + if i < n - 1: + callee = f"f{i + 1}" + callee_mod = f"pkg.mid{i + 1}" + else: + callee = "helper" + callee_mod = "pkg.placement" + all_src[f"pkg.mid{i}"] = ( + f"from {callee_mod} import {callee}\n" f"def {caller}(): {callee}()\n" + ) + all_src["pkg.placement"] = "from external import use_fn\ndef helper(): use_fn()\n" + index = _CgIndex( + module_to_source=all_src, + module_to_package={m: "pkg" for m in all_src}, + module_to_defs={m: _cg_collect_defined_names(s) for m, s in all_src.items()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", "def test_f(): f0()\n", ctx, index, "pkg.test_mod" + ) + assert path is None + assert truncated + + +def test_resolve_forking_path_candidates_truncated_modules(): + # Re-export chain of _CG_MAX_MODULES + 1 intermediate modules; the last one + # is cut off before pkg.placement (a terminal) is ever reached. + n = _CG_MAX_MODULES + 1 + ctx = _make_bfs_ctx() + src_map: dict = {} + for i in range(n): + next_mod = f"pkg.m{i + 1}" if i < n - 1 else "pkg.placement" + src_map[f"pkg.m{i}"] = f"from {next_mod} import helper\n" + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + src_map["pkg.placement"] = placement_src + test_src = "from pkg.m0 import helper\n" + all_src = {"pkg.test_mod": test_src, **src_map} + index = _CgIndex( + module_to_source=all_src, + module_to_package={m: "pkg" for m in all_src}, + module_to_defs={m: _cg_collect_defined_names(s) for m, s in all_src.items()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" + ) + assert path is None + assert truncated + + +def test_resolve_forking_path_candidates_original_module_only(): + # modified_source still has a function using use_fn; no new sub-file uses it. + # → only terminal is (pkg.orig, func_a) → unique resolution to original path. + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source=("from external import use_fn\n" "def func_a(): use_fn()\n"), + new_files={ + "placement.py": "from external import other\ndef helper(): other()\n" + }, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + orig_src = ctx.modified_source + test_src = "from pkg.orig import func_a\n" + modules = {"pkg.test_mod": test_src, "pkg.orig": orig_src} + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", "def test_f(): func_a()\n", ctx, index, "pkg.test_mod" + ) + assert path == "pkg.orig.use_fn" + assert cands == ["pkg.orig.use_fn"] + assert not truncated + + +def test_resolve_forking_path_candidates_original_and_new_both_candidates(): + # modified_source keeps func_a (uses use_fn); placement.py moves func_b + # (also uses use_fn). Test calls both → 2 candidates → ambiguous. + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source=("from external import use_fn\n" "def func_a(): use_fn()\n"), + new_files={ + "placement.py": ( + "from external import use_fn\n" "def func_b(): use_fn()\n" + ), + }, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + orig_src = ctx.modified_source + placement_src = ctx.new_files["placement.py"] + test_src = "from pkg.orig import func_a\nfrom pkg.placement import func_b\n" + modules = { + "pkg.test_mod": test_src, + "pkg.orig": orig_src, + "pkg.placement": placement_src, + } + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", + "def test_f(): func_a(); func_b()\n", + ctx, + index, + "pkg.test_mod", + ) + assert path is None # ambiguous + assert sorted(cands) == ["pkg.orig.use_fn", "pkg.placement.use_fn"] + assert not truncated + + +def test_expand_module_terminals_no_direct(): + # No direct terminal in this module → nothing added. + terminal: dict = {} + _expand_module_terminals( + "def a(): b()\ndef b(): pass\n", "pkg.mod", "use_fn", terminal + ) + assert terminal == {} + + +def test_expand_module_terminals_direct_only(): + # A direct terminal is seeded before calling; only transitive callers are added. + terminal: dict = {("pkg.mod", "b"): "pkg.mod.use_fn"} + _expand_module_terminals( + "def a(): b()\ndef b(): use_fn()\n", "pkg.mod", "use_fn", terminal + ) + # a calls b (direct terminal) → a becomes transitive terminal. + assert terminal[("pkg.mod", "a")] == "pkg.mod.use_fn" + # Original direct entry unchanged. + assert terminal[("pkg.mod", "b")] == "pkg.mod.use_fn" + + +def test_expand_module_terminals_multi_level(): + # c → b → a (direct); all three end up in terminal. + terminal: dict = {("pkg.mod", "a"): "pkg.mod.use_fn"} + src = "def a(): use_fn()\ndef b(): a()\ndef c(): b()\n" + _expand_module_terminals(src, "pkg.mod", "use_fn", terminal) + assert ("pkg.mod", "b") in terminal + assert ("pkg.mod", "c") in terminal + + +def test_expand_module_terminals_syntax_error(): + # Unparseable source → silently returns without modifying terminal. + terminal: dict = {("pkg.mod", "a"): "pkg.mod.use_fn"} + _expand_module_terminals("def (broken\n", "pkg.mod", "use_fn", terminal) + # Only the original entry remains. + assert list(terminal.keys()) == [("pkg.mod", "a")] + + +def test_expand_module_terminals_unrelated_module(): + # Direct terminal is in a different module → nothing added for pkg.other. + terminal: dict = {("pkg.mod", "a"): "pkg.mod.use_fn"} + _expand_module_terminals("def b(): a()\n", "pkg.other", "use_fn", terminal) + # b is in pkg.other which has no direct terminals → not added. + assert ("pkg.other", "b") not in terminal + + +def test_resolve_forking_path_candidates_intra_module_chain(): + # BFS follows locally-defined calls within a non-terminal intermediate module. + # Chain: test_mod → pkg.service.public_func + # (local) ↓ + # pkg.service._local_helper + # (import) ↓ + # pkg.placement.use_target ← terminal (calls use_fn) + # + # pkg.service is neither orig nor a new sub-file, so _expand_module_terminals + # never seeds it. The elif branch in the BFS must queue _local_helper from + # public_func's body so we eventually reach the terminal in pkg.placement. + placement_src = "from external import use_fn\ndef use_target(): use_fn()\n" + service_src = ( + "from pkg.placement import use_target\n" + "def _local_helper(): use_target()\n" + "def public_func(): _local_helper()\n" + ) + ctx2 = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="from .placement import use_target\n", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={"use_target": "placement.py"}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = "from pkg.service import public_func\n" + modules = { + "pkg.test_mod": test_src, + "pkg.service": service_src, + "pkg.placement": placement_src, + } + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", + "def test_f(): public_func()\n", + ctx2, + index, + "pkg.test_mod", + ) + assert path == "pkg.placement.use_fn" + assert cands == ["pkg.placement.use_fn"] + assert not truncated + + +def test_resolve_forking_path_candidates_intra_module_local_already_visited(): + # The elif branch fires but the visited guard suppresses re-queuing. + # A recursive function calls itself: when processing its body calls, itself + # is already in visited → (module, called_name) in visited → branch skipped. + placement_src = "from external import use_fn\ndef use_target(): use_fn()\n" + service_src = ( + "from pkg.placement import use_target\n" + # recursive_func calls use_target (imported) AND itself (local, recursive) + "def recursive_func(n): use_target() if n <= 0 else recursive_func(n-1)\n" + ) + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="from .placement import use_target\n", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={"use_target": "placement.py"}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = "from pkg.service import recursive_func\n" + modules = { + "pkg.test_mod": test_src, + "pkg.service": service_src, + "pkg.placement": placement_src, + } + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", + "def test_f(): recursive_func(5)\n", + ctx, + index, + "pkg.test_mod", + ) + # recursive_func's body calls reach use_target (terminal in pkg.placement). + assert path == "pkg.placement.use_fn" + assert not truncated + + +def test_resolve_forking_path_candidates_new_module_intra_chain(): + # BFS follows intra-module calls within a new submodule to reach a terminal. + # Chain: test_mod → pkg.placement.wrapper (new-module, not terminal) + # (local) ↓ + # pkg.placement._inner ← terminal (calls use_fn directly) + # + # Before the fix, the BFS hit pkg.placement in new_module_set and stopped at + # wrapper without following _inner — no candidate was found. After the fix, + # it follows the local call to _inner and discovers pkg.placement.use_fn. + placement_src = ( + "from external import use_fn\n" + "def _inner(): use_fn()\n" + "def wrapper(): _inner()\n" + ) + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={"wrapper": "placement.py", "_inner": "placement.py"}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = "from pkg.placement import wrapper\n" + modules = { + "pkg.test_mod": test_src, + "pkg.placement": placement_src, + } + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", + "def test_f(): wrapper()\n", + ctx, + index, + "pkg.test_mod", + ) + assert path == "pkg.placement.use_fn" + assert cands == ["pkg.placement.use_fn"] + assert not truncated + + +def test_resolve_forking_path_candidates_new_module_intra_chain_cycle(): + # Intra-module traversal inside a new submodule respects the visited guard: + # a mutually recursive pair (a calls b, b calls a) does not loop. + placement_src = ( + "from external import use_fn\n" + "def _inner(): use_fn()\n" + "def a(): b()\n" + "def b(): a(); _inner()\n" # b is terminal (uses use_fn via _inner) + ) + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={"a": "placement.py"}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = "from pkg.placement import a\n" + modules = { + "pkg.test_mod": test_src, + "pkg.placement": placement_src, + } + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", + "def test_f(): a()\n", + ctx, + index, + "pkg.test_mod", + ) + assert path == "pkg.placement.use_fn" + assert not truncated + + +def test_resolve_forking_path_candidates_new_module_cross_module_terminal(): + # BFS follows cross-module calls FROM a terminal function inside a new submodule. + # Scenario: + # pkg.main: _run_step() calls use_fn() directly (terminal) + # orchestrate() calls _run_step() [local] + do_step() [pkg.steps] + # _expand_module_terminals makes orchestrate terminal for pkg.main.use_fn # noqa: E501 + # pkg.steps: do_step() calls use_fn() (terminal for pkg.steps.use_fn) + # + # When BFS hits (pkg.main, orchestrate) — which IS in terminal — it should + # record pkg.main.use_fn AND then follow the cross-module call to + # (pkg.steps, do_step), discovering pkg.steps.use_fn as a second candidate. + # Line 1472 in the BFS is covered only by this cross-module append. + main_src = ( + "from external import use_fn\n" + "from pkg.steps import do_step\n" + "def _run_step(): use_fn()\n" + "def orchestrate(): _run_step(); do_step()\n" + ) + steps_src = "from external import use_fn\n" "def do_step(): use_fn()\n" + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"main.py": main_src, "steps.py": steps_src}, + new_module_paths={"main.py": "pkg.main", "steps.py": "pkg.steps"}, + entity_to_target={ + "_run_step": "main.py", + "orchestrate": "main.py", + "do_step": "steps.py", + }, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = "from pkg.main import orchestrate\n" + modules = { + "pkg.test_mod": test_src, + "pkg.main": main_src, + "pkg.steps": steps_src, + } + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", + "def test_f(): orchestrate()\n", + ctx, + index, + "pkg.test_mod", + ) + # Both submodules use use_fn — two candidates, no single resolved path. + assert path is None + assert sorted(cands) == ["pkg.main.use_fn", "pkg.steps.use_fn"] + assert not truncated + + +def test_resolve_forking_path_candidates_import_alias_direct(): + # Test uses ``import pkg.placement as pl; pl.helper()`` to call the terminal. + # The BFS must follow ``pl.helper`` by resolving alias ``pl`` → ``pkg.placement``. + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="from .placement import helper\n", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={"helper": "placement.py"}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = "import pkg.placement as pl\n" + # Import map: "pl" → ("pkg.placement", "pkg.placement"); call "pl.helper()" + # → _cg_collect_called_names emits "pl.helper"; BFS resolves alias pl → + # module pkg.placement, queues (pkg.placement, "helper") → terminal hit. + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src}, + module_to_package={"pkg.test_mod": "pkg"}, + module_to_defs={"pkg.test_mod": set()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", + "def test_f(): pl.helper()\n", + ctx, + index, + "pkg.test_mod", + ) + assert path == "pkg.placement.use_fn" + assert cands == ["pkg.placement.use_fn"] + assert not truncated + + +def test_resolve_forking_path_candidates_body_call_via_alias(): + # An intermediate function uses ``mod.helper()`` (module alias) to reach + # the terminal. The BFS body-call step must follow the alias. + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + service_src = "import pkg.placement as pl\n" "def public_func(): pl.helper()\n" + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="from .placement import helper\n", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={"helper": "placement.py"}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = "from pkg.service import public_func\n" + modules = { + "pkg.test_mod": test_src, + "pkg.service": service_src, + "pkg.placement": placement_src, + } + index = _CgIndex( + module_to_source=modules, + module_to_package={m: "pkg" for m in modules}, + module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, + file_to_module={}, + ) + path, cands, truncated, _static = _resolve_forking_path_candidates( + "use_fn", + "def test_f(): public_func()\n", + ctx, + index, + "pkg.test_mod", + ) + assert path == "pkg.placement.use_fn" + assert cands == ["pkg.placement.use_fn"] + assert not truncated diff --git a/tests/patch_rewriter/test_callgraph_update.py b/tests/patch_rewriter/test_callgraph_update.py new file mode 100644 index 0000000..3b88af1 --- /dev/null +++ b/tests/patch_rewriter/test_callgraph_update.py @@ -0,0 +1,874 @@ +from __future__ import annotations +from crispen.patch_rewriter import ( + RewriteAccumulator, + _CgIndex, + _FLContext, + _callgraph_update_file, + _candidates_check, + _cg_build_index, + _cg_collect_defined_names, + _rewrite_candidates_check, + apply_patch_callgraph, +) +from .helpers import _make_cuf_contexts, _make_cuf_index + + +def test_callgraph_update_file_no_functions(): + src = "x = 1\n" + result, changed, _unresolved = _callgraph_update_file( + src, {"pkg.orig.use_fn"}, _make_cuf_contexts() + ) + assert not changed + assert result == src + + +def test_callgraph_update_file_index_none(tmp_path): + # index=None → BFS skipped → no resolution even if test calls helper. + test_src = ( + "from pkg.placement import helper\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(mock_use_fn):\n" + " helper()\n" + ) + scan = str(tmp_path / "test_foo.py") + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + _make_cuf_contexts(), + scan_file=scan, + index=None, + ) + assert not changed + + +def test_callgraph_update_file_string_literal_resolved(tmp_path): + test_src = ( + "from pkg.placement import helper\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(mock_use_fn):\n" + " helper()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + _make_cuf_contexts(), + scan_file=scan, + index=index, + ) + assert changed + assert '@patch("pkg.placement.use_fn")' in result + + +def test_callgraph_update_file_acc_cg_resolved(tmp_path): + # _acc.cg_resolved incremented for each resolved path. + test_src = ( + "from pkg.placement import helper\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(mock_use_fn):\n" + " helper()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + acc = RewriteAccumulator() + _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + _make_cuf_contexts(), + scan_file=scan, + index=index, + _acc=acc, + ) + assert acc.cg_resolved == 1 + + +def test_callgraph_update_file_no_resolution(tmp_path): + # Test calls 'unrelated' — not imported → BFS queue empty → no resolution. + # Static fallback has 2 candidates (placement + conflict) → unresolved saved. + test_src = ( + '@patch("pkg.orig.use_fn")\n' "def test_f(mock_use_fn):\n" " unrelated()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + _make_cuf_contexts(), + scan_file=scan, + index=index, + ) + assert not changed + cands = unresolved.get("test_f", {}).get("pkg.orig.use_fn", []) + assert sorted(cands) == ["pkg.conflict.use_fn", "pkg.placement.use_fn"] + + +def test_callgraph_update_file_zero_cands_single_static_auto_resolve(tmp_path): + # BFS finds 0 candidates but static terminal has exactly 1 → auto-resolve. + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = ( + '@patch("pkg.orig.use_fn")\n' "def test_f(mock_use_fn):\n" " unrelated()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + [ctx], + scan_file=scan, + index=index, + ) + assert changed + assert '@patch("pkg.placement.use_fn")' in result + assert "test_f" not in unresolved + + +def test_callgraph_update_file_zero_cands_single_static_clears_unresolved(tmp_path): + # ctx_ambig: BFS finds 2 candidates (saves to unresolved). + # ctx_uniq_static: BFS finds 0, static has 1 → auto-resolves AND clears the + # previously saved unresolved entry (exercises the delete-entry branch). + test_src = ( + "from pkg.placement import helper\n" + "from pkg.conflict import resolve\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(m):\n" + " helper()\n" + " resolve()\n" + ) + ctx_ambig = _make_cuf_contexts()[0] # placement + conflict → 2 BFS candidates + ctx_uniq_static = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"singleton.py": "from external import use_fn\ndef fn(): use_fn()\n"}, + new_module_paths={"singleton.py": "pkg.singleton"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + scan = str(tmp_path / "test_foo.py") + # Index only knows pkg.test_mod; pkg.placement/conflict have no source so + # ctx_uniq_static's BFS reaches 0 candidates while static_cands = 1. + index = _make_cuf_index(scan, test_src) + result, changed, unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + [ctx_ambig, ctx_uniq_static], + scan_file=scan, + index=index, + ) + assert changed + assert '@patch("pkg.singleton.use_fn")' in result + assert "test_f" not in unresolved # static single-cand cleared the entry + + +def test_callgraph_update_file_const_ref_unanimous(tmp_path): + test_src = ( + "from pkg.placement import helper\n" + '_PATCH_USE = "pkg.orig.use_fn"\n' + "@patch(_PATCH_USE)\n" + "def test_a(mock_use_fn):\n" + " helper()\n" + "\n" + "@patch(_PATCH_USE)\n" + "def test_b(mock_use_fn):\n" + " helper()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + _make_cuf_contexts(), + scan_file=scan, + index=index, + ) + assert changed + assert '_PATCH_USE = "pkg.placement.use_fn"' in result + assert "@patch(_PATCH_USE)" in result + + +def test_callgraph_update_file_const_ref_conflicting(tmp_path): + # test_a: helper() → placement; test_b: resolve() → conflict → conflicting. + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", + "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", + }, + new_module_paths={ + "placement.py": "pkg.placement", + "conflict.py": "pkg.conflict", + }, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = ( + "from pkg.placement import helper\n" + "from pkg.conflict import resolve\n" + '_PATCH_USE = "pkg.orig.use_fn"\n' + "@patch(_PATCH_USE)\n" + "def test_a(mock_use_fn):\n" + " helper()\n" + "\n" + "@patch(_PATCH_USE)\n" + "def test_b(mock_use_fn):\n" + " resolve()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, {"pkg.orig.use_fn"}, [ctx], scan_file=scan, index=index + ) + assert changed + assert '_PATCH_USE = "pkg.orig.use_fn"' in result # const def unchanged + assert '@patch("pkg.placement.use_fn")' in result # test_a inlined + assert '@patch("pkg.conflict.use_fn")' in result # test_b inlined + + +def test_callgraph_update_file_non_forking_path_skipped(tmp_path): + test_src = ( + "from pkg.placement import helper\n" + '@patch("pkg.stable.some_func")\n' + '@patch("pkg.orig.use_fn")\n' + "def test_f(mock_use_fn, mock_some):\n" + " helper()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + _make_cuf_contexts(), + scan_file=scan, + index=index, + ) + assert changed + assert '@patch("pkg.placement.use_fn")' in result + assert '@patch("pkg.stable.some_func")' in result # unchanged + + +def test_callgraph_update_file_multi_context_second_matches(tmp_path): + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + ctx_a = _FLContext( + filepath="/proj/pkg/other.py", + old_module="pkg.other", + original_source="from external import other_fn\n", + modified_source="", + new_files={}, + new_module_paths={}, + entity_to_target={}, + forking_old_paths={"pkg.other.other_fn"}, + ) + ctx_b = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = ( + "from pkg.placement import helper\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(mock_use_fn):\n" + " helper()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn", "pkg.other.other_fn"}, + [ctx_a, ctx_b], + scan_file=scan, + index=index, + ) + assert changed + assert '@patch("pkg.placement.use_fn")' in result + + +def test_callgraph_update_file_const_ref_no_resolution_passthrough(tmp_path): + test_src = ( + '_PATCH_USE = "pkg.orig.use_fn"\n' + "@patch(_PATCH_USE)\n" + "def test_f(mock_use_fn):\n" + " unrelated()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + _make_cuf_contexts(), + scan_file=scan, + index=index, + ) + assert not changed + assert '_PATCH_USE = "pkg.orig.use_fn"' in result + + +def test_callgraph_update_file_const_ref_passthrough_single_proposal_updates_const( + tmp_path, +): + # test_a: BFS fails (calls unrelated()) → passthrough (if not resolved → continue). + # test_b: BFS → placement → single proposal for _PATCH_USE. + # Old: passthrough + single proposal → conflicting → inline test_b. + # New: single proposal (passthrough no longer blocks) → const def updated, no + # inline. + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", + }, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = ( + "from pkg.placement import helper\n" + '_PATCH_USE = "pkg.orig.use_fn"\n' + "@patch(_PATCH_USE)\n" + "def test_a(m):\n" + " unrelated()\n" + "\n" + "@patch(_PATCH_USE)\n" + "def test_b(m):\n" + " helper()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, {"pkg.orig.use_fn"}, [ctx], scan_file=scan, index=index + ) + assert changed + # Const definition updated (single proposal, passthrough no longer blocks). + assert '_PATCH_USE = "pkg.placement.use_fn"' in result + # Decorators stay as const refs — no per-function inlining. + assert "@patch(_PATCH_USE)" in result + assert '@patch("pkg.placement.use_fn")' not in result + + +def test_callgraph_update_file_const_ref_partial_resolution(tmp_path): + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn, other_fn\n", + modified_source="", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n" + }, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn", "pkg.orig.other_fn"}, + ) + test_src = ( + "from pkg.placement import helper\n" + '_PATCH_USE = "pkg.orig.use_fn"\n' + '_PATCH_OTHER = "pkg.orig.other_fn"\n' + "@patch(_PATCH_OTHER)\n" + "@patch(_PATCH_USE)\n" + "def test_f(mock_use, mock_other):\n" + " helper()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn", "pkg.orig.other_fn"}, + [ctx], + scan_file=scan, + index=index, + ) + assert changed + assert '_PATCH_USE = "pkg.placement.use_fn"' in result + assert '_PATCH_OTHER = "pkg.orig.other_fn"' in result + + +def test_callgraph_update_file_inline_no_inline_subs_continue(tmp_path): + # test_a: string literal (no const_refs → inline_subs empty → continue) + # test_b: const ref → placement; test_c: const ref → conflict → conflicting + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", + "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", + }, + new_module_paths={ + "placement.py": "pkg.placement", + "conflict.py": "pkg.conflict", + }, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = ( + "from pkg.placement import helper\n" + "from pkg.conflict import resolve\n" + '@patch("pkg.orig.use_fn")\n' + "def test_a(m):\n" + " helper()\n" + "\n" + '_PATCH_USE = "pkg.orig.use_fn"\n' + "@patch(_PATCH_USE)\n" + "def test_b(m):\n" + " helper()\n" + "\n" + "@patch(_PATCH_USE)\n" + "def test_c(m):\n" + " resolve()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, {"pkg.orig.use_fn"}, [ctx], scan_file=scan, index=index + ) + assert changed + assert '@patch("pkg.placement.use_fn")' in result # test_a updated + assert '@patch("pkg.conflict.use_fn")' in result # test_c inlined + + +def test_callgraph_update_file_inline_ref_from_different_file(tmp_path): + # Const ref from constants.py (≠ scan_file) → inline_subs empty → no change. + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", + "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", + }, + new_module_paths={ + "placement.py": "pkg.placement", + "conflict.py": "pkg.conflict", + }, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + constants_file = tmp_path / "constants.py" + constants_file.write_text('_PATCH_USE = "pkg.orig.use_fn"\n', encoding="utf-8") + test_src = ( + "from constants import _PATCH_USE\n" + "from pkg.placement import helper\n" + "from pkg.conflict import resolve\n" + "@patch(_PATCH_USE)\n" + "def test_b(m):\n" + " helper()\n" + "\n" + "@patch(_PATCH_USE)\n" + "def test_c(m):\n" + " resolve()\n" + ) + scan_file = tmp_path / "test_cases.py" + scan_file.write_text(test_src, encoding="utf-8") + scan = str(scan_file) + # Build index from disk so file_to_module is populated for test_cases.py + index = _cg_build_index(str(tmp_path), {}, [ctx]) + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + [ctx], + scan_file=scan, + repo_root=str(tmp_path), + index=index, + ) + assert not changed + + +def test_callgraph_update_file_inline_new_val_same_as_old(tmp_path): + # placement.py → "pkg.orig" (same as old_module); test_b→helper→same val; skipped. + # test_c → resolve → "pkg.conflict" → different val → inlined → changed. + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", + "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", + }, + new_module_paths={ + "placement.py": "pkg.orig", # same as old_module → new_val == old_val + "conflict.py": "pkg.conflict", + }, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = ( + "from pkg.orig import helper\n" + "from pkg.conflict import resolve\n" + '_PATCH_USE = "pkg.orig.use_fn"\n' + "@patch(_PATCH_USE)\n" + "def test_b(m):\n" + " helper()\n" + "\n" + "@patch(_PATCH_USE)\n" + "def test_c(m):\n" + " resolve()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, {"pkg.orig.use_fn"}, [ctx], scan_file=scan, index=index + ) + assert changed # test_c inlined to pkg.conflict.use_fn + + +def test_callgraph_update_file_inline_existing_splice_updated(tmp_path): + # test_a: use_fn → func_splice; other_fn const ref conflicting → inline. + # Inline finds existing splice and updates it. test_b: const → new splice. + placement_src = ( + "from external import use_fn, other_fn\n" "def helper(): use_fn(); other_fn()\n" + ) + conflict2_src = "from external import other_fn\ndef resolve2(): other_fn()\n" + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn, other_fn\n", + modified_source="", + new_files={"placement.py": placement_src, "conflict2.py": conflict2_src}, + new_module_paths={ + "placement.py": "pkg.placement", + "conflict2.py": "pkg.conflict2", + }, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn", "pkg.orig.other_fn"}, + ) + test_src = ( + "from pkg.placement import helper\n" + "from pkg.conflict2 import resolve2\n" + '_PATCH_OTHER = "pkg.orig.other_fn"\n' + '@patch("pkg.orig.use_fn")\n' + "@patch(_PATCH_OTHER)\n" + "def test_a(m_other, m_use):\n" + " helper()\n" + "\n" + "@patch(_PATCH_OTHER)\n" + "def test_b(m_other):\n" + " resolve2()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn", "pkg.orig.other_fn"}, + [ctx], + scan_file=scan, + index=index, + ) + assert changed + assert '@patch("pkg.placement.use_fn")' in result + assert '@patch("pkg.placement.other_fn")' in result + assert '@patch("pkg.conflict2.other_fn")' in result + + +def test_callgraph_update_file_verbose(tmp_path, capsys): + test_src = ( + "from pkg.placement import helper\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(mock_use_fn):\n" + " helper()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + _make_cuf_contexts(), + scan_file=scan, + index=index, + verbose=True, + ) + captured = capsys.readouterr() + assert "patch_callgraph" in captured.err + + +def test_callgraph_update_file_truncated_warns(tmp_path, capsys): + # Depth limit of 0 forces truncation for indirect calls; warning must be printed. + # Test calls an intermediate function (not a terminal); with max_depth=0 the + # first BFS hop immediately hits the limit before reaching the terminal. + test_src = ( + "from pkg.middle import middle_fn\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(mock_use_fn):\n" + " middle_fn()\n" + ) + scan = str(tmp_path / "test_foo.py") + scan_abs = str((tmp_path / "test_foo.py").resolve()) + # middle_fn → helper (terminal in pkg.placement), but BFS cuts off before that. + middle_src = "from pkg.placement import helper\ndef middle_fn(): helper()\n" + index = _CgIndex( + module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, + module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, + module_to_defs={ + "pkg.test_mod": set(), + "pkg.middle": _cg_collect_defined_names(middle_src), + }, + file_to_module={scan_abs: "pkg.test_mod"}, + ) + result, changed, _unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + _make_cuf_contexts(), + scan_file=scan, + index=index, + max_depth=0, + ) + assert not changed + captured = capsys.readouterr() + assert "traversal limit reached" in captured.err + assert "pkg.orig.use_fn" in captured.err + + +def test_candidates_check_no_candidates(): + # No candidates for any path → None. + assert _candidates_check({"pkg.orig.A": "pkg.sub.A"}, ["pkg.orig.A"], {}) is None + + +def test_candidates_check_rename_valid(): + # Rename is in candidates → None. + cands = {"pkg.orig.A": ["pkg.placement.A", "pkg.helpers.A"]} + assert ( + _candidates_check({"pkg.orig.A": "pkg.placement.A"}, ["pkg.orig.A"], cands) + is None + ) + + +def test_candidates_check_rename_invalid(): + # Rename proposes a path not in candidates → error message. + cands = {"pkg.orig.A": ["pkg.placement.A"]} + result = _candidates_check({"pkg.orig.A": "pkg.wrong.A"}, ["pkg.orig.A"], cands) + assert result is not None + assert "pkg.wrong.A" in result + assert "pkg.placement.A" in result + + +def test_candidates_check_no_change_with_candidates(): + # No rename proposed for a path that has candidates → error message. + cands = {"pkg.orig.A": ["pkg.placement.A"]} + result = _candidates_check({}, ["pkg.orig.A"], cands) + assert result is not None + assert "pkg.orig.A" in result + assert "pkg.placement.A" in result + + +def test_candidates_check_path_not_in_candidates(): + # Another path has no candidates → passes; only paths with candidates are checked. + cands = {"pkg.orig.A": ["pkg.placement.A"]} + # pkg.orig.B has no candidates; even though no rename proposed → None + assert _candidates_check({}, ["pkg.orig.B"], cands) is None + + +def test_candidates_check_no_change_when_old_in_candidates(): + # No rename proposed but old path is itself one of the candidates (e.g. the entity + # is still accessible at the original module via __init__.py re-export) → None. + cands = {"pkg.orig.A": ["pkg.orig.A", "pkg.resolver.A"]} + assert _candidates_check({}, ["pkg.orig.A"], cands) is None + + +def test_callgraph_update_file_multiple_candidates_saved(tmp_path): + # Both placement and conflict are reachable → 2 candidates → saved. + test_src = ( + "from pkg.placement import helper\n" + "from pkg.conflict import resolve\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(m):\n" + " helper()\n" + " resolve()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + _make_cuf_contexts(), + scan_file=scan, + index=index, + ) + assert not changed # ambiguous → no update + assert "test_f" in unresolved + assert "pkg.orig.use_fn" in unresolved["test_f"] + cands = unresolved["test_f"]["pkg.orig.use_fn"] + assert sorted(cands) == ["pkg.conflict.use_fn", "pkg.placement.use_fn"] + + +def test_callgraph_update_file_resolved_clears_candidates(tmp_path): + # Single ctx with unique resolution → no candidates saved. + placement_src = "from external import use_fn\ndef helper(): use_fn()\n" + ctx = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="", + new_files={"placement.py": placement_src}, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + test_src = ( + "from pkg.placement import helper\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(m):\n" + " helper()\n" + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + [ctx], + scan_file=scan, + index=index, + ) + assert changed + assert "test_f" not in unresolved # unique resolution → no candidates saved + + +def test_callgraph_update_file_resolved_clears_function_entry(tmp_path): + # ctx_ambig gives 2 candidates (saves to unresolved); ctx_uniq resolves uniquely → + # unresolved entry for the function is deleted (line 2695). + test_src = ( + "from pkg.placement import helper\n" + "from pkg.conflict import resolve\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(m):\n" + " helper()\n" + " resolve()\n" + ) + ctx_ambig = _make_cuf_contexts()[0] # both placement and conflict → 2 candidates + ctx_uniq = _FLContext( + filepath="/proj/pkg/orig.py", + old_module="pkg.orig", + original_source="from external import use_fn\n", + modified_source="from .placement import helper\n", + new_files={ + "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", + }, + new_module_paths={"placement.py": "pkg.placement"}, + entity_to_target={}, + forking_old_paths={"pkg.orig.use_fn"}, + ) + scan = str(tmp_path / "test_foo.py") + index = _make_cuf_index(scan, test_src) + result, changed, unresolved = _callgraph_update_file( + test_src, + {"pkg.orig.use_fn"}, + [ctx_ambig, ctx_uniq], + scan_file=scan, + index=index, + ) + assert changed + assert "test_f" not in unresolved # ctx_uniq resolved → entry deleted + + +def test_apply_patch_callgraph_candidates_out_per_file(tmp_path): + # Multiple candidates → saved in candidates_out for per_file entry. + test_src = ( + "from pkg.placement import helper\n" + "from pkg.conflict import resolve\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(m):\n" + " helper()\n" + " resolve()\n" + ) + test_file = tmp_path / "test_orig.py" + test_file.write_text(test_src, encoding="utf-8") + per_file = {str(test_file): {"source": test_src, "msgs": []}} + candidates_out: dict = {} + list( + apply_patch_callgraph( + _make_cuf_contexts(), per_file, str(tmp_path), candidates_out=candidates_out + ) + ) + abs_fp = str(test_file.resolve()) + assert abs_fp in candidates_out + assert "test_f" in candidates_out[abs_fp] + + +def test_apply_patch_callgraph_candidates_out_disk_file(tmp_path): + # Multiple candidates → saved in candidates_out for disk file. + test_src = ( + "from pkg.placement import helper\n" + "from pkg.conflict import resolve\n" + '@patch("pkg.orig.use_fn")\n' + "def test_f(m):\n" + " helper()\n" + " resolve()\n" + ) + test_file = tmp_path / "test_orig.py" + test_file.write_text(test_src, encoding="utf-8") + candidates_out: dict = {} + list( + apply_patch_callgraph( + _make_cuf_contexts(), {}, str(tmp_path), candidates_out=candidates_out + ) + ) + abs_fp = str(test_file.resolve()) + assert abs_fp in candidates_out + assert "test_f" in candidates_out[abs_fp] + + +def test_rewrite_candidates_check_no_candidates(): + # No candidates for any path → None. + text = '@patch("pkg.mod.A")\ndef test_f(m): pass\n' + assert _rewrite_candidates_check(["pkg.mod.A"], text, {}) is None + + +def test_rewrite_candidates_check_valid_rename(): + # Old path absent, one candidate present → None. + text = '@patch("pkg.placement.A")\ndef test_f(m): pass\n' + cands = {"pkg.mod.A": ["pkg.placement.A", "pkg.other.A"]} + assert _rewrite_candidates_check(["pkg.mod.A"], text, cands) is None + + +def test_rewrite_candidates_check_old_still_present(): + # Old path still present even though candidates exist → error. + text = '@patch("pkg.mod.A")\ndef test_f(m): pass\n' + cands = {"pkg.mod.A": ["pkg.placement.A"]} + result = _rewrite_candidates_check(["pkg.mod.A"], text, cands) + assert result is not None + assert "pkg.mod.A" in result + assert "pkg.placement.A" in result + + +def test_rewrite_candidates_check_renamed_to_unknown(): + # Old path absent, no known candidate appears — could be a wrong rename or a + # dead-code removal. Let the LLM verify step decide; no error returned here. + text = '@patch("pkg.wrong.A")\ndef test_f(m): pass\n' + cands = {"pkg.mod.A": ["pkg.placement.A", "pkg.other.A"]} + assert _rewrite_candidates_check(["pkg.mod.A"], text, cands) is None + + +def test_rewrite_candidates_check_deleted_patch(): + # Old path absent and decorator was removed entirely → dead-code removal is + # allowed; let the LLM verify step confirm correctness. + text = "def test_f(): pass\n" + cands = {"pkg.mod.A": ["pkg.placement.A", "pkg.other.A"]} + assert _rewrite_candidates_check(["pkg.mod.A"], text, cands) is None + + +def test_rewrite_candidates_check_path_without_candidates_ignored(): + # A path with no candidates in the dict → skip it. + text = '@patch("pkg.mod.B")\ndef test_f(m): pass\n' + cands = {"pkg.mod.A": ["pkg.placement.A"]} # A has candidates, B does not + assert _rewrite_candidates_check(["pkg.mod.B"], text, cands) is None diff --git a/tests/patch_rewriter/test_const_refs.py b/tests/patch_rewriter/test_const_refs.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/tests/patch_rewriter/test_const_refs.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/patch_rewriter/test_context_builders.py b/tests/patch_rewriter/test_context_builders.py new file mode 100644 index 0000000..eca5104 --- /dev/null +++ b/tests/patch_rewriter/test_context_builders.py @@ -0,0 +1,864 @@ +from __future__ import annotations +from crispen.patch_rewriter import ( + _CG_CANDIDATES_LLM_THRESHOLD, + _FLContext, + _build_classify_prompt, + _build_context_message, + _build_func_verify_prompt, + _build_no_change_verify_prompt, + _build_rewrite_func_prompt, + _build_rewrite_verify_prompt, + _extract_migration_reminder, + _extract_patch_lookup, + _extract_still_imported_names, + _get_external_import_names, + _import_header, + _name_reference_map, + _splice_function, +) +from .helpers import _ctx_msg, _make_fl_ctx, _make_fl_ctx_simple + + +def test_build_context_no_diff(): + # Diff is no longer included — only imports header and entity migration. + ctx = _make_fl_ctx() + msg = _build_context_message([ctx]) + assert "```diff" not in msg + + +def test_build_context_new_file_imports_and_refs(): + # New-file section shows imports header and name-reference map; no bodies. + src = "import os\n\ndef my_func():\n os.path.join('a', 'b')\n" + ctx = _make_fl_ctx(new_files={"sub_a.py": src, "sub_b.py": "class B: pass\n"}) + msg = _build_context_message([ctx]) + assert "**Imports:**" in msg + assert "import os" in msg + assert "**Name references**" in msg + assert "`os`: `my_func`" in msg + assert "def my_func" not in msg # body not included + + +def test_build_context_entity_migration_present(): + ctx = _make_fl_ctx() + msg = _build_context_message([ctx]) + assert "sub_a.py" in msg + assert "pkg.sub_a" in msg + + +def test_build_context_empty_new_files_and_entities(): + # Covers the zero-iteration branches of the two for-loops. + ctx = _make_fl_ctx(new_files={}, new_module_paths={}, entity_to_target={}) + msg = _build_context_message([ctx]) + assert "Split module" in msg + assert "Entity migration" in msg + + +def test_build_context_multiple_contexts(): + ctx1 = _make_fl_ctx(old_module="pkg.big", filepath="/p/pkg/big.py") + ctx2 = _make_fl_ctx(old_module="pkg.large", filepath="/p/pkg/large.py") + msg = _build_context_message([ctx1, ctx2]) + assert "pkg.big" in msg + assert "pkg.large" in msg + + +def test_import_header_stops_before_def(): + src = "import os\nfrom x import y\n\ndef foo():\n pass\n" + assert _import_header(src) == "import os\nfrom x import y\n" + + +def test_import_header_stops_before_class(): + src = "import os\n\nclass Foo:\n pass\n" + assert _import_header(src) == "import os\n" + + +def test_import_header_stops_before_async_def(): + src = "import os\nasync def foo(): pass\n" + assert _import_header(src) == "import os\n" + + +def test_import_header_no_defs_returns_all(): + src = "import os\nfrom x import y\n" + assert _import_header(src) == "import os\nfrom x import y\n" + + +def test_import_header_empty_source(): + assert _import_header("") == "" + + +def test_import_header_strips_trailing_blanks(): + src = "import os\n\n\ndef foo(): pass\n" + assert _import_header(src) == "import os\n" + + +def test_name_reference_map_basic(): + src = ( + "import os\n" + "from x import Foo\n" + "\n" + "def alpha():\n" + " os.getcwd()\n" + " Foo()\n" + "\n" + "def beta():\n" + " os.path.join('a', 'b')\n" + ) + refs = _name_reference_map(src) + assert refs["os"] == ["alpha", "beta"] + assert refs["Foo"] == ["alpha"] + + +def test_name_reference_map_alias(): + src = "import libcst as cst\n\ndef run():\n cst.parse_module('x')\n" + refs = _name_reference_map(src) + assert refs["cst"] == ["run"] + + +def test_name_reference_map_unused_import(): + # Imported but never referenced in a function body → absent from map. + src = "import os\n\ndef alpha():\n pass\n" + refs = _name_reference_map(src) + assert "os" not in refs + + +def test_name_reference_map_no_imports(): + src = "def alpha():\n x = 1\n" + assert _name_reference_map(src) == {} + + +def test_name_reference_map_star_import_ignored(): + # ``from x import *`` should not add anything (alias.name == "*" branch). + src = "from x import *\n\ndef alpha():\n foo()\n" + refs = _name_reference_map(src) + assert refs == {} + + +def test_name_reference_map_syntax_error(): + assert _name_reference_map("def (broken:") == {} + + +def test_name_reference_map_class(): + src = ( + "from x import Dep\n" + "\n" + "class MyClass:\n" + " def method(self):\n" + " return Dep()\n" + ) + refs = _name_reference_map(src) + assert refs["Dep"] == ["MyClass"] + + +def test_splice_function_basic(): + source = "line1\nline2\nline3\nline4\n" + result = _splice_function(source, 2, 3, "new2\nnew3\n") + assert result == "line1\nnew2\nnew3\nline4\n" + + +def test_splice_function_single_line(): + source = "line1\nline2\nline3\n" + result = _splice_function(source, 2, 2, "replacement\n") + assert result == "line1\nreplacement\nline3\n" + + +def test_splice_function_size_change(): + # Replace 1 line with 3 lines. + source = "a\nb\nc\n" + result = _splice_function(source, 2, 2, "x\ny\nz\n") + assert result == "a\nx\ny\nz\nc\n" + + +def test_splice_function_no_trailing_newline(): + # new_func_text without trailing newline gets one added. + source = "a\nb\nc\n" + result = _splice_function(source, 2, 2, "replacement") + assert result == "a\nreplacement\nc\n" + + +def test_splice_function_empty_new_text(): + # Empty string: no trailing newline added (falsy check), splitlines gives []. + source = "a\nb\nc\n" + result = _splice_function(source, 2, 2, "") + assert result == "a\nc\n" + + +def test_extract_migration_reminder_basic(): + ctx_msg = _build_context_message([_make_fl_ctx()]) + reminder = _extract_migration_reminder(ctx_msg) + assert "Entity migration (quick reference)" in reminder + assert "pkg.sub_a" in reminder + assert "pkg.sub_b" in reminder + + +def test_extract_migration_reminder_empty_context(): + reminder = _extract_migration_reminder("no migration here") + assert reminder == "" + + +def test_extract_migration_reminder_no_entities(): + ctx = _make_fl_ctx(entity_to_target={}, new_module_paths={}) + ctx_msg = _build_context_message([ctx]) + # Empty entity_to_target → no bullets → reminder is empty string + reminder = _extract_migration_reminder(ctx_msg) + assert reminder == "" + + +def test_extract_migration_reminder_heading_stops_capture(): + # When a second fl_context follows the first, a new ## heading appears after + # the entity migration section — the extractor must stop capturing there. + ctx1 = _make_fl_ctx(old_module="pkg.big", filepath="/p/pkg/big.py") + ctx2 = _make_fl_ctx(old_module="pkg.large", filepath="/p/pkg/large.py") + ctx_msg = _build_context_message([ctx1, ctx2]) + reminder = _extract_migration_reminder(ctx_msg) + # The reminder should contain migration bullets from both contexts but + # not any heading markers. + assert "### Entity migration:" not in reminder + assert "## Split module:" not in reminder + assert "pkg.sub_a" in reminder + + +def test_get_external_import_names_absolute(): + src = "from pkg import Foo\nimport os\n" + names = _get_external_import_names(src) + assert "Foo" in names + assert "os" in names + + +def test_get_external_import_names_level1_skipped(): + src = "from .sub import Bar\nfrom . import Baz\n" + names = _get_external_import_names(src) + assert names == set() + + +def test_get_external_import_names_level2_included(): + src = "from ..pkg import Foo\nfrom ...llm_client import call_with_tool\n" + names = _get_external_import_names(src) + assert "Foo" in names + assert "call_with_tool" in names + + +def test_get_external_import_names_star_import_skipped(): + src = "from pkg import *\n" + names = _get_external_import_names(src) + assert names == set() + + +def test_get_external_import_names_asname(): + src = "import libcst as cst\nfrom pkg import Foo as F\n" + names = _get_external_import_names(src) + assert "cst" in names + assert "F" in names + assert "libcst" not in names + assert "Foo" not in names + + +def test_get_external_import_names_syntax_error(): + assert _get_external_import_names("def (broken:") == set() + + +def _make_ctx_with_ext_imports() -> _FLContext: + """Context where original_source has real external imports that moved.""" + orig = "from ...llm_client import call_with_tool\ndef foo(): pass\n" + mod = "from .llm_planning import call_with_tool\n" + new_files = { + "llm_planning.py": ( + "from ...llm_client import call_with_tool\ndef advise(): call_with_tool()\n" + ) + } + return _make_fl_ctx( + original_source=orig, + modified_source=mod, + new_files=new_files, + new_module_paths={"llm_planning.py": "pkg.llm_planning"}, + entity_to_target={"advise": "llm_planning.py"}, + ) + + +def test_extract_patch_lookup_basic(): + ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) + lookup = _extract_patch_lookup(ctx_msg) + assert "Patch target lookup" in lookup + assert "call_with_tool" in lookup + assert "pkg.llm_planning" in lookup + + +def test_extract_patch_lookup_no_section(): + # Default fixture has no external imports → no lookup section generated. + ctx_msg = _build_context_message([_make_fl_ctx()]) + assert _extract_patch_lookup(ctx_msg) == "" + + +def test_extract_patch_lookup_multiple_contexts(): + ctx1 = _make_ctx_with_ext_imports() + orig2 = "from ...config import CrispenConfig\ndef bar(): pass\n" + mod2 = "from .cfg import CrispenConfig\n" + new2 = {"cfg.py": "from ...config import CrispenConfig\ndef run(): pass\n"} + ctx2 = _make_fl_ctx( + old_module="pkg.other", + filepath="/proj/pkg/other.py", + original_source=orig2, + modified_source=mod2, + new_files=new2, + new_module_paths={"cfg.py": "pkg.cfg"}, + entity_to_target={"run": "cfg.py"}, + ) + ctx_msg = _build_context_message([ctx1, ctx2]) + lookup = _extract_patch_lookup(ctx_msg) + assert "call_with_tool" in lookup + assert "CrispenConfig" in lookup + + +def test_extract_patch_lookup_still_in_section(): + # Name in both original and modified → appears under "still imported". + orig = "from ...llm_client import call_with_tool, make_client\ndef foo(): pass\n" + mod = ( + "from ...llm_client import make_client\n" + "from .llm_planning import call_with_tool\n" + ) + new_files = { + "llm_planning.py": ( + "from ...llm_client import call_with_tool\ndef advise(): pass\n" + ) + } + ctx = _make_fl_ctx( + original_source=orig, + modified_source=mod, + new_files=new_files, + new_module_paths={"llm_planning.py": "pkg.llm_planning"}, + entity_to_target={"advise": "llm_planning.py"}, + ) + ctx_msg = _build_context_message([ctx]) + lookup = _extract_patch_lookup(ctx_msg) + assert "call_with_tool" in lookup + assert "make_client" in lookup + assert "still" in lookup + + +def test_extract_patch_lookup_name_not_in_new_files(): + # Name moved out but not found in any new file → "(not found in new files)". + orig = "from ...llm_client import call_with_tool\ndef foo(): pass\n" + mod = "" # name removed + new_files = {"sub.py": "class X: pass\n"} # no imports + ctx = _make_fl_ctx( + original_source=orig, + modified_source=mod, + new_files=new_files, + new_module_paths={"sub.py": "pkg.sub"}, + entity_to_target={}, + ) + ctx_msg = _build_context_message([ctx]) + lookup = _extract_patch_lookup(ctx_msg) + assert "not found in new files" in lookup + + +def test_extract_still_imported_names_basic(): + ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) + # _make_ctx_with_ext_imports has call_with_tool moved out — not still imported. + names = _extract_still_imported_names(ctx_msg) + assert "call_with_tool" not in names + + +def test_extract_still_imported_names_finds_retained(): + # Build a context where a name is retained in the modified original. + orig = "from ...llm_client import call_with_tool, make_client\ndef foo(): pass\n" + mod = ( + "from ...llm_client import make_client\n" + "from .llm_planning import call_with_tool\n" + ) + new_files = { + "llm_planning.py": ( + "from ...llm_client import call_with_tool\ndef advise(): pass\n" + ) + } + ctx = _make_fl_ctx( + original_source=orig, + modified_source=mod, + new_files=new_files, + new_module_paths={"llm_planning.py": "pkg.llm_planning"}, + entity_to_target={"advise": "llm_planning.py"}, + ) + ctx_msg = _build_context_message([ctx]) + names = _extract_still_imported_names(ctx_msg) + assert "make_client" in names + assert "call_with_tool" not in names + + +def test_extract_still_imported_names_no_section(): + # No lookup section in context → empty set. + names = _extract_still_imported_names("no relevant section here") + assert names == set() + + +def test_extract_still_imported_names_section_ends_at_non_bullet(): + # Section capture stops when a non-bullet line is encountered. + ctx_msg = ( + "Names still externally imported in the modified original (check):\n" + "- `alpha`\n" + "- `beta`\n" + "\n" # blank line — not a bullet, stops capture + "- `gamma`\n" # not captured + ) + names = _extract_still_imported_names(ctx_msg) + assert "alpha" in names + assert "beta" in names + assert "gamma" not in names + + +def test_extract_still_imported_names_malformed_bullet_ignored(): + # A bullet that starts with "- `" but has no closing backtick is silently skipped. + ctx_msg = ( + "Names still externally imported in the modified original (check):\n" + "- `valid`\n" + "- `\n" # malformed — no closing backtick → end <= 3 branch + ) + names = _extract_still_imported_names(ctx_msg) + assert "valid" in names + assert len(names) == 1 + + +def test_build_context_lookup_present_when_names_moved(): + ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) + assert "Patch target lookup" in ctx_msg + assert "call_with_tool" in ctx_msg + + +def test_build_context_lookup_annotates_using_entities(): + # When a moved-out name is used by a top-level entity in a new file, the + # lookup entry should include "used by: " so the LLM can pick the + # right sub-module when the name appears in multiple new files. + orig = "from ...llm_client import call_with_tool\ndef foo(): pass\n" + mod = "from .sub import call_with_tool\n" + new_files = { + "sub.py": ( + "from ...llm_client import call_with_tool\n" + "def _do_work(): call_with_tool()\n" + ) + } + ctx = _make_fl_ctx( + original_source=orig, + modified_source=mod, + new_files=new_files, + new_module_paths={"sub.py": "pkg.sub"}, + entity_to_target={"_do_work": "sub.py"}, + ) + ctx_msg = _build_context_message([ctx]) + assert "used by" in ctx_msg + assert "_do_work" in ctx_msg + + +def test_build_context_lookup_no_using_entities_when_name_unused(): + # If a moved-out name is imported but not referenced by any top-level entity, + # the entry should not include a "used by" annotation. + orig = "from ...llm_client import call_with_tool\ndef foo(): pass\n" + mod = "from .sub import call_with_tool\n" + new_files = {"sub.py": "from ...llm_client import call_with_tool\n"} + ctx = _make_fl_ctx( + original_source=orig, + modified_source=mod, + new_files=new_files, + new_module_paths={"sub.py": "pkg.sub"}, + entity_to_target={}, + ) + ctx_msg = _build_context_message([ctx]) + assert "used by" not in ctx_msg + + +def test_build_context_lookup_absent_when_no_ext_imports(): + # Default fixture has class defs only — no external imports. + ctx_msg = _build_context_message([_make_fl_ctx()]) + assert "Patch target lookup" not in ctx_msg + + +def test_build_context_lookup_only_still_in(): + # All external imports preserved in modified original → only "still imported" + # section, no "moved" section. Covers the if moved_out: False branch. + # sub.py does NOT import make_client → "NOT imported in any new submodule". + orig = "from ...llm_client import make_client\ndef foo(): pass\n" + mod = "from ...llm_client import make_client\nfrom .sub import helper\n" + ctx = _make_fl_ctx( + original_source=orig, + modified_source=mod, + new_files={"sub.py": "def helper(): pass\n"}, + new_module_paths={"sub.py": "pkg.sub"}, + entity_to_target={"helper": "sub.py"}, + ) + ctx_msg = _build_context_message([ctx]) + assert "Patch target lookup" in ctx_msg + assert "still" in ctx_msg + assert "moved" not in ctx_msg + assert "NOT imported in any new submodule" in ctx_msg + + +def test_build_context_lookup_still_in_also_in_new_submodule_with_users(): + # A still-in name imported by a new submodule whose entity USES it → + # annotation shows "used by" and the migration-based guidance. + orig = "from ...llm_client import make_client\ndef foo(): pass\n" + mod = "from ...llm_client import make_client\nfrom .sub import helper\n" + ctx = _make_fl_ctx( + original_source=orig, + modified_source=mod, + new_files={ + "sub.py": ( + "from ...llm_client import make_client\n" + "def helper(): make_client()\n" + ) + }, + new_module_paths={"sub.py": "pkg.sub"}, + entity_to_target={"helper": "sub.py"}, + ) + ctx_msg = _build_context_message([ctx]) + assert "also externally imported in" in ctx_msg + assert "pkg.sub" in ctx_msg + assert "used by" in ctx_msg + assert "helper" in ctx_msg + assert "migrated to that submodule" in ctx_msg + assert "Name references" in ctx_msg + + +def test_build_context_lookup_still_in_also_in_new_submodule_no_users(): + # A still-in name imported by a new submodule but NOT referenced by any + # top-level entity → annotation shows the submodule without "used by". + orig = "from ...llm_client import make_client\ndef foo(): pass\n" + mod = "from ...llm_client import make_client\nfrom .sub import helper\n" + ctx = _make_fl_ctx( + original_source=orig, + modified_source=mod, + new_files={ + "sub.py": "from ...llm_client import make_client\ndef helper(): pass\n" + }, + new_module_paths={"sub.py": "pkg.sub"}, + entity_to_target={"helper": "sub.py"}, + ) + ctx_msg = _build_context_message([ctx]) + assert "also externally imported in" in ctx_msg + assert "pkg.sub" in ctx_msg + # No entity in sub.py uses make_client → no "(used by: ...)" parenthetical. + assert "(used by:" not in ctx_msg + + +def test_build_classify_prompt_no_prev(): + prompt = _build_classify_prompt( + _ctx_msg(), "def test_f(): pass", ["crispen.before.X"] + ) + assert "crispen.before.X" in prompt + assert "Previous attempt was rejected" not in prompt + assert "patch_renames" in prompt + assert "Entity migration (quick reference)" in prompt + + +def test_build_classify_prompt_with_prev(): + prompt = _build_classify_prompt( + _ctx_msg(), + "def test_f(): pass", + ["crispen.before.X"], + prev_issue="wrong module", + prev_proposed="{'crispen.before.X': 'bad.mod.X'}", + ) + assert "Previous attempt was rejected" in prompt + assert "wrong module" in prompt + assert "bad.mod.X" in prompt + + +def test_build_classify_prompt_multiple_paths(): + prompt = _build_classify_prompt( + _ctx_msg(), "def test_f(): pass", ["crispen.before.X", "crispen.before.Y"] + ) + assert "crispen.before.X" in prompt + assert "crispen.before.Y" in prompt + + +def test_build_classify_prompt_with_lookup(): + # When the context has a patch target lookup, it appears in the classify prompt + # and the simplified lookup-based algorithm is used. + ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) + prompt = _build_classify_prompt( + ctx_msg, "def test_f(): pass", ["pkg.big.call_with_tool"] + ) + assert "Patch target lookup" in prompt + assert "call_with_tool" in prompt + assert "pkg.llm_planning" in prompt + assert "patch_renames" in prompt + assert "Entity migration (quick reference)" in prompt + + +def test_build_classify_prompt_with_stable_paths(): + # stable_patch_paths appear in a separate "already correct" section and + # the forking path remains in the "needs updating" section. + prompt = _build_classify_prompt( + _ctx_msg(), + "def test_f(): pass", + ["crispen.before.X"], + stable_patch_paths=["crispen.after.Y"], + ) + assert "crispen.before.X" in prompt + assert "crispen.after.Y" in prompt + assert "already correct" in prompt + assert "do not modify" in prompt + + +def test_build_func_verify_prompt_basic(): + prompt = _build_func_verify_prompt( + _ctx_msg(), + "def test_f(): pass", + {"crispen.before.X": "crispen.after.X"}, + ) + assert "crispen.before.X" in prompt + assert "crispen.after.X" in prompt + assert "correct" in prompt + + +def test_build_func_verify_prompt_multiple_renames(): + prompt = _build_func_verify_prompt( + _ctx_msg(), + "def test_f(): pass", + {"crispen.before.X": "crispen.after.X", "crispen.before.Y": "crispen.after.Y"}, + ) + assert "crispen.before.X" in prompt + assert "crispen.before.Y" in prompt + assert "crispen.after.X" in prompt + assert "crispen.after.Y" in prompt + + +def test_build_func_verify_prompt_includes_patch_lookup(): + # When the context has a patch lookup section, it should be repeated near + # the verify instructions. + ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) + prompt = _build_func_verify_prompt( + ctx_msg, + "def test_f(): pass", + {"pkg.old.call_with_tool": "pkg.llm_planning.call_with_tool"}, + ) + assert "Patch target lookup" in prompt + + +def test_build_no_change_verify_prompt_includes_migration_reminder(): + # Prompt built with a context that has migration entries should include + # the migration quick-reference block near the instructions. + prompt = _build_no_change_verify_prompt( + _ctx_msg(), + "def test_f(): pass", + ["crispen.before.X"], + ) + assert "crispen.before.X" in prompt + assert "Entity migration" in prompt + + +def test_build_no_change_verify_prompt_includes_patch_lookup(): + # When the context has a patch lookup section, it should be repeated near + # the verify instructions so the model doesn't have to scan the full context. + ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) + prompt = _build_no_change_verify_prompt( + ctx_msg, + "def test_f(): pass", + ["pkg.old.call_with_tool"], + ) + assert "Patch target lookup" in prompt + + +def test_build_no_change_verify_prompt_with_stable_paths(): + # stable_patch_paths appear in a separate "already correct" section and + # the instruction tells the verifier not to include them in corrections. + prompt = _build_no_change_verify_prompt( + _ctx_msg(), + "def test_f(): pass", + ["crispen.before.X"], + stable_patch_paths=["crispen.after.Y"], + ) + assert "crispen.before.X" in prompt + assert "crispen.after.Y" in prompt + assert "already correct" in prompt + assert "do not include" in prompt + + +def test_build_rewrite_func_prompt_no_error(): + prompt = _build_rewrite_func_prompt( + _ctx_msg(), "def test_f(): pass", ["crispen.before.X"] + ) + assert "crispen.before.X" in prompt + assert "Previous rewrite" not in prompt + assert "Rewrite the complete function" in prompt + + +def test_build_rewrite_func_prompt_with_error(): + prompt = _build_rewrite_func_prompt( + _ctx_msg(), + "def test_f(): pass", + ["crispen.before.X"], + prev_error="SyntaxError on line 3", + ) + assert "Previous rewrite was rejected" in prompt + assert "SyntaxError on line 3" in prompt + + +def test_build_rewrite_func_prompt_with_stable_paths(): + # stable_patch_paths appear in a separate "already correct" section and + # the instruction tells the LLM not to modify them. + prompt = _build_rewrite_func_prompt( + _ctx_msg(), + "def test_f(): pass", + ["crispen.before.X"], + stable_patch_paths=["crispen.after.Y"], + ) + assert "crispen.before.X" in prompt + assert "crispen.after.Y" in prompt + assert "already correct" in prompt + assert "do not modify" in prompt.lower() + + +def test_build_rewrite_verify_prompt_basic(): + prompt = _build_rewrite_verify_prompt( + _ctx_msg(), + "def test_f(): pass", + '@patch("crispen.after.X")\ndef test_f(mock_x):\n pass\n', + ) + assert "Original test function" in prompt + assert "Rewritten test function" in prompt + assert "crispen.after.X" in prompt + assert "correct" in prompt + + +def test_build_classify_prompt_with_candidates(): + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + prompt = _build_classify_prompt( + context_msg, + "def test_f(): pass\n", + ["pkg.big.A"], + candidates_per_path={"pkg.big.A": ["pkg.sub_a.A", "pkg.sub_b.A"]}, + ) + assert "Call-graph candidate paths" in prompt + assert "pkg.sub_a.A" in prompt + assert "pkg.sub_b.A" in prompt + + +def test_build_classify_prompt_candidates_above_threshold(): + # Candidates count > threshold → section not included. + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + many_cands = [f"pkg.sub_{i}.A" for i in range(_CG_CANDIDATES_LLM_THRESHOLD + 1)] + prompt = _build_classify_prompt( + context_msg, + "def test_f(): pass\n", + ["pkg.big.A"], + candidates_per_path={"pkg.big.A": many_cands}, + ) + assert "Call-graph candidate paths" not in prompt + + +def test_build_func_verify_prompt_with_candidates(): + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + prompt = _build_func_verify_prompt( + context_msg, + "def test_f(): pass\n", + {"pkg.big.A": "pkg.sub_a.A"}, + candidates_per_path={"pkg.big.A": ["pkg.sub_a.A", "pkg.sub_b.A"]}, + ) + assert "Call-graph candidate paths" in prompt + assert "pkg.sub_a.A" in prompt + + +def test_build_func_verify_prompt_candidates_above_threshold(): + # All candidate lists exceed the threshold → section not included. + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + many_cands = [f"pkg.sub_{i}.A" for i in range(_CG_CANDIDATES_LLM_THRESHOLD + 1)] + prompt = _build_func_verify_prompt( + context_msg, + "def test_f(): pass\n", + {"pkg.big.A": "pkg.sub_a.A"}, + candidates_per_path={"pkg.big.A": many_cands}, + ) + assert "Call-graph candidate paths" not in prompt + + +def test_build_no_change_verify_prompt_with_candidates(): + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + prompt = _build_no_change_verify_prompt( + context_msg, + "def test_f(): pass\n", + ["pkg.big.A"], + candidates_per_path={"pkg.big.A": ["pkg.sub_a.A"]}, + ) + assert "Call-graph candidate paths" in prompt + assert "pkg.sub_a.A" in prompt + + +def test_build_no_change_verify_prompt_candidates_above_threshold(): + # All candidate lists exceed the threshold → section not included. + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + many_cands = [f"pkg.sub_{i}.A" for i in range(_CG_CANDIDATES_LLM_THRESHOLD + 1)] + prompt = _build_no_change_verify_prompt( + context_msg, + "def test_f(): pass\n", + ["pkg.big.A"], + candidates_per_path={"pkg.big.A": many_cands}, + ) + assert "Call-graph candidate paths" not in prompt + + +def test_build_rewrite_func_prompt_with_candidates(): + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + prompt = _build_rewrite_func_prompt( + context_msg, + "def test_f(): pass\n", + ["pkg.big.A"], + candidates_per_path={"pkg.big.A": ["pkg.sub_a.A", "pkg.helpers.A"]}, + ) + assert "Call-graph candidate paths" in prompt + assert "pkg.sub_a.A" in prompt + assert "pkg.helpers.A" in prompt + + +def test_build_rewrite_func_prompt_candidates_above_threshold(): + # All candidate lists exceed the threshold → section not included. + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + many_cands = [f"pkg.sub_{i}.A" for i in range(_CG_CANDIDATES_LLM_THRESHOLD + 1)] + prompt = _build_rewrite_func_prompt( + context_msg, + "def test_f(): pass\n", + ["pkg.big.A"], + candidates_per_path={"pkg.big.A": many_cands}, + ) + assert "Call-graph candidate paths" not in prompt + + +def test_build_rewrite_verify_prompt_with_candidates(): + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + prompt = _build_rewrite_verify_prompt( + context_msg, + "def test_f(): pass\n", + "def test_f(): pass\n", + candidates_per_path={"pkg.big.A": ["pkg.sub_a.A", "pkg.sub_b.A"]}, + ) + assert "Call-graph candidate paths" in prompt + assert "pkg.sub_a.A" in prompt + + +def test_build_rewrite_verify_prompt_candidates_above_threshold(): + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + many_cands = [f"pkg.sub_{i}.A" for i in range(_CG_CANDIDATES_LLM_THRESHOLD + 1)] + prompt = _build_rewrite_verify_prompt( + context_msg, + "def test_f(): pass\n", + "def test_f(): pass\n", + candidates_per_path={"pkg.big.A": many_cands}, + ) + assert "Call-graph candidate paths" not in prompt + + +def test_build_rewrite_verify_prompt_no_candidates(): + ctx = _make_fl_ctx_simple() + context_msg = _build_context_message([ctx]) + prompt = _build_rewrite_verify_prompt( + context_msg, + "def test_f(): pass\n", + "def test_f(): pass\n", + ) + assert "Call-graph candidate paths" not in prompt + assert "Verify that the rewrite is correct" in prompt diff --git a/tests/patch_rewriter/test_misc.py b/tests/patch_rewriter/test_misc.py new file mode 100644 index 0000000..1ee249b --- /dev/null +++ b/tests/patch_rewriter/test_misc.py @@ -0,0 +1,270 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch as mock_patch +from crispen.patch_rewriter import _process_file_source +from .helpers import _CFG, _PATCH_CALL_TOOL, _VERIFY_OK, _ok + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_passthrough_votes_conflict_with_rename_proposal(mock_call, tmp_path): + """One test (A) renames Y but not X → casts "keep old" vote for X. + Another test (B) renames X → casts "rename" vote for X. + "keep old" + "rename" → conflicting proposals → inline test_b with new value; + test_a's decorator unchanged. TARGET2 (Y) has a single rename vote → updated + via same_file_const_map. + + Covers: + - "keep old" vote (new_val is None) entered into same_file_proposals + - conflict detection (len > 1) → conflicting_old_vals + - per-function inline for test_b (existing_idx is None → append) + - test_a in conflicting inline loop with new_val=None → inline_subs empty + → continue + - single-proposal for TARGET2 (value != old) → same_file_const_map update + """ + src = ( + 'TARGET = "crispen.before.X"\n' + 'TARGET2 = "crispen.before.Y"\n' + "\n" + "@patch(TARGET)\n" + "@patch(TARGET2)\n" + "def test_a(mock_y, mock_x):\n" + " pass\n" + "\n" + "@patch(TARGET)\n" + "def test_b(mock_x):\n" + " pass\n" + ) + scan = str(tmp_path / "test_foo.py") + # test_a renames Y but NOT X → X gets a "keep old" vote, Y gets a rename vote. + # test_b renames X → X gets a "rename to after.X" vote. + # X proposals: {old, after.X} → conflicting → inline test_b, test_a unchanged. + # Y proposals: {after.Y} → single, != old → same_file_const_map update. + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.Y": "crispen.after.Y"}, + } + ), + _ok(_VERIFY_OK), + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.after.X"}, + } + ), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + src, + {"crispen.before.X", "crispen.before.Y"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + ) + assert changed is True + # X has conflicting votes → TARGET NOT updated globally. + assert 'TARGET = "crispen.before.X"' in result + # Y has single vote → TARGET2 updated via same_file_const_map. + assert 'TARGET2 = "crispen.after.Y"' in result + # test_b's X decorator is inlined individually. + assert '@patch("crispen.after.X")' in result + # test_a's decorator unchanged (its inline_subs were empty). + assert "@patch(TARGET)" in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_passthrough_identity_proposal_skipped(mock_call, tmp_path): + """One test renames Y but not X. X only receives a "keep old" identity vote. + Expected: TARGET not updated (identity guard: proposed == old); TARGET2 updated. + + Covers the ``next(iter(new_set)) != old`` identity guard in same_file_const_map + that drops entries where the sole proposal equals the existing value. + """ + src = ( + 'TARGET = "crispen.before.X"\n' + 'TARGET2 = "crispen.before.Y"\n' + "\n" + "@patch(TARGET)\n" + "@patch(TARGET2)\n" + "def test_a(mock_y, mock_x):\n" + " pass\n" + ) + scan = str(tmp_path / "test_foo.py") + # test_a: renames Y → after.Y, does not rename X. + # X proposals: {"crispen.before.X"} → len==1, value==old → identity skip. + # Y proposals: {"crispen.after.Y"} → len==1, value!=old → const_map update. + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.Y": "crispen.after.Y"}, + } + ), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + src, + {"crispen.before.X", "crispen.before.Y"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + ) + assert changed is True + # X got only an identity vote → not in same_file_const_map → TARGET unchanged. + assert 'TARGET = "crispen.before.X"' in result + # Y got a rename vote → TARGET2 updated. + assert 'TARGET2 = "crispen.after.Y"' in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_conflict_two_renames_existing_splice(mock_call, tmp_path): + """Two tests rename the same constant to *different* targets → conflict. + test_a also renames a literal patch → it gets a func_splice from string_swap. + Expected: both functions get inlined with their respective literals; + test_a's existing splice is *updated in place* (existing_idx path). + + Covers: + - lines 1763-1772 (loop, build inline_subs) + - line 1787-False (inlined != base_text) + - line 1789-True (existing_idx not None → update splice) + - line 1792 (existing_idx is None → append splice, for test_b) + """ + src = ( + 'TARGET = "crispen.before.X"\n' + "\n" + "@patch(TARGET)\n" + '@patch("crispen.before.Z")\n' + "def test_a(mock_z, mock_x):\n" + " pass\n" + "\n" + "@patch(TARGET)\n" + "def test_b(mock_x):\n" + " pass\n" + ) + scan = str(tmp_path / "test_foo.py") + # test_a renames X → after_a.X and Z → after.Z. + # test_b renames X → after_b.X. + # Two different targets for X → conflict → inline each function individually. + # test_a's Z literal rename creates an existing func_splice; the inline step + # must update that existing splice rather than appending a new one. + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": { + "crispen.before.X": "crispen.after_a.X", + "crispen.before.Z": "crispen.after.Z", + }, + } + ), + _ok(_VERIFY_OK), + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.after_b.X"}, + } + ), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + src, + {"crispen.before.X", "crispen.before.Z"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + ) + assert changed is True + # The shared TARGET constant must NOT be updated (conflict). + assert 'TARGET = "crispen.before.X"' in result + # test_a: Z literal renamed, X constant inlined. + assert '@patch("crispen.after_a.X")' in result + assert '@patch("crispen.after.Z")' in result + # test_b: X constant inlined with its own target. + assert '@patch("crispen.after_b.X")' in result + # No original constant-style decorator survives. + assert "@patch(TARGET)" not in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_conflict_two_proposals_passthrough_function_continue( + mock_call, tmp_path +): + """Two functions propose *different* values for TARGET → conflicting_old_vals. + A third function also uses TARGET but only renames a different const (TARGET_Y). + Expected: the third function is in string_swap_results but triggers the + ``continue`` branch in the conflicting_old_vals inline loop (inline_subs + empty for X); the other two get their decorators inlined individually. + + Covers the ``if not inline_subs: continue`` branch inside the + ``if conflicting_old_vals:`` block (via two sub-paths): + - ref.resolved_value NOT in conflicting_old_vals (Y ref → loop continues) + - ref.resolved_value in conflicting_old_vals but new_val is None (X ref) + """ + src = ( + 'TARGET = "crispen.before.X"\n' + 'TARGET_Y = "crispen.before.Y"\n' + "\n" + "@patch(TARGET)\n" + "def test_a(mock_x):\n" + " pass\n" + "\n" + "@patch(TARGET)\n" + "def test_b(mock_x):\n" + " pass\n" + "\n" + "@patch(TARGET)\n" + "@patch(TARGET_Y)\n" + "def test_c(mock_y, mock_x):\n" + " pass\n" + ) + scan = str(tmp_path / "test_foo.py") + # test_a → after_a.X; test_b → after_b.X (two different proposals → conflicting) + # test_c → renames Y only (not X) → in string_swap_results but inline_subs empty + # for X → continue. Y gets a single proposal → same_file_const_map update. + mock_call.side_effect = [ + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.after_a.X"}, + } + ), + _ok(_VERIFY_OK), + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.X": "crispen.after_b.X"}, + } + ), + _ok(_VERIFY_OK), + _ok( + { + "needs_rewrite": False, + "patch_renames": {"crispen.before.Y": "crispen.after.Y"}, + } + ), + _ok(_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + src, + {"crispen.before.X", "crispen.before.Y"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file=scan, + ) + assert changed is True + # X: conflicting (two proposals) → const unchanged, test_a and test_b inlined. + assert 'TARGET = "crispen.before.X"' in result + assert '@patch("crispen.after_a.X")' in result + assert '@patch("crispen.after_b.X")' in result + # Y: single proposal → const updated via same_file_const_map. + assert 'TARGET_Y = "crispen.after.Y"' in result + # test_c: in string_swap_results (renamed Y) but X inline_subs empty → continue. + assert "@patch(TARGET)" in result diff --git a/tests/patch_rewriter/test_patch_detection.py b/tests/patch_rewriter/test_patch_detection.py new file mode 100644 index 0000000..2eee214 --- /dev/null +++ b/tests/patch_rewriter/test_patch_detection.py @@ -0,0 +1,337 @@ +from __future__ import annotations +from crispen.patch_rewriter import ( + _compiles, + _find_test_functions_to_update, + _find_with_patch_paths_in_body, + _is_patch_call, + _matches_any, + _patch_strings_in_text, +) +import libcst as cst + + +def test_is_patch_call_name_match(): + call_node = cst.parse_expression('patch("foo")') + assert _is_patch_call(call_node) is True + + +def test_is_patch_call_attribute_match(): + call_node = cst.parse_expression('mock.patch("foo")') + assert _is_patch_call(call_node) is True + + +def test_is_patch_call_other_name(): + call_node = cst.parse_expression('other("foo")') + assert _is_patch_call(call_node) is False + + +def test_matches_any_exact(): + assert _matches_any("a.b.C", {"a.b.C"}) is True + + +def test_matches_any_prefix(): + assert _matches_any("a.b.C.method", {"a.b.C"}) is True + + +def test_matches_any_near_miss(): + # "a.b.CExtra" should NOT match "a.b.C" + assert _matches_any("a.b.CExtra", {"a.b.C"}) is False + + +def test_matches_any_no_match(): + assert _matches_any("x.y.Z", {"a.b.C"}) is False + + +def test_compiles_valid(): + assert _compiles("x = 1\n") is True + + +def test_compiles_invalid(): + assert _compiles("def f(:\n pass\n") is False + + +def test_find_empty_old_paths(): + src = '@patch("crispen.before.X")\ndef test_f(): pass\n' + assert _find_test_functions_to_update(src, set()) == [] + + +def test_find_parse_error(): + assert _find_test_functions_to_update("def f(:\n", {"crispen.before.X"}) == [] + + +def test_find_no_match(): + src = '@patch("other.mod.Y")\ndef test_f(): pass\n' + assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] + + +def test_find_match_exact(): + src = '@patch("crispen.before.X")\ndef test_f(): pass\n' + result = _find_test_functions_to_update(src, {"crispen.before.X"}) + assert len(result) == 1 + assert result[0].function_name == "test_f" + assert "crispen.before.X" in result[0].old_patch_paths + + +def test_find_match_prefix(): + src = '@patch("crispen.before.X.method")\ndef test_f(): pass\n' + result = _find_test_functions_to_update(src, {"crispen.before.X"}) + assert len(result) == 1 + assert "crispen.before.X.method" in result[0].old_patch_paths + + +def test_find_not_a_call_decorator(): + # @patch used as a bare name (no parentheses), not a Call node. + src = "@patch\ndef test_f(): pass\n" + assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] + + +def test_find_no_args(): + src = "@patch()\ndef test_f(): pass\n" + assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] + + +def test_find_arg_not_simple_string(): + # @patch(some_variable) — first arg is a Name, not a SimpleString. + src = "@patch(some_var)\ndef test_f(): pass\n" + assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] + + +def test_find_prefixed_string(): + # b"..." — raw[0] is 'b', not a quote character. + src = '@patch(b"crispen.before.X")\ndef test_f(): pass\n' + assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] + + +def test_find_triple_quoted(): + src = '@patch("""crispen.before.X""")\ndef test_f(): pass\n' + assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] + + +def test_find_not_patch_name(): + # @decorate("crispen.before.X") — attribute name is not "patch". + src = '@decorate("crispen.before.X")\ndef test_f(): pass\n' + assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] + + +def test_find_attribute_patch(): + # @mock.patch("crispen.before.X") — Attribute form. + src = '@mock.patch("crispen.before.X")\ndef test_f(): pass\n' + result = _find_test_functions_to_update(src, {"crispen.before.X"}) + assert len(result) == 1 + assert result[0].function_name == "test_f" + + +def test_find_multiple_functions(): + src = ( + '@patch("crispen.before.X")\ndef test_a(): pass\n\n' + '@patch("crispen.before.Y")\ndef test_b(): pass\n' + ) + result = _find_test_functions_to_update( + src, {"crispen.before.X", "crispen.before.Y"} + ) + assert {f.function_name for f in result} == {"test_a", "test_b"} + + +def test_find_full_text_includes_decorator(): + src = '@patch("crispen.before.X")\ndef test_f():\n pass\n' + result = _find_test_functions_to_update(src, {"crispen.before.X"}) + assert '@patch("crispen.before.X")' in result[0].full_text + assert "def test_f" in result[0].full_text + + +def test_find_start_end_lines(): + # line 1: # header, line 2: @patch..., line 3: def test_f, line 4: pass + src = "# header\n" '@patch("crispen.before.X")\n' "def test_f():\n" " pass\n" + result = _find_test_functions_to_update(src, {"crispen.before.X"}) + assert result[0].start_line == 2 # @patch line (first decorator) + assert result[0].end_line == 4 # last line of body + + +def test_find_body_with_patch_no_decorator(): + # Function has no @patch decorator but uses ``with patch(...)`` in the body. + src = ( + "def test_f():\n" ' with patch("crispen.before.X") as m:\n' " pass\n" + ) + result = _find_test_functions_to_update(src, {"crispen.before.X"}) + assert len(result) == 1 + assert result[0].function_name == "test_f" + assert "crispen.before.X" in result[0].old_patch_paths + # start_line should be the ``def`` line (no decorators). + assert result[0].start_line == 1 + + +def test_find_body_with_patch_combined_with_decorator(): + # Function has both an @patch decorator and a body-level with patch(...). + src = ( + '@patch("crispen.before.Y")\n' + "def test_f(mock_y):\n" + ' with patch("crispen.before.X") as m:\n' + " pass\n" + ) + result = _find_test_functions_to_update( + src, {"crispen.before.X", "crispen.before.Y"} + ) + assert len(result) == 1 + paths = result[0].old_patch_paths + assert "crispen.before.X" in paths + assert "crispen.before.Y" in paths + + +def test_body_scan_syntax_error(): + assert _find_with_patch_paths_in_body("def f(:\n", {"old.X"}, {}, {}) == [] + + +def test_body_scan_no_funcdef(): + # Parsed text has no FunctionDef at the top level. + assert _find_with_patch_paths_in_body("x = 1\n", {"old.X"}, {}, {}) == [] + + +def test_body_scan_simple_match(): + src = 'def test_f():\n with patch("old.X") as m:\n pass\n' + result = _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) + assert result == ["old.X"] + + +def test_body_scan_no_match(): + src = 'def test_f():\n with patch("other.Y") as m:\n pass\n' + assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] + + +def test_body_scan_attribute_patch(): + # ``with mock.patch(...)`` form. + src = 'def test_f():\n with mock.patch("old.X") as m:\n pass\n' + result = _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) + assert result == ["old.X"] + + +def test_body_scan_not_patch_call(): + src = 'def test_f():\n with other("old.X") as m:\n pass\n' + assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] + + +def test_body_scan_no_args(): + src = "def test_f():\n with patch() as m:\n pass\n" + assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] + + +def test_body_scan_non_call_context_manager(): + # Context manager is a plain Name, not a Call. + src = "def test_f():\n with ctx as m:\n pass\n" + assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] + + +def test_body_scan_non_string_arg(): + # First arg is a Call expression (not string/Name/Attribute). + src = "def test_f():\n with patch(get_target()) as m:\n pass\n" + assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] + + +def test_body_scan_name_const_match(): + const_map = {"MY_TARGET": ("old.X", "/file.py")} + src = "def test_f():\n with patch(MY_TARGET) as m:\n pass\n" + result = _find_with_patch_paths_in_body(src, {"old.X"}, const_map, {}) + assert result == ["old.X"] + + +def test_body_scan_name_const_no_match(): + # Constant value doesn't match old_paths. + const_map = {"MY_TARGET": ("other.Y", "/file.py")} + src = "def test_f():\n with patch(MY_TARGET) as m:\n pass\n" + assert _find_with_patch_paths_in_body(src, {"old.X"}, const_map, {}) == [] + + +def test_body_scan_name_not_in_const_map(): + src = "def test_f():\n with patch(unknown_var) as m:\n pass\n" + assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] + + +def test_body_scan_attr_const_match(): + attr_const_map = {"consts": {"TARGET": ("old.X", "/consts.py")}} + src = "def test_f():\n with patch(consts.TARGET) as m:\n pass\n" + result = _find_with_patch_paths_in_body(src, {"old.X"}, {}, attr_const_map) + assert result == ["old.X"] + + +def test_body_scan_attr_const_module_not_in_map(): + src = "def test_f():\n with patch(unknown_mod.X) as m:\n pass\n" + assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] + + +def test_body_scan_attr_const_attr_not_in_map(): + attr_const_map = {"consts": {"OTHER": ("old.X", "/consts.py")}} + src = "def test_f():\n with patch(consts.MISSING) as m:\n pass\n" + assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, attr_const_map) == [] + + +def test_body_scan_attr_const_no_match(): + # Attribute constant value doesn't match old_paths. + attr_const_map = {"consts": {"TARGET": ("other.Y", "/consts.py")}} + src = "def test_f():\n with patch(consts.TARGET) as m:\n pass\n" + assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, attr_const_map) == [] + + +def test_body_scan_nested_funcdef_excluded(): + # ``with patch(...)`` inside a nested function should NOT trigger inclusion of + # the outer function — the nested function is its own unit. + src = ( + "def test_outer():\n" + " def inner():\n" + ' with patch("old.X") as m:\n' + " pass\n" + ) + assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] + + +def test_body_scan_multiple_with_items(): + # ``with patch("a") as m, patch("b") as n:`` — both items should be found. + src = ( + "def test_f():\n" + ' with patch("old.X") as m, patch("old.Y") as n:\n' + " pass\n" + ) + result = _find_with_patch_paths_in_body(src, {"old.X", "old.Y"}, {}, {}) + assert set(result) == {"old.X", "old.Y"} + + +def test_body_scan_async_with(): + src = 'async def test_f():\n async with patch("old.X") as m:\n pass\n' + result = _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) + assert result == ["old.X"] + + +def test_body_scan_nested_in_if(): + # ``with patch(...)`` inside an ``if`` block should still be found. + src = ( + "def test_f():\n" + " if True:\n" + ' with patch("old.X") as m:\n' + " pass\n" + ) + result = _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) + assert result == ["old.X"] + + +def test_patch_strings_in_text_decorator(): + text = '@patch("pkg.mod.A")\ndef test_f(m): pass\n' + assert _patch_strings_in_text(text) == {"pkg.mod.A"} + + +def test_patch_strings_in_text_attribute_decorator(): + text = '@mock.patch("pkg.mod.B")\ndef test_f(m): pass\n' + assert _patch_strings_in_text(text) == {"pkg.mod.B"} + + +def test_patch_strings_in_text_context_manager(): + text = 'def test_f():\n with patch("pkg.mod.C") as m: pass\n' + assert _patch_strings_in_text(text) == {"pkg.mod.C"} + + +def test_patch_strings_in_text_multiple(): + text = ( + '@patch("pkg.mod.A")\n' '@mock.patch("pkg.mod.B")\n' "def test_f(a, b): pass\n" + ) + assert _patch_strings_in_text(text) == {"pkg.mod.A", "pkg.mod.B"} + + +def test_patch_strings_in_text_empty(): + assert _patch_strings_in_text("def test_f(): pass\n") == set() diff --git a/tests/patch_rewriter/test_process_basic.py b/tests/patch_rewriter/test_process_basic.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/tests/patch_rewriter/test_process_basic.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/patch_rewriter/test_process_rewrite.py b/tests/patch_rewriter/test_process_rewrite.py new file mode 100644 index 0000000..2ce5370 --- /dev/null +++ b/tests/patch_rewriter/test_process_rewrite.py @@ -0,0 +1,457 @@ +from __future__ import annotations +from unittest.mock import MagicMock, patch as mock_patch +from crispen.config import CrispenConfig +from crispen.llm_client import LLMCallResult +from crispen.patch_rewriter import ( + RewriteAccumulator, + _FLContext, + _build_context_message, + _process_file_source, +) +from .helpers import ( + _CFG, + _CFG_NO_LLM_VERIFY, + _CLASSIFY_REWRITE, + _FORKING_PATHS, + _PATCH_CALL_PR, + _PATCH_CALL_TOOL, + _PATCH_GET_KEY_PR, + _REWRITE_VERIFY_OK, + _REWRITE_VERIFY_REJECT, + _SRC_WITH_PATCH, + _VALID_REWRITE, + _make_process_cfg, + _ok, + _truncated_ok, +) +from . import helpers + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_needs_rewrite_success(mock_call): + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(_REWRITE_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is True + assert "crispen.after.X" in result + assert "crispen.after.Y" in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_needs_rewrite_tool_none(mock_call): + # Rewrite call returns tool_input=None → no update. + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok(None), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert result == _SRC_WITH_PATCH + assert changed is False + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_needs_rewrite_empty_text(mock_call): + # Rewrite returns empty string → no update. + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": ""}), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is False + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_needs_rewrite_non_string(mock_call): + # Rewrite returns non-string value → no update. + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": 42}), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is False + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_needs_rewrite_compile_error_retry(mock_call): + # First rewrite has syntax error; second is valid. + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": "def f(:\n pass\n"}), # invalid + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(_REWRITE_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 + ) + assert changed is True + assert "crispen.after.X" in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_needs_rewrite_compile_error_exhausted(mock_call): + # Both rewrite attempts fail to compile → no update. + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": "def f(:\n pass\n"}), # invalid + _ok({"rewritten_function": "def f(:\n pass\n"}), # still invalid + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 + ) + assert changed is False + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_needs_rewrite_verify_none_accept(mock_call): + # Verify returns tool_input=None → accept the rewrite. + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(None), # verify returns None → accept + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 + ) + assert changed is True + assert "crispen.after.X" in result + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_needs_rewrite_verify_truncated_reject(mock_call): + # Rewrite verify truncated → treated as rejection, rewrite not accepted. + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": _VALID_REWRITE}), + _truncated_ok(), # verify truncated → reject + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 1 + ) + assert result == _SRC_WITH_PATCH + assert changed is False + assert mock_call.call_count == 3 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_needs_rewrite_verify_rejected_then_accept(mock_call): + # Verify rejects first rewrite; second rewrite+verify is accepted. + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(_REWRITE_VERIFY_REJECT), + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(_REWRITE_VERIFY_OK), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 + ) + assert changed is True + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_needs_rewrite_verify_rejected_exhausted(mock_call): + # Verify rejects with llm_verify_retries=0 → no update. + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(_REWRITE_VERIFY_REJECT), + ] + result, changed, cross = _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 1 + ) + assert result == _SRC_WITH_PATCH + assert changed is False + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_acc_rewrite_accumulates(mock_call): + """Full rewrite path accumulates classify, rewrite, and verify calls.""" + mock_call.side_effect = [ + LLMCallResult( + tool_input=_CLASSIFY_REWRITE, + elapsed=0.5, + input_tokens=100, + output_tokens=10, + ), + LLMCallResult( + tool_input={"rewritten_function": _VALID_REWRITE}, + elapsed=1.5, + input_tokens=300, + output_tokens=60, + ), + LLMCallResult( + tool_input=_REWRITE_VERIFY_OK, + elapsed=0.2, + input_tokens=80, + output_tokens=5, + ), + ] + acc = RewriteAccumulator() + _process_file_source( + _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1, _acc=acc + ) + assert acc.calls == 3 + assert abs(acc.elapsed - 2.2) < 1e-9 + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verbose_rewrite_path(mock_call, capsys): + """verbose=True prints 'rewriting', 'verifying rewrite', and 'rewrote'.""" + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(_REWRITE_VERIFY_OK), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "rewriting" in err + assert "verifying rewrite" in err + assert "rewrote" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verbose_rewrite_verify_rejected(mock_call, capsys): + """verbose=True prints 'REJECTED' and issue when rewrite verify fails.""" + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(_REWRITE_VERIFY_REJECT), + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(_REWRITE_VERIFY_OK), + ] + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + _CFG, + 2, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "REJECTED" in err + assert "wrong mock setup" in err + assert "ACCEPTED" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_verbose_rewrite_compile_retry(mock_call, capsys): + """verbose=True prints '(retry)' when rewrite compile fails.""" + mock_call.side_effect = [ + _ok(_CLASSIFY_REWRITE), + _ok({"rewritten_function": "def f(:\n pass\n"}), # invalid + _ok({"rewritten_function": _VALID_REWRITE}), + _ok(_REWRITE_VERIFY_OK), + ] + cfg = CrispenConfig(patch_update_retries=1, timing="detailed") + _process_file_source( + _SRC_WITH_PATCH, + _FORKING_PATHS, + "ctx", + MagicMock(), + cfg, + 2, + scan_file="tests/test_foo.py", + verbose=True, + ) + err = capsys.readouterr().err + assert "rewriting" in err + assert "(retry)" in err + + +@mock_patch(_PATCH_CALL_TOOL) +def test_process_rewrite_restores_unchanged_const_ref(mock_call): + """After full rewrite, @patch("value") left unchanged → reverted to @patch(NAME).""" + src = ( + 'STABLE = "pkg.stable.X"\n' + 'TARGET = "pkg.big.A"\n\n' + "@patch(STABLE)\n" + "@patch(TARGET)\n" + "def test_f(mock_stable, mock_target):\n" + " pass\n" + ) + # LLM updates TARGET but leaves STABLE's substituted literal unchanged. + rewritten = ( + '@patch("pkg.stable.X")\n' + '@patch("pkg.sub_a.A")\n' + "def test_f(mock_stable, mock_target):\n" + " pass\n" + ) + mock_call.side_effect = [ + _ok({"needs_rewrite": True}), + _ok({"rewritten_function": rewritten}), + _ok({"correct": True, "issue": ""}), + ] + result, changed, _ = _process_file_source( + src, + {"pkg.big.A"}, + "ctx", + MagicMock(), + _CFG, + 1, + scan_file="tests/test_foo.py", + ) + assert changed is True + # STABLE decorator reverted to named constant form. + assert "@patch(STABLE)" in result + assert '@patch("pkg.stable.X")' not in result + # TARGET decorator keeps the LLM's updated literal value. + assert '@patch("pkg.sub_a.A")' in result + + +@mock_patch(_PATCH_CALL_PR) +@mock_patch(helpers._PATCH_MAKE_CLIENT) +@mock_patch(_PATCH_GET_KEY_PR, return_value="key") +def test_process_file_source_rewrite_candidates_reject_and_retry( + mock_key, mock_client, mock_call +): + # Rewrite returns old path still present → rewrite candidates check rejects + # without calling verify → retry; second rewrite uses valid candidate → accepted. + src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n complex_logic()\n' + ctx = _FLContext( + filepath="/repo/pkg/big.py", + old_module="pkg.big", + original_source="from external import A\ndef f(): A()\n", + modified_source="from .sub_a import f\n", + new_files={"sub_a.py": "from external import A\ndef f(): A()\n"}, + new_module_paths={"sub_a.py": "pkg.sub_a"}, + entity_to_target={"f": "sub_a.py"}, + forking_old_paths={"pkg.big.A"}, + ) + context_msg = _build_context_message([ctx]) + bad_rewrite = '@patch("pkg.big.A")\ndef test_f(mock_a):\n complex_logic()\n' + good_rewrite = '@patch("pkg.sub_a.A")\ndef test_f(mock_a):\n complex_logic()\n' + mock_call.side_effect = [ + # classify → needs rewrite + LLMCallResult( + tool_input={"needs_rewrite": True}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # rewrite 1: old path still present → rejected by _rewrite_candidates_check + LLMCallResult( + tool_input={"rewritten_function": bad_rewrite}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # rewrite 2: valid candidate + LLMCallResult( + tool_input={"rewritten_function": good_rewrite}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # verify + LLMCallResult( + tool_input={"correct": True, "issue": ""}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + ] + cg_candidates = {"test_f": {"pkg.big.A": ["pkg.sub_a.A"]}} + new_src, changed, _ = _process_file_source( + src, + {"pkg.big.A"}, + context_msg, + mock_client.return_value, + _make_process_cfg(), + max_attempts=2, + cg_candidates=cg_candidates, + ) + assert changed + assert "pkg.sub_a.A" in new_src + assert mock_call.call_count == 4 # classify + bad_rw + good_rw + verify + + +@mock_patch(_PATCH_CALL_PR) +@mock_patch(helpers._PATCH_MAKE_CLIENT) +@mock_patch(_PATCH_GET_KEY_PR, return_value="key") +def test_process_file_source_candidates_all_retries_escalates_to_rewrite( + mock_key, mock_client, mock_call, capsys +): + # All classify retries exhausted with persistent candidates check rejections → + # escalate to full rewrite rather than silently leaving the test broken. + src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' + ctx = _FLContext( + filepath="/repo/pkg/big.py", + old_module="pkg.big", + original_source="from external import A\ndef f(): A()\n", + modified_source="from .sub_a import f\n", + new_files={"sub_a.py": "from external import A\ndef f(): A()\n"}, + new_module_paths={"sub_a.py": "pkg.sub_a"}, + entity_to_target={"f": "sub_a.py"}, + forking_old_paths={"pkg.big.A"}, + ) + context_msg = _build_context_message([ctx]) + good_rewrite = '@patch("pkg.sub_a.A")\ndef test_f(mock_a):\n pass\n' + mock_call.side_effect = [ + # First classify: no rename → rejected by candidates check (not last attempt). + LLMCallResult( + tool_input={"needs_rewrite": False, "patch_renames": {}}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # Second classify: still no rename → last attempt → escalate to rewrite. + LLMCallResult( + tool_input={"needs_rewrite": False, "patch_renames": {}}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # Rewrite (escalated from candidates check failure): + LLMCallResult( + tool_input={"rewritten_function": good_rewrite}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + # Rewrite verify: + LLMCallResult( + tool_input={"correct": True, "issue": ""}, + elapsed=0.1, + input_tokens=10, + output_tokens=5, + ), + ] + # Two candidates → ambiguous → LLM keeps returning no_change. + cg_candidates = {"test_f": {"pkg.big.A": ["pkg.sub_a.A", "pkg.sub_b.A"]}} + new_src, changed, _ = _process_file_source( + src, + {"pkg.big.A"}, + context_msg, + mock_client.return_value, + _make_process_cfg(), + max_attempts=2, + cg_candidates=cg_candidates, + verbose=True, + ) + assert changed + assert "pkg.sub_a.A" in new_src + assert mock_call.call_count == 4 # classify x2 + rewrite + verify + err = capsys.readouterr().err + assert "candidates check retries exhausted" in err diff --git a/tests/patch_rewriter/test_rename_guard.py b/tests/patch_rewriter/test_rename_guard.py new file mode 100644 index 0000000..a6c46e3 --- /dev/null +++ b/tests/patch_rewriter/test_rename_guard.py @@ -0,0 +1,259 @@ +from __future__ import annotations +from crispen.patch_rewriter import _build_rename_guard_sets, _is_bad_rename +from .helpers import _make_fl_ctx + + +def test_build_rename_guard_sets_moved_out(): + # call_with_tool is in original_source but removed from modified_source. + ctx = _make_fl_ctx( + original_source="from ...llm_client import call_with_tool\ndef f(): pass\n", + modified_source="from .sub import call_with_tool\n", + new_files={"sub.py": "from ...llm_client import call_with_tool\n"}, + ) + moved_out, still_in, orig_users, new_mod_imports = _build_rename_guard_sets([ctx]) + assert "call_with_tool" in moved_out + assert "call_with_tool" not in still_in + + +def test_build_rename_guard_sets_still_imported(): + # make_client stays in modified_source as an external import. + ctx = _make_fl_ctx( + original_source=( + "from ...llm_client import make_client, call_with_tool\n" + "def advise(): make_client()\n" + ), + modified_source=( + "from ...llm_client import make_client\ndef advise(): make_client()\n" + ), + new_files={"sub.py": "from ...llm_client import call_with_tool\n"}, + ) + moved_out, still_in, orig_users, new_mod_imports = _build_rename_guard_sets([ctx]) + assert "make_client" in still_in + assert "call_with_tool" in moved_out + assert "make_client" not in moved_out + + +def test_build_rename_guard_sets_orig_users_map(): + # make_client is still imported and used by advise in modified_source. + ctx = _make_fl_ctx( + original_source="from ...llm_client import make_client\ndef advise(): pass\n", + modified_source=( + "from ...llm_client import make_client\ndef advise(): make_client()\n" + ), + new_files={}, + ) + _, _, orig_users, *_ = _build_rename_guard_sets([ctx]) + assert orig_users.get("make_client") == ["advise"] + + +def test_build_rename_guard_sets_no_users_not_in_map(): + # make_client is still imported but not referenced by any top-level def. + ctx = _make_fl_ctx( + original_source="from ...llm_client import make_client\ndef advise(): pass\n", + modified_source="from ...llm_client import make_client\ndef advise(): pass\n", + new_files={}, + ) + _, _, orig_users, *_ = _build_rename_guard_sets([ctx]) + assert "make_client" not in orig_users + + +def test_build_rename_guard_sets_empty_contexts(): + moved_out, still_in, orig_users, new_mod_imports = _build_rename_guard_sets([]) + assert moved_out == set() + assert still_in == set() + assert orig_users == {} + assert new_mod_imports == {} + + +def test_build_rename_guard_sets_merges_multiple_contexts(): + # Two contexts each contributing one still-in name with users. + ctx1 = _make_fl_ctx( + original_source="from ...a import foo\ndef f1(): foo()\n", + modified_source="from ...a import foo\ndef f1(): foo()\n", + new_files={}, + ) + ctx2 = _make_fl_ctx( + original_source="from ...b import bar\ndef f2(): bar()\n", + modified_source="from ...b import bar\ndef f2(): bar()\n", + new_files={}, + ) + _, still_in, orig_users, *_ = _build_rename_guard_sets([ctx1, ctx2]) + assert "foo" in still_in + assert "bar" in still_in + assert orig_users["foo"] == ["f1"] + assert orig_users["bar"] == ["f2"] + + +def test_build_rename_guard_sets_deduplicates_merged_users(): + # Same name+user in two contexts → appears once in orig_users_map. + ctx1 = _make_fl_ctx( + original_source="from ...a import foo\ndef f1(): foo()\n", + modified_source="from ...a import foo\ndef f1(): foo()\n", + new_files={}, + ) + ctx2 = _make_fl_ctx( + original_source="from ...a import foo\ndef f1(): foo()\n", + modified_source="from ...a import foo\ndef f1(): foo()\n", + new_files={}, + ) + _, _, orig_users, *_ = _build_rename_guard_sets([ctx1, ctx2]) + assert orig_users["foo"].count("f1") == 1 + + +def test_is_bad_rename_pattern_a_shallowing_moved_out(): + # advisor.placement.call_with_tool → advisor.call_with_tool + # call_with_tool is moved out; new_depth < old_depth → bad + assert _is_bad_rename( + "crispen.advisor.placement.call_with_tool", + "crispen.advisor.call_with_tool", + moved_out_names={"call_with_tool"}, + still_imported=set(), + orig_users_map={}, + test_text="", + ) + + +def test_is_bad_rename_pattern_a_deepening_moved_out_ok(): + # Deepening a moved-out name is fine (not shallowing). + assert not _is_bad_rename( + "crispen.advisor.call_with_tool", + "crispen.advisor.placement.call_with_tool", + moved_out_names={"call_with_tool"}, + still_imported=set(), + orig_users_map={}, + test_text="", + ) + + +def test_is_bad_rename_pattern_b_deepening_still_in_with_orig_user_in_test(): + # advisor.make_client → advisor.placement.make_client + # make_client is still_imported, orig user advise_file_limiter is in test body → bad + assert _is_bad_rename( + "crispen.advisor.make_client", + "crispen.advisor.placement.make_client", + moved_out_names=set(), + still_imported={"make_client"}, + orig_users_map={"make_client": ["advise_file_limiter"]}, + test_text="def test_foo():\n advise_file_limiter(src)\n", + ) + + +def test_is_bad_rename_pattern_b_deepening_still_in_no_orig_user_in_test(): + # Same deepening but test body doesn't contain advise_file_limiter → ok + assert not _is_bad_rename( + "crispen.advisor.make_client", + "crispen.advisor.placement.make_client", + moved_out_names=set(), + still_imported={"make_client"}, + orig_users_map={"make_client": ["advise_file_limiter"]}, + test_text="def test_foo():\n _propose_files_step(src)\n", + ) + + +def test_is_bad_rename_pattern_b_deepening_no_orig_users_map(): + # Name is still_imported but not in orig_users_map → not blocked + assert not _is_bad_rename( + "crispen.advisor.make_client", + "crispen.advisor.placement.make_client", + moved_out_names=set(), + still_imported={"make_client"}, + orig_users_map={}, + test_text="def test_foo():\n advise_file_limiter(src)\n", + ) + + +def test_is_bad_rename_not_bad_when_no_relevant_sets(): + assert not _is_bad_rename( + "a.b.foo", + "a.b.c.foo", + moved_out_names=set(), + still_imported=set(), + orig_users_map={}, + test_text="", + ) + + +def test_is_bad_rename_pattern_c_target_module_missing_name(): + # Target module "pkg.advisor.placement" exists in new_module_imports + # but doesn't import call_with_tool; name is in moved_out_names → bad rename. + assert _is_bad_rename( + "pkg.advisor.call_with_tool", + "pkg.advisor.placement.call_with_tool", + moved_out_names={"call_with_tool"}, + still_imported=set(), + orig_users_map={}, + test_text="", + new_module_imports={"pkg.advisor.placement": {"make_client"}}, + ) + + +def test_is_bad_rename_pattern_c_target_module_has_name(): + # Target module imports the name → not blocked by Pattern C. + assert not _is_bad_rename( + "pkg.advisor.call_with_tool", + "pkg.advisor.placement.call_with_tool", + moved_out_names={"call_with_tool"}, + still_imported=set(), + orig_users_map={}, + test_text="", + new_module_imports={"pkg.advisor.placement": {"call_with_tool"}}, + ) + + +def test_is_bad_rename_pattern_c_target_module_unknown(): + # Target module not in new_module_imports (unknown module) → not blocked. + assert not _is_bad_rename( + "pkg.advisor.call_with_tool", + "pkg.advisor.placement.call_with_tool", + moved_out_names={"call_with_tool"}, + still_imported=set(), + orig_users_map={}, + test_text="", + new_module_imports={"pkg.advisor.schemas": {"call_with_tool"}}, + ) + + +def test_is_bad_rename_pattern_c_name_not_tracked(): + # Name is not in moved_out_names or still_imported → Pattern C skipped + # even if the target module doesn't import it (locally-defined symbols). + assert not _is_bad_rename( + "pkg.big.A", + "pkg.sub_a.A", + moved_out_names=set(), + still_imported=set(), + orig_users_map={}, + test_text="", + new_module_imports={"pkg.sub_a": set()}, + ) + + +def test_is_bad_rename_pattern_c_none_new_module_imports(): + # new_module_imports=None (not passed) → Pattern C skipped entirely. + assert not _is_bad_rename( + "pkg.advisor.call_with_tool", + "pkg.advisor.placement.call_with_tool", + moved_out_names={"call_with_tool"}, + still_imported=set(), + orig_users_map={}, + test_text="", + new_module_imports=None, + ) + + +def test_build_rename_guard_sets_new_module_imports(): + # new_files with known module paths populate new_module_imports correctly. + ctx = _make_fl_ctx( + original_source="from ...llm_client import call_with_tool, make_client\n", + modified_source="from .placement import call_with_tool\n", + new_files={ + "placement.py": "from ...llm_client import call_with_tool\n", + "schemas.py": "from ...llm_client import make_client\n", + }, + new_module_paths={ + "placement.py": "pkg.advisor.placement", + "schemas.py": "pkg.advisor.schemas", + }, + ) + _, _, _, new_mod_imports = _build_rename_guard_sets([ctx]) + assert new_mod_imports["pkg.advisor.placement"] == {"call_with_tool"} + assert new_mod_imports["pkg.advisor.schemas"] == {"make_client"} diff --git a/tests/runner/__init__.py b/tests/runner/__init__.py new file mode 100644 index 0000000..6d9be1d --- /dev/null +++ b/tests/runner/__init__.py @@ -0,0 +1 @@ +"""Tests for file_limiter.runner — 100% branch coverage.""" diff --git a/tests/runner/test_preservation.py b/tests/runner/test_preservation.py new file mode 100644 index 0000000..20af456 --- /dev/null +++ b/tests/runner/test_preservation.py @@ -0,0 +1,379 @@ +from __future__ import annotations +from crispen.config import CrispenConfig +from crispen.file_limiter.advisor import GroupPlacement +from crispen.file_limiter.classifier import ClassifiedEntities +from crispen.file_limiter.code_gen import SplitResult +from crispen.file_limiter.entity_parser import Entity, EntityKind +from crispen.file_limiter.runner import _strip_imports_by_line, _verify_preservation + + +_CONFIG = CrispenConfig() +# Zero-retry config for tests that exercise a single-attempt failure path. +_CONFIG_NO_RETRY = CrispenConfig(file_limiter_retries=0) +_PATCH_CLASSIFY = "crispen.file_limiter.runner.classify_entities" +_PATCH_ADVISE = "crispen.file_limiter.runner.advise_file_limiter" +_PATCH_GEN = "crispen.file_limiter.runner.generate_file_splits" +_PATCH_RESOLVE = "crispen.file_limiter.runner.resolve_naming_conflicts" + + +def _make_entity(name: str, start: int, end: int) -> Entity: + return Entity(EntityKind.FUNCTION, name, start, end, [name]) + + +def _make_classified(entities=None) -> ClassifiedEntities: + return ClassifiedEntities( + entities=entities or [], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[], + set_3_groups=[], + abort=False, + ) + + +def _good_split(entity_name: str = "foo", target: str = "utils.py") -> SplitResult: + return SplitResult( + new_files={target: f"def {entity_name}():\n pass"}, + original_source="# original updated\n", + abort=False, + ) + + +def test_verify_entity_source_in_original(): + # Entity that stayed in the original file — passes verification but is not + # counted (it wasn't a FileLimiter edit). + post_source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + split = SplitResult( + new_files={}, + original_source="def foo():\n pass\n", + abort=False, + ) + vr = _verify_preservation([entity], split, post_source, []) + assert vr.failures == [] + assert vr.verified_functions == 0 + assert vr.verified_lines == 0 + + +def test_verify_entity_source_in_new_file(): + # Entity that was migrated — passes verification and is counted. + post_source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + split = SplitResult( + new_files={"utils.py": "def foo():\n pass"}, + original_source="# original\n", + abort=False, + ) + placements = [GroupPlacement(group=["foo"], target_file="utils.py")] + vr = _verify_preservation([entity], split, post_source, placements) + assert vr.failures == [] + assert vr.verified_functions == 1 + assert vr.verified_lines == 2 # "def foo():\n pass" → 2 lines matched + + +def test_verify_entity_source_missing(): + post_source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + split = SplitResult( + new_files={}, + original_source="# nothing relevant\n", + abort=False, + ) + vr = _verify_preservation([entity], split, post_source, []) + assert len(vr.failures) == 1 + assert "'foo'" in vr.failures[0] + assert "1" in vr.failures[0] # start line + assert "2" in vr.failures[0] # end line + assert vr.verified_lines == 0 + + +def test_verify_entity_source_missing_long(): + # Entity with more than 3 lines → preview includes trailing "..." + post_source = "def foo():\n a = 1\n b = 2\n c = 3\n pass\n" + entity = _make_entity("foo", 1, 5) + split = SplitResult( + new_files={}, + original_source="# nothing relevant\n", + abort=False, + ) + vr = _verify_preservation([entity], split, post_source, []) + assert len(vr.failures) == 1 + assert "..." in vr.failures[0] + + +def test_verify_empty_entity_source_skipped(): + # Entity spanning only a blank line → rstrip → "" → falsy → skipped. + post_source = "\n" + entity = _make_entity("_block_1", 1, 1) + split = SplitResult( + new_files={}, + original_source="# completely different", + abort=False, + ) + vr = _verify_preservation([entity], split, post_source, []) + assert vr.failures == [] + assert vr.verified_lines == 0 + + +def test_verify_top_level_entity_skipped(): + # TOP_LEVEL entities (import/docstring blocks) are always skipped — + # they are intentionally restructured when the file is split. + post_source = "from __future__ import annotations\nimport os\n" + entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 2, ["annotations", "os"]) + split = SplitResult( + new_files={}, + original_source="# completely different", + abort=False, + ) + vr = _verify_preservation([entity], split, post_source, []) + assert vr.failures == [] + assert vr.verified_lines == 0 + + +def test_verify_annotation_migrated(): + # Failure for an entity that was in the plan → annotated "migrated → target". + post_source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + split = SplitResult( + new_files={"utils.py": "# empty"}, + original_source="# empty", + abort=False, + ) + placements = [GroupPlacement(group=["foo"], target_file="utils.py")] + vr = _verify_preservation([entity], split, post_source, placements) + assert len(vr.failures) == 1 + assert "migrated" in vr.failures[0] + assert "utils.py" in vr.failures[0] + + +def test_verify_annotation_stayed(): + # Failure for an entity not in any placement → annotated "stayed in original". + post_source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + split = SplitResult( + new_files={}, + original_source="# empty", + abort=False, + ) + vr = _verify_preservation([entity], split, post_source, []) + assert len(vr.failures) == 1 + assert "stayed in original" in vr.failures[0] + + +def test_verify_pruned_inline_import_passes(): + # Entity has an inline import; the new file has it pruned to a top-level one. + # Both sides are stripped before comparison, so the match succeeds. + # verified_lines counts only the non-import lines of the migrated entity. + post_source = "def foo():\n import os\n return os.getcwd()\n" + entity = _make_entity("foo", 1, 3) + split = SplitResult( + new_files={"utils.py": "import os\n\ndef foo():\n return os.getcwd()"}, + original_source="# original\n", + abort=False, + ) + placements = [GroupPlacement(group=["foo"], target_file="utils.py")] + vr = _verify_preservation([entity], split, post_source, placements) + assert vr.failures == [] + # "def foo():\n return os.getcwd()" → 2 lines (import stripped) + assert vr.verified_lines == 2 + + +def test_verify_inline_import_not_pruned_also_passes(): + # Import was NOT pruned — it appears on both sides. Stripping both sides + # still produces a match. + post_source = "def foo():\n import os\n return os.getcwd()\n" + entity = _make_entity("foo", 1, 3) + split = SplitResult( + new_files={"utils.py": "def foo():\n import os\n return os.getcwd()"}, + original_source="# original\n", + abort=False, + ) + placements = [GroupPlacement(group=["foo"], target_file="utils.py")] + vr = _verify_preservation([entity], split, post_source, placements) + assert vr.failures == [] + assert vr.verified_lines == 2 + + +def test_verify_multiline_import_stripped(): + # Multi-line imports are removed correctly using AST line spans. + post_source = ( + "def foo():\n" + " from os import (\n" + " path,\n" + " getcwd,\n" + " )\n" + " return getcwd()\n" + ) + entity = _make_entity("foo", 1, 6) + # New file has the multi-line import removed (3 lines gone). + split = SplitResult( + new_files={ + "utils.py": "from os import path, getcwd\n\ndef foo():\n return getcwd()" + }, + original_source="# original\n", + abort=False, + ) + placements = [GroupPlacement(group=["foo"], target_file="utils.py")] + vr = _verify_preservation([entity], split, post_source, placements) + assert vr.failures == [] + # "def foo():\n return getcwd()" → 2 lines (4-line import stripped) + assert vr.verified_lines == 2 + + +def test_verify_async_def_entity_passes(): + # Async functions are found after import stripping (no imports involved). + post_source = "async def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + split = SplitResult( + new_files={"utils.py": "async def foo():\n pass"}, + original_source="# original\n", + abort=False, + ) + placements = [GroupPlacement(group=["foo"], target_file="utils.py")] + vr = _verify_preservation([entity], split, post_source, placements) + assert vr.failures == [] + assert vr.verified_lines == 2 + + +def test_verify_blank_line_collapse_after_pruning_passes(): + # Regression: when multiple consecutive inline imports are all pruned to + # top-level, the resulting consecutive blank lines (3+ newlines before + # indented content) are collapsed by _normalize_blank_lines in code_gen. + # Verification must apply the same normalization to entity_no_imports so + # the substring match doesn't fail due to a blank-line count mismatch. + post_source = ( + "def test_seq():\n" + ' """Docstring."""\n' + " import libcst as cst\n" + "\n" + " from libcst.metadata import MetadataWrapper\n" + "\n" + " from foo import Bar\n" + "\n" + " x = cst.parse_module('')\n" + " w = MetadataWrapper(x)\n" + " b = Bar()\n" + ) + entity = _make_entity("test_seq", 1, 13) + # New file has all 3 inline imports hoisted to top-level and pruned from + # the function body; _normalize_blank_lines collapsed the 3+ consecutive + # blank lines down to 1. + new_file_src = ( + "import libcst as cst\n" + "from libcst.metadata import MetadataWrapper\n" + "from foo import Bar\n" + "\n" + "def test_seq():\n" + ' """Docstring."""\n' + "\n" + " x = cst.parse_module('')\n" + " w = MetadataWrapper(x)\n" + " b = Bar()\n" + ) + split = SplitResult( + new_files={"test_collectors.py": new_file_src}, + original_source="# original\n", + abort=False, + ) + placements = [GroupPlacement(group=["test_seq"], target_file="test_collectors.py")] + vr = _verify_preservation([entity], split, post_source, placements) + assert vr.failures == [] + assert vr.verified_functions == 1 + + +def test_verify_inline_import_not_pruned_with_surrounding_blanks_passes(): + # Regression: entity has an inline import surrounded by blank lines that is + # NOT pruned to a top-level import in the new file. After + # _strip_imports_by_line removes the import from the new file's content, the + # two surrounding blank lines merge into 3+ consecutive newlines before + # indented code — which _normalize_blank_lines in the new file did NOT + # collapse (it only runs before the import was stripped in verification). + # Verification must apply the same _EXCESS_BLANK_BODY_RE normalization to + # combined_no_imports so the blank-line count matches entity_no_imports. + post_source = ( + "def test_foo():\n" + " x = 1\n" + "\n" + " import pathlib\n" + "\n" + " y = pathlib.Path('.')\n" + " return y\n" + ) + entity = _make_entity("test_foo", 1, 8) + # New file keeps the inline import (not pruned — no module-level pathlib). + new_file_src = ( + "def test_foo():\n" + " x = 1\n" + "\n" + " import pathlib\n" + "\n" + " y = pathlib.Path('.')\n" + " return y\n" + ) + split = SplitResult( + new_files={"test_patch.py": new_file_src}, + original_source="# original\n", + abort=False, + ) + placements = [GroupPlacement(group=["test_foo"], target_file="test_patch.py")] + vr = _verify_preservation([entity], split, post_source, placements) + assert vr.failures == [] + assert vr.verified_functions == 1 + + +def test_verify_entity_with_name_rewrites_passes(): + # The original entity references SAFE_MODE; after splitting it becomes + # conversion.SAFE_MODE in the new file. Verification must apply the + # name_rewrites before the substring check so it passes rather than + # reporting a false failure. + post_source = ( + "def create_runtime(safe_mode=None):\n" + " if safe_mode is None:\n" + " safe_mode = SAFE_MODE\n" + ) + entity = _make_entity("create_runtime", 1, 3) + new_file_src = ( + "def create_runtime(safe_mode=None):\n" + " if safe_mode is None:\n" + " safe_mode = conversion.SAFE_MODE\n" + ) + split = SplitResult( + new_files={"runtime.py": new_file_src}, + original_source="# re-exports\n", + abort=False, + entity_name_rewrites={"create_runtime": {"SAFE_MODE": "conversion.SAFE_MODE"}}, + ) + placements = [GroupPlacement(group=["create_runtime"], target_file="runtime.py")] + vr = _verify_preservation([entity], split, post_source, placements) + assert vr.failures == [] + # The function passes verification. Only the 1 rewritten line is excluded; + # the other 2 unchanged lines are credited. + assert vr.verified_functions == 1 + assert vr.verified_lines == 2 + + +def test_strip_imports_no_imports(): + src = "def foo():\n return 1\n" + assert _strip_imports_by_line(src) == src + + +def test_strip_imports_single_line(): + src = "import os\nx = 1\n" + assert _strip_imports_by_line(src) == "x = 1\n" + + +def test_strip_imports_multiline(): + src = "from os import (\n path,\n getcwd,\n)\nx = 1\n" + assert _strip_imports_by_line(src) == "x = 1\n" + + +def test_strip_imports_inner_import(): + # Imports inside a function body are also stripped. + src = "def foo():\n import os\n return os.getcwd()\n" + assert _strip_imports_by_line(src) == "def foo():\n return os.getcwd()\n" + + +def test_strip_imports_syntax_error_returns_unchanged(): + src = "def foo(:\n pass\n" + assert _strip_imports_by_line(src) == src diff --git a/tests/runner/test_runner_advanced.py b/tests/runner/test_runner_advanced.py new file mode 100644 index 0000000..8902cac --- /dev/null +++ b/tests/runner/test_runner_advanced.py @@ -0,0 +1,930 @@ +from __future__ import annotations +from unittest.mock import patch +from crispen.config import CrispenConfig +from crispen.file_limiter.advisor import FileLimiterPlan, GroupPlacement +from crispen.file_limiter.classifier import ClassifiedEntities +from crispen.file_limiter.code_gen import SplitResult +from crispen.file_limiter.runner import ( + _MAIN_SUBDIR_SUFFIXES, + _detect_naming_conflicts, + _has_main_block, + run_file_limiter, +) +from .test_preservation import ( + _CONFIG, + _CONFIG_NO_RETRY, + _PATCH_ADVISE, + _PATCH_CLASSIFY, + _PATCH_GEN, + _PATCH_RESOLVE, + _good_split, + _make_classified, + _make_entity, +) +from .test_runner_core import _plan_with + + +# A two-line source whose diff_ranges covers the whole file, triggering subdir +# split for "big.py" → subdir_name="big". Path("big") must not exist on disk. +_SUBDIR_SRC = "x = 1\ny = 2\n" +_SUBDIR_RANGES = [(1, 2)] + + +def _plan_two_same_target() -> FileLimiterPlan: + """Two groups, both assigned to the same target file.""" + return FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="utils.py"), + ], + abort=False, + ) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_all_in_one_file_subdir_retries_and_fails( + mock_classify, mock_advise, mock_gen +): + # Subdir split + all groups → same file → guard triggers every attempt. + # Two groups required so the n_groups > 1 pre-loop check doesn't fire first. + mock_classify.return_value = ClassifiedEntities( + entities=[_make_entity("foo", 1, 1), _make_entity("bar", 2, 2)], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[["foo"], ["bar"]], + set_3_groups=[], + abort=False, + ) + mock_advise.return_value = _plan_two_same_target() + cfg = CrispenConfig(file_limiter_retries=0) + + result = run_file_limiter("big.py", "", _SUBDIR_SRC, _SUBDIR_RANGES, cfg) + + assert result.abort is False + assert any("single file" in m for m in result.messages) + mock_gen.assert_not_called() + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_all_in_one_file_subdir_retries_and_succeeds( + mock_classify, mock_advise, mock_gen +): + # Subdir split: first attempt all in one file, second splits into two. + entity1 = _make_entity("foo", 1, 1) + entity2 = _make_entity("bar", 2, 2) + # Two groups required so the n_groups > 1 pre-loop check doesn't fire first. + mock_classify.return_value = ClassifiedEntities( + entities=[entity1, entity2], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[["foo"], ["bar"]], + set_3_groups=[], + abort=False, + ) + mock_advise.side_effect = [ + _plan_two_same_target(), + FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="helpers.py"), + ], + abort=False, + ), + ] + mock_gen.return_value = SplitResult( + new_files={ + "big/utils.py": "x = 1", + "big/helpers.py": "y = 2", + }, + original_source=_SUBDIR_SRC, + abort=False, + ) + cfg = CrispenConfig(file_limiter_retries=1) + + result = run_file_limiter("big.py", "", _SUBDIR_SRC, _SUBDIR_RANGES, cfg) + + assert result.abort is False + assert any("single file" in m for m in result.messages) + assert any("FileLimiter: moved" in m for m in result.messages) + assert mock_advise.call_args_list[1].kwargs["prev_placement_failure"] != "" + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_all_in_one_file_non_subdir_allowed( + mock_classify, mock_advise, mock_gen +): + # Non-subdir split: all groups → same file is always fine. + entity1 = _make_entity("foo", 1, 2) + entity2 = _make_entity("bar", 3, 4) + mock_classify.return_value = _make_classified(entities=[entity1, entity2]) + mock_advise.return_value = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="utils.py"), + ], + abort=False, + ) + mock_gen.return_value = SplitResult( + new_files={"utils.py": "def foo():\n pass\ndef bar():\n pass"}, + original_source="# reduced\n", + abort=False, + ) + + # diff_ranges=[] → not a whole-file diff → subdir_name=None → guard inactive. + result = run_file_limiter( + "big.py", + "", + "def foo():\n pass\ndef bar():\n pass\n", + [], + _CONFIG_NO_RETRY, + ) + + assert result.abort is False + assert not any("single file" in m for m in result.messages) + mock_gen.assert_called_once() + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_single_group_subdir_aborts_silently( + mock_classify, mock_advise, mock_gen +): + # Subdir split with only 1 group: moving it would just rename the file, + # not split it, causing infinite subdirectory nesting across runs. + # Abort immediately without calling the LLM. + entity = _make_entity("foo", 1, 1) + mock_classify.return_value = ClassifiedEntities( + entities=[entity], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[["foo"]], + set_3_groups=[], + abort=False, + ) + cfg = CrispenConfig(file_limiter_retries=0) + + result = run_file_limiter("big.py", "", _SUBDIR_SRC, _SUBDIR_RANGES, cfg) + + assert result.abort is True + assert result.messages == [] + mock_advise.assert_not_called() + mock_gen.assert_not_called() + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_adds_test_prefix_to_new_files(mock_classify, mock_advise, mock_gen): + # When the source file is test_*.py, target files in the same directory + # must also have the test_ prefix so pytest can discover the moved tests. + source = "def test_foo():\n pass\n" + entity = _make_entity("test_foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["test_foo"], "helpers.py") + mock_gen.return_value = SplitResult( + new_files={"test_helpers.py": "def test_foo():\n pass"}, + original_source="# original\n", + abort=False, + ) + + result = run_file_limiter("tests/test_big.py", "", source, [], _CONFIG) + + assert result.abort is False + # The placement target passed to generate_file_splits must have been + # normalised — verify via the success message. + assert any("test_helpers.py" in m for m in result.messages) + assert not any( + "helpers.py" in m and "test_helpers.py" not in m for m in result.messages + ) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_test_prefix_already_present(mock_classify, mock_advise, mock_gen): + # Target file already starts with test_ → name is left unchanged. + source = "def test_foo():\n pass\n" + entity = _make_entity("test_foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["test_foo"], "test_helpers.py") + mock_gen.return_value = SplitResult( + new_files={"test_helpers.py": "def test_foo():\n pass"}, + original_source="# original\n", + abort=False, + ) + + result = run_file_limiter("tests/test_big.py", "", source, [], _CONFIG) + + assert result.abort is False + assert any("test_helpers.py" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_no_test_prefix_for_helper_only_group( + mock_classify, mock_advise, mock_gen +): + # Source is test_*.py but the group contains only helper functions (no + # test_/Test* names) — the target file must NOT get a test_ prefix so + # pytest does not try to collect it. + source = "def _helper():\n pass\n" + entity = _make_entity("_helper", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["_helper"], "helpers.py") + mock_gen.return_value = SplitResult( + new_files={"helpers.py": "def _helper():\n pass"}, + original_source="# original\n", + abort=False, + ) + + result = run_file_limiter("tests/test_big.py", "", source, [], _CONFIG) + + assert result.abort is False + assert any("helpers.py" in m for m in result.messages) + assert not any("test_helpers.py" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_init_not_renamed_by_test_prefix_logic( + mock_classify, mock_advise, mock_gen +): + # Defence-in-depth: __init__.py placements must not get the test_ prefix. + source = "def test_foo():\n pass\n\ndef _setup():\n pass\n" + e1 = _make_entity("test_foo", 1, 2) + e2 = _make_entity("_setup", 4, 5) + mock_classify.return_value = _make_classified(entities=[e1, e2]) + mock_advise.return_value = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["test_foo"], target_file="cases.py"), + GroupPlacement(group=["_setup"], target_file="__init__.py"), + ], + abort=False, + ) + mock_gen.return_value = SplitResult( + new_files={ + "test_cases.py": "def test_foo():\n pass", + "__init__.py": "def _setup():\n pass", + }, + original_source="# original\n", + abort=False, + ) + + # tests/runner/ has no __init__.py so it won't appear in existing_files. + result = run_file_limiter("tests/runner/test_big.py", "", source, [], _CONFIG) + + assert result.abort is False + # cases.py → test_cases.py (has test_foo in group) + assert any("test_cases.py" in m for m in result.messages) + # __init__.py untouched + assert any("__init__.py" in m for m in result.messages) + assert not any("test___init__.py" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_no_test_prefix_for_non_test_file(mock_classify, mock_advise, mock_gen): + # Source file is NOT a test module — target file names are left as-is. + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["foo"], "helpers.py") + mock_gen.return_value = SplitResult( + new_files={"helpers.py": "def foo():\n pass"}, + original_source="# original\n", + abort=False, + ) + + result = run_file_limiter("big.py", "", source, [], _CONFIG) + + assert result.abort is False + assert any("helpers.py" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_strips_tests_suffix_and_adds_prefix( + mock_classify, mock_advise, mock_gen +): + # LLM returns a filename ending with _tests.py — strip the suffix and add + # the test_ prefix so pytest discovers the file. + source = "class TestFoo:\n def test_bar(self):\n pass\n" + entity = _make_entity("TestFoo", 1, 3) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["TestFoo"], "foo_tests.py") + mock_gen.return_value = SplitResult( + new_files={ + "test_foo.py": "class TestFoo:\n def test_bar(self):\n pass" + }, + original_source="# original\n", + abort=False, + ) + + result = run_file_limiter("tests/test_big.py", "", source, [], _CONFIG) + + assert result.abort is False + assert any("test_foo.py" in m for m in result.messages) + assert not any("foo_tests.py" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_adds_prefix_for_test_class_group(mock_classify, mock_advise, mock_gen): + # Group contains a Test-prefixed class (not test_ function) — must still + # get the test_ file prefix. + source = "class TestFoo:\n def test_bar(self):\n pass\n" + entity = _make_entity("TestFoo", 1, 3) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["TestFoo"], "foo_cases.py") + mock_gen.return_value = SplitResult( + new_files={ + "test_foo_cases.py": "class TestFoo:\n def test_bar(self):\n pass" + }, + original_source="# original\n", + abort=False, + ) + + result = run_file_limiter("tests/test_big.py", "", source, [], _CONFIG) + + assert result.abort is False + assert any("test_foo_cases.py" in m for m in result.messages) + assert not any( + "foo_cases.py" in m and "test_foo_cases.py" not in m for m in result.messages + ) + + +def test_detect_conflicts_no_conflicts(): + placements = [ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="helpers.py"), + ] + assert _detect_naming_conflicts(placements, frozenset(), frozenset()) == [] + + +def test_detect_conflicts_plan_vs_plan(): + # Plan contains both 'utils.py' and 'utils/io.py' → conflict on stem 'utils'. + placements = [ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="utils/io.py"), + ] + conflicts = _detect_naming_conflicts(placements, frozenset(), frozenset()) + assert len(conflicts) == 1 + assert "'utils.py'" in conflicts[0] + assert "'utils/'" in conflicts[0] + + +def test_detect_conflicts_plan_file_vs_existing_dir(): + # Plan proposes 'models.py' but 'models' directory already exists on disk. + placements = [GroupPlacement(group=["foo"], target_file="models.py")] + conflicts = _detect_naming_conflicts(placements, frozenset(), frozenset({"models"})) + assert len(conflicts) == 1 + assert "'models.py'" in conflicts[0] + assert "'models/'" in conflicts[0] + + +def test_detect_conflicts_plan_dir_vs_existing_file(): + # Plan proposes 'helpers/io.py' but 'helpers.py' already exists on disk. + placements = [GroupPlacement(group=["bar"], target_file="helpers/io.py")] + conflicts = _detect_naming_conflicts( + placements, frozenset({"helpers.py"}), frozenset() + ) + assert len(conflicts) == 1 + assert "'helpers/'" in conflicts[0] + assert "'helpers.py'" in conflicts[0] + + +def test_detect_conflicts_no_filesystem_conflict(): + # Proposed 'utils.py'; existing dir named 'other' — no overlap. + placements = [GroupPlacement(group=["foo"], target_file="utils.py")] + assert _detect_naming_conflicts(placements, frozenset(), frozenset({"other"})) == [] + + +def test_detect_conflicts_multiple_conflicts(): + # Three separate conflicts in one plan. + placements = [ + GroupPlacement(group=["a"], target_file="alpha.py"), # vs alpha/ dir on disk + GroupPlacement(group=["b"], target_file="beta/x.py"), # vs beta.py on disk + GroupPlacement(group=["c"], target_file="gamma.py"), # vs gamma/ in plan + GroupPlacement(group=["d"], target_file="gamma/y.py"), # vs gamma.py in plan + ] + conflicts = _detect_naming_conflicts( + placements, frozenset({"beta.py"}), frozenset({"alpha"}) + ) + assert len(conflicts) == 3 # alpha (disk dir), beta (disk file), gamma (plan) + + +def test_detect_conflicts_subdir_only_no_conflict(): + # All targets are in different subdirectories — no stem overlap. + placements = [ + GroupPlacement(group=["a"], target_file="pkg/models.py"), + GroupPlacement(group=["b"], target_file="pkg/helpers.py"), + ] + # Both land in 'pkg/' — that's fine; only 'pkg.py' vs 'pkg/' would conflict. + assert _detect_naming_conflicts(placements, frozenset(), frozenset()) == [] + + +def test_detect_conflicts_flat_target_in_existing_files(): + # Flat target whose filename is in existing_files (e.g. conftest.py) → conflict. + placements = [GroupPlacement(group=["fix"], target_file="conftest.py")] + conflicts = _detect_naming_conflicts( + placements, frozenset({"conftest.py"}), frozenset() + ) + assert len(conflicts) == 1 + assert "conftest.py" in conflicts[0] + + +@patch(_PATCH_GEN) +@patch(_PATCH_RESOLVE) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_naming_conflict_resolve_succeeds( + mock_classify, mock_advise, mock_resolve, mock_gen +): + # Conflict → resolve returns updated placements → generate called once, advise once. + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + conflicting_plan = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="utils/helpers.py"), # conflict! + ], + abort=False, + ) + resolved_placements = [GroupPlacement(group=["foo"], target_file="models.py")] + mock_advise.return_value = conflicting_plan + mock_resolve.return_value = resolved_placements + mock_gen.return_value = _good_split(entity_name="foo", target="models.py") + + result = run_file_limiter( + "big.py", "", "def foo():\n pass\n", [], _CONFIG_NO_RETRY + ) + + assert result.abort is False + assert mock_advise.call_count == 1 # no outer retry needed + assert mock_resolve.call_count == 1 + assert mock_gen.call_count == 1 + # No SKIP message — resolve handled the conflict without retrying advise. + assert not any("naming conflicts" in m for m in result.messages) + assert any("FileLimiter: moved" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_RESOLVE) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_naming_conflict_resolve_fails_then_outer_retry( + mock_classify, mock_advise, mock_resolve, mock_gen +): + # resolve returns None → outer retry → second advise succeeds. + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + conflicting_plan = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["foo"], target_file="pkg.py"), + GroupPlacement(group=["bar"], target_file="pkg/mod.py"), # conflict! + ], + abort=False, + ) + mock_advise.side_effect = [conflicting_plan, _plan_with(["foo"], "models.py")] + mock_resolve.return_value = None # targeted rename fails + mock_gen.return_value = _good_split(entity_name="foo", target="models.py") + cfg = CrispenConfig(file_limiter_retries=1) + + result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) + + assert result.abort is False + assert mock_advise.call_count == 2 + assert mock_resolve.call_count == 1 + assert any("naming conflicts" in m for m in result.messages) + assert any("FileLimiter: moved" in m for m in result.messages) + # Conflict description was forwarded as feedback for the second advise call. + prev_pf = mock_advise.call_args_list[1].kwargs["prev_placement_failure"] + assert "naming conflicts" in prev_pf + + +@patch(_PATCH_RESOLVE) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_naming_conflict_exhausts_all(mock_classify, mock_advise, mock_resolve): + # resolve always fails, all retries exhausted → abort=True, 2 SKIP messages. + mock_classify.return_value = _make_classified() + conflicting_plan = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["foo"], target_file="pkg.py"), + GroupPlacement(group=["bar"], target_file="pkg/mod.py"), # conflict! + ], + abort=False, + ) + mock_advise.return_value = conflicting_plan + mock_resolve.return_value = None # always fails + cfg = CrispenConfig(file_limiter_retries=1) + + result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) + + assert result.abort is True + assert mock_advise.call_count == 2 + assert mock_resolve.call_count == 2 + assert sum(1 for m in result.messages if "naming conflicts" in m) == 2 + + +def test_has_main_block_detects_dunder_main(): + src = "def foo():\n pass\n\nif __name__ == '__main__':\n foo()\n" + assert _has_main_block(src) is True + + +def test_has_main_block_no_main(): + assert _has_main_block("def foo():\n pass\n") is False + + +def test_has_main_block_syntax_error(): + assert _has_main_block("def (:\n") is False + + +@patch(_PATCH_CLASSIFY) +def test_runner_subdir_split_dir_exists_aborts(mock_classify, tmp_path): + mock_classify.return_value = _make_classified() + # Create a directory named 'service' alongside the source file. + service_dir = tmp_path / "service" + service_dir.mkdir() + filepath = str(tmp_path / "service.py") + + source = "def foo():\n pass\n" + # Whole-file diff: ranges cover all 2 lines. + cfg = CrispenConfig(file_limiter_subdir_split=True) + result = run_file_limiter(filepath, source, source, [(1, 2)], cfg) + + assert result.abort is True + assert result.new_files == {} + assert any("already exists" in m for m in result.messages) + assert any("service/" in m for m in result.messages) + + +@patch(_PATCH_CLASSIFY) +def test_runner_subdir_split_sibling_py_aborts(mock_classify, tmp_path): + mock_classify.return_value = _make_classified() + # Create a sibling 'service.py' alongside the source file — the intended + # subdirectory 'service/' would shadow it. + (tmp_path / "service.py").write_text("# helper\n") + filepath = str(tmp_path / "test_service.py") + + source = "def test_foo():\n pass\n" + # Whole-file diff: ranges cover all 2 lines. + cfg = CrispenConfig(file_limiter_subdir_split=True) + result = run_file_limiter(filepath, source, source, [(1, 2)], cfg) + + assert result.abort is True + assert result.new_files == {} + assert any("shadow" in m for m in result.messages) + assert any("service/" in m for m in result.messages) + + +@patch(_PATCH_CLASSIFY) +def test_runner_subdir_split_disabled(mock_classify, tmp_path): + # file_limiter_subdir_split=False — subdir detection is skipped entirely. + mock_classify.return_value = ClassifiedEntities( + entities=[], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[], + set_3_groups=[], + abort=True, # force early abort so advise is not called + ) + filepath = str(tmp_path / "service.py") + source = "def foo():\n pass\n" + cfg = CrispenConfig(file_limiter_subdir_split=False) + result = run_file_limiter(filepath, source, source, [(1, 2)], cfg) + + # abort comes from classifier, not from subdir detection + assert result.abort is True + assert "already exists" not in " ".join(result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_subdir_split_non_test_success(mock_classify, mock_advise, mock_gen): + # Whole-file diff on a non-test file → placements get subdir prefix, + # original_source is unchanged, and __init__.py carries the split content. + source = "def foo():\n pass\ndef bar():\n pass\n" + entity1 = _make_entity("foo", 1, 2) + entity2 = _make_entity("bar", 3, 4) + # Two groups required so the n_groups > 1 subdir guard doesn't fire. + mock_classify.return_value = ClassifiedEntities( + entities=[entity1, entity2], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[["foo"], ["bar"]], + set_3_groups=[], + abort=False, + ) + # LLM returns flat filenames (no subdir prefix yet). + mock_advise.return_value = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="helpers.py"), + ], + abort=False, + ) + mock_gen.return_value = SplitResult( + new_files={ + "service/utils.py": "def foo():\n pass", + "service/helpers.py": "def bar():\n pass", + }, + original_source="# init content\n", + abort=False, + ) + + cfg = CrispenConfig(file_limiter_subdir_split=True) + result = run_file_limiter("service.py", source, source, [(1, 4)], cfg) + + assert result.abort is False + # service/__init__.py carries the post-split original source. + assert "service/__init__.py" in result.new_files + assert result.new_files["service/__init__.py"] == "# init content\n" + # original_source is reset to the input (so service.py is not modified). + assert result.original_source == source + assert result.subdir_name == "service" + # The moved-message includes the prefixed target file. + assert any("service/utils.py" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_subdir_split_test_file_keeps_original( + mock_classify, mock_advise, mock_gen +): + # Whole-file diff on a test file → placements get subdir prefix but + # original_source (re-export stubs in test_service.py) is written back. + source = "def test_foo():\n pass\ndef test_bar():\n pass\n" + entity1 = _make_entity("test_foo", 1, 2) + entity2 = _make_entity("test_bar", 3, 4) + # Two groups required so the n_groups > 1 subdir guard doesn't fire. + mock_classify.return_value = ClassifiedEntities( + entities=[entity1, entity2], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[["test_foo"], ["test_bar"]], + set_3_groups=[], + abort=False, + ) + mock_advise.return_value = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["test_foo"], target_file="helpers.py"), + GroupPlacement(group=["test_bar"], target_file="extras.py"), + ], + abort=False, + ) + mock_gen.return_value = SplitResult( + new_files={ + "service/test_helpers.py": "def test_foo():\n pass", + "service/test_extras.py": "def test_bar():\n pass", + }, + original_source="# re-export stubs\n", + abort=False, + ) + + cfg = CrispenConfig(file_limiter_subdir_split=True) + result = run_file_limiter("tests/test_service.py", source, source, [(1, 4)], cfg) + + assert result.abort is False + # No __init__.py injected for test files. + assert "service/__init__.py" not in result.new_files + # original_source has the re-export stubs (NOT reset to input). + assert result.original_source == "# re-export stubs\n" + assert result.subdir_name == "service" + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_subdir_split_strips_test_prefix_from_stem( + mock_classify, mock_advise, mock_gen +): + # test_big.py → subdir "big/" (strip "test_" prefix from stem). + source = "def test_foo():\n pass\ndef test_bar():\n pass\n" + entity1 = _make_entity("test_foo", 1, 2) + entity2 = _make_entity("test_bar", 3, 4) + # Two groups required so the n_groups > 1 subdir guard doesn't fire. + mock_classify.return_value = ClassifiedEntities( + entities=[entity1, entity2], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[["test_foo"], ["test_bar"]], + set_3_groups=[], + abort=False, + ) + mock_advise.return_value = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["test_foo"], target_file="helpers.py"), + GroupPlacement(group=["test_bar"], target_file="extras.py"), + ], + abort=False, + ) + mock_gen.return_value = SplitResult( + new_files={ + "big/test_helpers.py": "def test_foo():\n pass", + "big/test_extras.py": "def test_bar():\n pass", + }, + original_source="# stubs\n", + abort=False, + ) + + cfg = CrispenConfig(file_limiter_subdir_split=True) + result = run_file_limiter("tests/test_big.py", source, source, [(1, 4)], cfg) + + assert result.abort is False + assert result.subdir_name == "big" + # "helpers.py" → test_ prefix → "test_helpers.py" → "big/test_helpers.py". + assert any("big/test_helpers.py" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_subdir_split_has_main_uses_lib_suffix( + mock_classify, mock_advise, mock_gen, tmp_path +): + # Non-test file with __main__: subdir uses "_lib" suffix, original_source + # is the split content (re-export stubs + __main__), and has_main=True. + # No blank lines between entities so entity ranges don't pick up leading \n. + source = ( + "def foo():\n pass\n" + "def bar():\n pass\n" + "if __name__ == '__main__':\n foo()\n" + ) + entity1 = _make_entity("foo", 1, 2) + entity2 = _make_entity("bar", 3, 4) + mock_classify.return_value = ClassifiedEntities( + entities=[entity1, entity2], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[["foo"], ["bar"]], + set_3_groups=[], + abort=False, + ) + mock_advise.return_value = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="helpers.py"), + ], + abort=False, + ) + mock_gen.return_value = SplitResult( + new_files={ + "service_lib/utils.py": "def foo():\n pass", + "service_lib/helpers.py": "def bar():\n pass", + }, + original_source=( + "from service_lib.utils import foo\n\n" + "if __name__ == '__main__':\n foo()\n" + ), + abort=False, + ) + + cfg = CrispenConfig(file_limiter_subdir_split=True) + filepath = str(tmp_path / "service.py") + result = run_file_limiter(filepath, source, source, [(1, 6)], cfg) + + assert result.abort is False + assert result.has_main is True + assert result.subdir_name == "service_lib" + # original_source keeps the split content (re-exports + __main__), not reset. + assert "__main__" in result.original_source + # No __init__.py injected: original file stays as the entry point. + assert "service_lib/__init__.py" not in result.new_files + assert any("service_lib/utils.py" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_subdir_split_has_main_fallback_suffix( + mock_classify, mock_advise, mock_gen, tmp_path +): + # When service_lib/ already exists, fall back to the next suffix (_helpers). + source = ( + "def foo():\n pass\n" + "def bar():\n pass\n" + "if __name__ == '__main__':\n foo()\n" + ) + (tmp_path / "service_lib").mkdir() + entity1 = _make_entity("foo", 1, 2) + entity2 = _make_entity("bar", 3, 4) + mock_classify.return_value = ClassifiedEntities( + entities=[entity1, entity2], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[["foo"], ["bar"]], + set_3_groups=[], + abort=False, + ) + mock_advise.return_value = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="helpers.py"), + ], + abort=False, + ) + mock_gen.return_value = SplitResult( + new_files={ + "service_helpers/utils.py": "def foo():\n pass", + "service_helpers/helpers.py": "def bar():\n pass", + }, + original_source="# stubs\n", + abort=False, + ) + + cfg = CrispenConfig(file_limiter_subdir_split=True) + filepath = str(tmp_path / "service.py") + result = run_file_limiter(filepath, source, source, [(1, 6)], cfg) + + assert result.abort is False + assert result.subdir_name == "service_helpers" + assert result.has_main is True + + +@patch(_PATCH_CLASSIFY) +def test_runner_subdir_split_has_main_all_suffixes_conflict_aborts( + mock_classify, tmp_path +): + # All _lib/_helpers/etc. directories already exist → abort with a clear message. + source = "def foo():\n pass\n\nif __name__ == '__main__':\n foo()\n" + for suffix in _MAIN_SUBDIR_SUFFIXES: + (tmp_path / f"service{suffix}").mkdir() + mock_classify.return_value = _make_classified() + + cfg = CrispenConfig(file_limiter_subdir_split=True) + filepath = str(tmp_path / "service.py") + result = run_file_limiter(filepath, source, source, [(1, 5)], cfg) + + assert result.abort is True + assert result.new_files == {} + assert any("__main__" in m for m in result.messages) + assert any("conflict" in m for m in result.messages) + + +@patch(_PATCH_CLASSIFY) +def test_runner_init_py_skips_subdir_split(mock_classify, tmp_path): + """__init__.py with a whole-file diff must not trigger subdir-split detection. + + A subdir split for __init__.py would create an ``__init__/`` subdirectory, + which is nonsensical. Instead it should fall through to the normal in-place + split (siblings in the same package directory). + """ + # Classify returns abort so the LLM path is skipped; we only care that + # subdir_name is NOT set on the result. + mock_classify.return_value = ClassifiedEntities( + entities=[], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[], + set_3_groups=[], + abort=True, + ) + filepath = str(tmp_path / "__init__.py") + # Make the source long enough to be a "whole-file diff". + source = "".join(f"def func_{i}():\n pass\n\n" for i in range(10)) + cfg = CrispenConfig(file_limiter_subdir_split=True) + result = run_file_limiter( + filepath, source, source, [(1, len(source.splitlines()))], cfg + ) + + # Abort comes from the classifier — subdir conflict detection was bypassed. + assert result.abort is True + assert result.subdir_name is None + assert "already exists" not in " ".join(result.messages) diff --git a/tests/runner/test_runner_core.py b/tests/runner/test_runner_core.py new file mode 100644 index 0000000..abdc49c --- /dev/null +++ b/tests/runner/test_runner_core.py @@ -0,0 +1,659 @@ +from __future__ import annotations +from unittest.mock import patch +from crispen.config import CrispenConfig +from crispen.file_limiter.advisor import FileLimiterPlan, GroupPlacement +from crispen.file_limiter.classifier import ClassifiedEntities +from crispen.file_limiter.code_gen import SplitResult +from crispen.file_limiter.entity_parser import Entity, EntityKind +from crispen.file_limiter.runner import _is_whole_file_diff, run_file_limiter +from .test_preservation import ( + _CONFIG, + _CONFIG_NO_RETRY, + _PATCH_ADVISE, + _PATCH_CLASSIFY, + _PATCH_GEN, + _good_split, + _make_classified, + _make_entity, +) + + +def _abort_plan() -> FileLimiterPlan: + return FileLimiterPlan(set3_migrate=[], placements=[], abort=True) + + +def _empty_plan() -> FileLimiterPlan: + return FileLimiterPlan(set3_migrate=[], placements=[], abort=False) + + +def _plan_with(group: list, target: str) -> FileLimiterPlan: + return FileLimiterPlan( + set3_migrate=[], + placements=[GroupPlacement(group=group, target_file=target)], + abort=False, + ) + + +def _classified_with_groups(entities=None) -> ClassifiedEntities: + """Classified result with non-empty set_3_groups (triggers LLM advise).""" + ents = entities or [_make_entity("foo", 1, 2)] + return ClassifiedEntities( + entities=ents, + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[], + set_3_groups=[[e.name for e in ents]], + abort=False, + ) + + +@patch(_PATCH_CLASSIFY) +def test_runner_dashed_parent_dir_skips(mock_classify): + # A filepath whose parent contains a dash must be skipped immediately, + # before classify_entities is ever called. + result = run_file_limiter( + "tests/cross-engine/test_lever.py", "", "x = 1\n", [(1, 1)], _CONFIG + ) + assert result.abort is True + assert "cross-engine" in result.messages[0] + assert "dash" in result.messages[0] + mock_classify.assert_not_called() + + +@patch(_PATCH_CLASSIFY) +def test_runner_dashed_parent_dir_deep_skips(mock_classify): + # Dash anywhere in the ancestor chain (not just the immediate parent). + result = run_file_limiter( + "src/my-pkg/sub/module.py", "", "x = 1\n", [(1, 1)], _CONFIG + ) + assert result.abort is True + assert "my-pkg" in result.messages[0] + mock_classify.assert_not_called() + + +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_nonexistent_parent_dir_existing_dirs_empty(mock_classify, mock_advise): + """When source dir doesn't exist, iterdir raises FileNotFoundError → empty set.""" + mock_classify.return_value = ClassifiedEntities( + entities=[], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[], + set_3_groups=[], + abort=True, + ) + # /nonexistent/parent doesn't exist; iterdir() will raise FileNotFoundError. + result = run_file_limiter( + "/nonexistent/parent/module.py", + "", + "def foo(): pass\n", + [(1, 1)], + _CONFIG, + ) + # Abort from classifier, but the FileNotFoundError branch was hit. + assert result.abort is True + + +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_classifier_abort(mock_classify, mock_advise): + # classified.abort=True → early return before LLM; advise never called. + mock_classify.return_value = ClassifiedEntities( + entities=[_make_entity("a", 1, 2), _make_entity("b", 3, 4)], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[], + set_3_groups=[], + abort=True, + abort_reason="", + ) + + result = run_file_limiter("big.py", "", "def a(): b()\ndef b(): a()\n", [], _CONFIG) + + assert result.abort is True + mock_advise.assert_not_called() + assert any("cannot be split" in m for m in result.messages) + + +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_classifier_abort_with_reason(mock_classify, mock_advise): + mock_classify.return_value = ClassifiedEntities( + entities=[_make_entity("a", 1, 2), _make_entity("b", 3, 4)], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[], + set_3_groups=[], + abort=True, + abort_reason="all 2 top-level entities form one dependency cycle", + ) + + result = run_file_limiter("big.py", "", "def a(): b()\ndef b(): a()\n", [], _CONFIG) + + assert result.abort is True + mock_advise.assert_not_called() + assert any("dependency cycle" in m for m in result.messages) + + +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_plan_abort(mock_classify, mock_advise): + mock_classify.return_value = _make_classified() + mock_advise.return_value = _abort_plan() + + result = run_file_limiter( + "big.py", "", "def foo():\n pass\n", [], _CONFIG_NO_RETRY + ) + + assert result.abort is True + assert result.new_files == {} + assert any("cannot be split" in m for m in result.messages) + + +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_plan_abort_with_reason(mock_classify, mock_advise): + mock_classify.return_value = _make_classified() + mock_advise.return_value = FileLimiterPlan( + set3_migrate=[], placements=[], abort=True, abort_reason="all 3 entities cycle" + ) + + result = run_file_limiter( + "big.py", "", "def foo():\n pass\n", [], _CONFIG_NO_RETRY + ) + + assert result.abort is True + assert any("all 3 entities cycle" in m for m in result.messages) + + +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_plan_abort_retries_and_fails(mock_classify, mock_advise): + # retries=1: both attempts produce plan.abort (set-3 failure) → 2 SKIP messages. + mock_classify.return_value = _make_classified() + mock_advise.return_value = FileLimiterPlan( + set3_migrate=[], + placements=[], + abort=True, + abort_reason="LLM failed to plan set-3 groups", + ) + cfg = CrispenConfig(file_limiter_retries=1) + + result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) + + assert result.abort is True + assert mock_advise.call_count == 2 + assert sum(1 for m in result.messages if "cannot be split" in m) == 2 + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_plan_abort_retries_and_succeeds(mock_classify, mock_advise, mock_gen): + # retries=1: first plan.abort (placement failure), second succeeds. + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.side_effect = [ + FileLimiterPlan( + set3_migrate=[], + placements=[], + abort=True, + abort_reason="LLM failed to assign file placements", + ), + _plan_with(["foo"], "utils.py"), + ] + mock_gen.return_value = _good_split() + cfg = CrispenConfig(file_limiter_retries=1) + + result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) + + assert result.abort is False + assert mock_advise.call_count == 2 + # Failed attempt message is preserved alongside the success message. + assert any("cannot be split" in m for m in result.messages) + assert any("FileLimiter: moved" in m for m in result.messages) + # Feedback was forwarded on the second call. + assert mock_advise.call_args_list[1].kwargs["prev_placement_failure"] != "" + + +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_no_placements(mock_classify, mock_advise): + mock_classify.return_value = _make_classified() + mock_advise.return_value = _empty_plan() + + source = "def foo():\n pass\n" + result = run_file_limiter("big.py", "", source, [], _CONFIG) + + assert result.abort is False + assert result.new_files == {} + assert result.original_source == source + assert result.messages == [] + + +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_no_placements_with_groups(mock_classify, mock_advise): + # When set_3_groups is non-empty but the LLM selects nothing to migrate, + # runner should emit a SKIP message so the user knows the file was examined. + mock_classify.return_value = _classified_with_groups() + mock_advise.return_value = _empty_plan() + + source = "def foo():\n pass\n" + result = run_file_limiter("big.py", "", source, [], _CONFIG_NO_RETRY) + + assert result.abort is False + assert result.new_files == {} + assert any("no entities selected for migration" in m for m in result.messages) + + +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_no_migration_retries_and_fails(mock_classify, mock_advise): + # retries=1: both attempts → no entities selected → 2 SKIP msgs, abort=False. + mock_classify.return_value = _classified_with_groups() + mock_advise.return_value = _empty_plan() + cfg = CrispenConfig(file_limiter_retries=1) + + result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) + + assert result.abort is False + assert mock_advise.call_count == 2 + assert sum(1 for m in result.messages if "no entities selected" in m) == 2 + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_no_migration_retries_and_succeeds(mock_classify, mock_advise, mock_gen): + # retries=1: first attempt → no migration; second → placements and success. + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _classified_with_groups(entities=[entity]) + mock_advise.side_effect = [_empty_plan(), _plan_with(["foo"], "utils.py")] + mock_gen.return_value = _good_split() + cfg = CrispenConfig(file_limiter_retries=1) + + result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) + + assert result.abort is False + assert mock_advise.call_count == 2 + # Failed attempt message is preserved alongside the success message. + assert any("no entities selected" in m for m in result.messages) + assert any("FileLimiter: moved" in m for m in result.messages) + # Feedback about all-stay was forwarded on the second call. + assert mock_advise.call_args_list[1].kwargs["prev_set3_failure"] != "" + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_verification_fails(mock_classify, mock_advise, mock_gen): + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["foo"], "utils.py") + # Return a split where foo's source is NOT present anywhere. + mock_gen.return_value = SplitResult( + new_files={"utils.py": "# empty placeholder"}, + original_source="# empty original", + abort=False, + ) + + result = run_file_limiter("big.py", "", source, [], _CONFIG) + + assert result.abort is True + assert result.original_source == source + assert any("verification failed" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_success(mock_classify, mock_advise, mock_gen): + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["foo"], "utils.py") + mock_gen.return_value = SplitResult( + new_files={"utils.py": "def foo():\n pass"}, + original_source="# original updated\n", + abort=False, + ) + + result = run_file_limiter("big.py", "", source, [], _CONFIG) + + assert result.abort is False + assert "utils.py" in result.new_files + assert result.original_source == "# original updated\n" + assert any("FileLimiter: moved" in m for m in result.messages) + assert any("foo" in m for m in result.messages) + assert any("utils.py" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_passes_pytest_conftest_to_generate( + mock_classify, mock_advise, mock_gen +): + # Verify config.file_limiter_pytest_conftest is forwarded to generate_file_splits. + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["foo"], "utils.py") + mock_gen.return_value = _good_split() + + config_false = CrispenConfig(file_limiter_pytest_conftest=False) + run_file_limiter("big.py", "", source, [], config_false) + + _, call_kwargs = mock_gen.call_args + assert call_kwargs.get("pytest_conftest") is False + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_split_aborts_on_cycle(mock_classify, mock_advise, mock_gen): + # generate_file_splits detects a cycle and returns abort=True with no + # new_files. run_file_limiter must emit a SKIP message (not bogus "moved" + # messages) and return abort=True so the engine skips the file. + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["foo"], "utils.py") + mock_gen.return_value = SplitResult( + new_files={}, + original_source=source, + abort=True, + ) + + result = run_file_limiter("big.py", "", source, [], _CONFIG_NO_RETRY) + + assert result.abort is True + assert result.new_files == {} + assert result.original_source == source + # Must not claim to have moved anything. + assert not any("FileLimiter: moved" in m for m in result.messages) + assert any("cannot be split" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_split_aborts_with_reason(mock_classify, mock_advise, mock_gen): + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["foo"], "utils.py") + mock_gen.return_value = SplitResult( + new_files={}, + original_source=source, + abort=True, + abort_reason="proposed split would create circular file imports", + ) + + result = run_file_limiter("big.py", "", source, [], _CONFIG_NO_RETRY) + + assert result.abort is True + assert any("circular file imports" in m for m in result.messages) + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_split_abort_retries_and_fails(mock_classify, mock_advise, mock_gen): + # retries=1: both attempts produce split.abort → 2 SKIP messages, abort=True. + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["foo"], "utils.py") + mock_gen.return_value = SplitResult( + new_files={}, + original_source=source, + abort=True, + abort_reason="proposed split would create circular file imports", + ) + cfg = CrispenConfig(file_limiter_retries=1) + + result = run_file_limiter("big.py", "", source, [], cfg) + + assert result.abort is True + assert mock_advise.call_count == 2 + assert mock_gen.call_count == 2 + assert sum(1 for m in result.messages if "cannot be split" in m) == 2 + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_split_abort_retries_and_succeeds(mock_classify, mock_advise, mock_gen): + # retries=1: first split.abort, second succeeds → only "moved" message, no SKIP. + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["foo"], "utils.py") + mock_gen.side_effect = [ + SplitResult( + new_files={}, + original_source=source, + abort=True, + abort_reason="circular imports", + ), + _good_split(), + ] + cfg = CrispenConfig(file_limiter_retries=1) + + result = run_file_limiter("big.py", "", source, [], cfg) + + assert result.abort is False + assert mock_advise.call_count == 2 + assert mock_gen.call_count == 2 + # Failed attempt message is preserved alongside the success message. + assert any("cannot be split" in m for m in result.messages) + assert any("FileLimiter: moved" in m for m in result.messages) + # Circular-import feedback was forwarded on the second call. + prev_pf = mock_advise.call_args_list[1].kwargs["prev_placement_failure"] + assert "circular" in prev_pf + + +def test_is_whole_file_diff_empty_ranges(): + assert _is_whole_file_diff([], 5) is False + + +def test_is_whole_file_diff_zero_lines(): + assert _is_whole_file_diff([(1, 3)], 0) is False + + +def test_is_whole_file_diff_gap(): + # Lines 1-2 and 4-5 — line 3 is missing. + assert _is_whole_file_diff([(1, 2), (4, 5)], 5) is False + + +def test_is_whole_file_diff_doesnt_start_at_one(): + # Range starts at line 2 — line 1 is not covered. + assert _is_whole_file_diff([(2, 5)], 5) is False + + +def test_is_whole_file_diff_partial_coverage(): + # Covers lines 1-3 but file has 5 lines. + assert _is_whole_file_diff([(1, 3)], 5) is False + + +def test_is_whole_file_diff_exact_coverage(): + assert _is_whole_file_diff([(1, 5)], 5) is True + + +def test_is_whole_file_diff_multi_range_contiguous(): + # Two adjacent ranges that together cover 1..5. + assert _is_whole_file_diff([(1, 3), (4, 5)], 5) is True + + +def test_is_whole_file_diff_overshoots(): + # Ranges cover more lines than n_lines — still counts as whole-file. + assert _is_whole_file_diff([(1, 10)], 5) is True + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_verbose_prints_to_stderr(mock_classify, mock_advise, mock_gen, capsys): + """verbose=True prints analysis/verification messages to stderr.""" + source = "def foo():\n pass\n" + entity = _make_entity("foo", 1, 2) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["foo"], "utils.py") + mock_gen.return_value = SplitResult( + new_files={"utils.py": "def foo():\n pass"}, + original_source="# original updated\n", + abort=False, + ) + + result = run_file_limiter("big.py", "", source, [], _CONFIG, verbose=True) + + assert result.abort is False + err = capsys.readouterr().err + assert "FileLimiter" in err + assert "big.py" in err + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_success_with_class_entity(mock_classify, mock_advise, mock_gen): + """Verification loop increments verified_classes for CLASS entities.""" + source = "class Foo:\n pass\n" + entity = Entity(EntityKind.CLASS, "Foo", 1, 2, ["Foo"]) + mock_classify.return_value = _make_classified(entities=[entity]) + mock_advise.return_value = _plan_with(["Foo"], "models.py") + mock_gen.return_value = SplitResult( + new_files={"models.py": "class Foo:\n pass"}, + original_source="# original\n", + abort=False, + ) + + result = run_file_limiter("big.py", "", source, [], _CONFIG) + + assert result.abort is False + assert result.verified_classes == 1 + assert result.verified_functions == 0 + assert result.verified_lines == 2 + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_success_with_top_level_entity(mock_classify, mock_advise, mock_gen): + """TOP_LEVEL entities are skipped in the verification count loop.""" + source = "import os\ndef foo():\n pass\n" + import_entity = Entity(EntityKind.TOP_LEVEL, "_block_0", 1, 1, ["os"]) + func_entity = _make_entity("foo", 2, 3) + mock_classify.return_value = _make_classified(entities=[import_entity, func_entity]) + mock_advise.return_value = _plan_with(["foo"], "utils.py") + mock_gen.return_value = SplitResult( + new_files={"utils.py": "def foo():\n pass"}, + original_source="import os\n", + abort=False, + ) + + result = run_file_limiter("big.py", "", source, [], _CONFIG) + + assert result.abort is False + # Only the function counts; TOP_LEVEL is skipped. + assert result.verified_functions == 1 + assert result.verified_classes == 0 + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_runner_success_with_empty_entity_source(mock_classify, mock_advise, mock_gen): + """Entities whose source is blank after rstrip are skipped in the count. + + Also covers verification of an entity that stays in the original file + (stays_entity is verified and counted alongside migrated entities). + """ + source = "def foo():\n pass\n\ndef bar():\n pass\n" + # blank_entity has empty source → skipped. foo migrated; bar stays in original. + blank_entity = _make_entity("_block_1", 3, 3) + func_entity = _make_entity("foo", 1, 2) + stays_entity = _make_entity("bar", 4, 5) + mock_classify.return_value = _make_classified( + entities=[func_entity, blank_entity, stays_entity] + ) + mock_advise.return_value = _plan_with(["foo"], "utils.py") + mock_gen.return_value = SplitResult( + new_files={"utils.py": "def foo():\n pass"}, + original_source="def bar():\n pass\n", + abort=False, + ) + + result = run_file_limiter("big.py", "", source, [], _CONFIG) + + assert result.abort is False + # blank_entity: empty source → skipped. bar: stays in original → not counted. + # Only foo (migrated) counts. + assert result.verified_functions == 1 + assert result.verified_lines == 2 + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_entity_to_target_populated_on_success(mock_classify, mock_advise, mock_gen): + """On a successful run, entity_to_target maps entity names to target files.""" + source = "def foo():\n pass\ndef bar():\n pass\n" + entity_foo = _make_entity("foo", 1, 2) + entity_bar = _make_entity("bar", 3, 4) + mock_classify.return_value = _make_classified(entities=[entity_foo, entity_bar]) + # Plan: foo → utils.py, bar → helpers.py + + mock_advise.return_value = FileLimiterPlan( + set3_migrate=[], + placements=[ + GroupPlacement(group=["foo"], target_file="utils.py"), + GroupPlacement(group=["bar"], target_file="helpers.py"), + ], + abort=False, + ) + mock_gen.return_value = SplitResult( + new_files={ + "utils.py": "def foo():\n pass", + "helpers.py": "def bar():\n pass", + }, + original_source="# original updated\n", + abort=False, + ) + + result = run_file_limiter("big.py", "", source, [], _CONFIG) + + assert result.abort is False + assert result.entity_to_target == { + "foo": "utils.py", + "bar": "helpers.py", + } + + +@patch(_PATCH_GEN) +@patch(_PATCH_ADVISE) +@patch(_PATCH_CLASSIFY) +def test_entity_to_target_empty_on_abort(mock_classify, mock_advise, mock_gen): + """Abort result has empty entity_to_target.""" + mock_classify.return_value = ClassifiedEntities( + entities=[], + entity_class={}, + graph={}, + set_1=[], + set_2_groups=[], + set_3_groups=[], + abort=True, + ) + + result = run_file_limiter("big.py", "", "x = 1\n", [], _CONFIG_NO_RETRY) + + assert result.abort is True + assert result.entity_to_target == {} diff --git a/tests/test_advisor.py b/tests/test_advisor.py index 252f59c..f5a8e65 100644 --- a/tests/test_advisor.py +++ b/tests/test_advisor.py @@ -1,2167 +1,3 @@ """Tests for file_limiter.advisor — 100% branch coverage.""" from __future__ import annotations - -from unittest.mock import MagicMock, patch - -import pytest - -from crispen.config import CrispenConfig -from crispen.errors import CrispenAPIError -from crispen.llm_client import LLMCallResult -from crispen.file_limiter.advisor import ( - _LLMAccumulator, - _PLACEMENT_CHUNK_SIZE, - _advise_set3, - _assign_placements_chunk, - _build_group_mermaid, - _compute_projected_lines, - _find_conflicting_placement_indices, - _group_summary, - _propose_files_step, - _refine_merge_tiny, - advise_file_limiter, - GroupPlacement, - resolve_naming_conflicts, -) -from crispen.file_limiter.classifier import ClassifiedEntities -from crispen.file_limiter.entity_parser import Entity, EntityKind - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_entity( - name: str, - start: int, - end: int, - *, - docstring=None, - params=None, -) -> Entity: - return Entity( - EntityKind.FUNCTION, - name, - start, - end, - [name], - docstring=docstring, - params=params or [], - ) - - -def _classified( - *, - entities=None, - entity_class=None, - graph=None, - set_1=None, - set_2_groups=None, - set_3_groups=None, - abort=False, -) -> ClassifiedEntities: - return ClassifiedEntities( - entities=entities or [], - entity_class=entity_class or {}, - graph=graph if graph is not None else {}, - set_1=set_1 or [], - set_2_groups=set_2_groups or [], - set_3_groups=set_3_groups or [], - abort=abort, - ) - - -def _make_llm_result(tool_input) -> LLMCallResult: - """Wrap a dict (or None) in LLMCallResult for mock_call.return_value.""" - return LLMCallResult( - tool_input=tool_input, elapsed=0.01, input_tokens=10, output_tokens=5 - ) - - -def _propose_ok(*filenames: str) -> LLMCallResult: - """Return a valid propose_output_files LLM response for the given filenames.""" - return _make_llm_result( - {"files": [{"filename": f, "description": "auto-generated"} for f in filenames]} - ) - - -_CONFIG = CrispenConfig() -_PATCH_KEY = "crispen.file_limiter.advisor.get_api_key" -_PATCH_CLIENT = "crispen.file_limiter.advisor.make_client" -_PATCH_CALL = "crispen.file_limiter.advisor.call_with_tool" - - -# --------------------------------------------------------------------------- -# Early-exit paths (no LLM calls) -# --------------------------------------------------------------------------- - - -def test_plan_abort_when_classified_abort(): - """classified.abort=True → FileLimiterPlan(abort=True), no LLM calls.""" - c = _classified(abort=True) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - assert plan.abort is True - assert plan.set3_migrate == [] - assert plan.placements == [] - - -def test_plan_no_movable_groups(): - """set_2=[], set_3=[] → empty plan, no LLM calls.""" - c = _classified() - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - assert plan.abort is False - assert plan.placements == [] - - -# --------------------------------------------------------------------------- -# API key error propagates -# --------------------------------------------------------------------------- - - -def test_plan_api_key_error_propagates(monkeypatch): - """Missing API key raises CrispenAPIError before any LLM call.""" - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - c = _classified( - entities=[_make_entity("foo", 1, 5)], - set_2_groups=[["foo"]], - ) - with pytest.raises(CrispenAPIError): - advise_file_limiter(c, "src/big.py", _CONFIG) - - -# --------------------------------------------------------------------------- -# Set 2 only (skip Set 3 call) -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_set2_only_skips_set3_call(mock_key, mock_client, mock_call): - """set_2 groups only: no set3 call; propose + assign = 2 LLM calls.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _propose_ok("utils.py"), - _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), - ] - c = _classified( - entities=[_make_entity("foo", 1, 10)], - set_2_groups=[["foo"]], - ) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - - assert plan.abort is False - assert plan.set3_migrate == [] - assert len(plan.placements) == 1 - assert plan.placements[0].group == ["foo"] - assert plan.placements[0].target_file == "utils.py" - assert ( - mock_call.call_count == 2 - ) # propose + assign (no refinement: only 1 tiny file) - - -# --------------------------------------------------------------------------- -# Set 3 — stay and migrate paths -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_set3_all_stay_no_placement(mock_key, mock_client, mock_call): - """All Set 3 groups stay → no propose/assign call, empty plan.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - {"decisions": [{"group_id": 0, "action": "stay"}]} - ) - - c = _classified( - entities=[_make_entity("bar", 1, 10)], - set_3_groups=[["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - - assert plan.abort is False - assert plan.set3_migrate == [] - assert plan.placements == [] - assert mock_call.call_count == 1 # only set3 advice call - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_set3_migrate(mock_key, mock_client, mock_call): - """Set 3 group migrates → set3 + propose + assign = 3 LLM calls.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _make_llm_result({"decisions": [{"group_id": 0, "action": "migrate"}]}), - _propose_ok("helpers.py"), - _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "helpers.py"}]} - ), - ] - c = _classified( - entities=[_make_entity("bar", 1, 10)], - set_3_groups=[["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - - assert plan.abort is False - assert plan.set3_migrate == [["bar"]] - assert len(plan.placements) == 1 - assert plan.placements[0].group == ["bar"] - assert plan.placements[0].target_file == "helpers.py" - assert mock_call.call_count == 3 - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_set3_test_subdir_skips_advise_call(mock_key, mock_client, mock_call): - """Test-file subdir split: set-3 groups migrate without an LLM advice call.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - # Only propose + assign calls; no set3-advice call. - mock_call.side_effect = [ - _propose_ok("test_helpers.py"), - _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "test_helpers.py"}]} - ), - ] - c = _classified( - entities=[_make_entity("test_bar", 1, 10)], - set_3_groups=[["test_bar"]], - ) - plan = advise_file_limiter(c, "tests/test_big.py", _CONFIG, subdir_name="big") - - assert plan.abort is False - assert plan.set3_migrate == [["test_bar"]] - assert len(plan.placements) == 1 - assert plan.placements[0].target_file == "test_helpers.py" - assert mock_call.call_count == 2 # no set3-advice call - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_set2_and_set3_migrate(mock_key, mock_client, mock_call): - """set_2 + migrating set_3 → both groups in placement call.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _make_llm_result({"decisions": [{"group_id": 0, "action": "migrate"}]}), - _propose_ok("new_stuff.py", "changed.py"), - _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "new_stuff.py"}, - {"group_id": 1, "target_file": "changed.py"}, - ] - } - ), - ] - c = _classified( - entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 15)], - set_2_groups=[["foo"]], - set_3_groups=[["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - - assert plan.abort is False - assert plan.set3_migrate == [["bar"]] - assert len(plan.placements) == 2 - targets = {p.target_file for p in plan.placements} - assert targets == {"new_stuff.py", "changed.py"} - - -# --------------------------------------------------------------------------- -# LLM returns None → abort -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_set3_call_returns_none_aborts(mock_key, mock_client, mock_call): - """Call 1 (set3 advice) returns None → abort.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result(None) - - c = _classified( - entities=[_make_entity("bar", 1, 10)], - set_3_groups=[["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - assert plan.abort is True - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_placement_call_returns_none_aborts(mock_key, mock_client, mock_call): - """Propose succeeds then assignment chunk exhausts retries → abort.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - # file_limiter_retries=0 → 1 attempt for propose, 1 attempt for assign. - mock_call.side_effect = [ - _make_llm_result({"decisions": [{"group_id": 0, "action": "migrate"}]}), - _propose_ok("helpers.py"), - _make_llm_result(None), # assignment fails - ] - c = _classified( - entities=[_make_entity("bar", 1, 10)], - set_3_groups=[["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) - assert plan.abort is True - - -# --------------------------------------------------------------------------- -# Invalid LLM responses — set3 advice -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_set3_invalid_group_id_treated_as_stay(mock_key, mock_client, mock_call): - """Out-of-range group_id in set3 advice → skipped (treated as stay).""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - { - "decisions": [ - {"group_id": 99, "action": "migrate"}, # invalid — out of range - {"group_id": 0, "action": "stay"}, - ] - } - ) - c = _classified( - entities=[_make_entity("bar", 1, 10)], - set_3_groups=[["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - assert plan.abort is False - assert plan.set3_migrate == [] - assert plan.placements == [] - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_set3_non_int_group_id_treated_as_stay(mock_key, mock_client, mock_call): - """Non-integer group_id in set3 advice → isinstance check fails → stay.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - {"decisions": [{"group_id": "zero", "action": "migrate"}]} - ) - c = _classified( - entities=[_make_entity("bar", 1, 10)], - set_3_groups=[["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - assert plan.set3_migrate == [] - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_set3_unknown_action_treated_as_stay(mock_key, mock_client, mock_call): - """Unknown action value in set3 advice → action != 'migrate' → stay.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - {"decisions": [{"group_id": 0, "action": "delete"}]} # not in enum - ) - c = _classified( - entities=[_make_entity("bar", 1, 10)], - set_3_groups=[["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - assert plan.set3_migrate == [] - - -# --------------------------------------------------------------------------- -# Invalid LLM responses — placement assignment -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_placement_incomplete_aborts(mock_key, mock_client, mock_call): - """Placement missing some group_ids → len mismatch → abort.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - # Two groups but only one placement returned; retries=0 → immediate abort. - mock_call.side_effect = [ - _propose_ok("utils.py"), - _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), - ] - c = _classified( - entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)], - set_2_groups=[["foo"], ["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) - assert plan.abort is True - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_placement_duplicate_group_id_aborts(mock_key, mock_client, mock_call): - """Duplicate group_id in placement → only first counted → len mismatch → abort.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _propose_ok("utils.py", "other.py"), - _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "utils.py"}, - {"group_id": 0, "target_file": "other.py"}, # duplicate - ] - } - ), - ] - c = _classified( - entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)], - set_2_groups=[["foo"], ["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) - assert plan.abort is True - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_placement_empty_target_aborts(mock_key, mock_client, mock_call): - """Empty target_file → falsy check fails → treated as missing → abort.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _propose_ok("utils.py"), - _make_llm_result({"placements": [{"group_id": 0, "target_file": ""}]}), - ] - c = _classified( - entities=[_make_entity("foo", 1, 5)], - set_2_groups=[["foo"]], - ) - plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) - assert plan.abort is True - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_placement_out_of_range_group_id_aborts(mock_key, mock_client, mock_call): - """Out-of-range group_id in placement → skipped → len mismatch → abort.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _propose_ok("utils.py"), - _make_llm_result({"placements": [{"group_id": 99, "target_file": "utils.py"}]}), - ] - c = _classified( - entities=[_make_entity("foo", 1, 5)], - set_2_groups=[["foo"]], - ) - plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) - assert plan.abort is True - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_placement_non_int_group_id_aborts(mock_key, mock_client, mock_call): - """Non-integer group_id in placement → isinstance check fails → abort.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _propose_ok("utils.py"), - _make_llm_result( - {"placements": [{"group_id": "zero", "target_file": "utils.py"}]} - ), - ] - c = _classified( - entities=[_make_entity("foo", 1, 5)], - set_2_groups=[["foo"]], - ) - plan = advise_file_limiter(c, "src/big.py", CrispenConfig(file_limiter_retries=0)) - assert plan.abort is True - - -# --------------------------------------------------------------------------- -# Placement target not in proposed list → abort -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_placement_targets_outside_proposed_aborts( - mock_key, mock_client, mock_call -): - """LLM returns target not in proposed list → constrained check fails → abort.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - # Propose "utils.py" but assignment tries to use "existing.py" (not proposed). - mock_call.side_effect = [ - _propose_ok("utils.py"), - _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "existing.py"}]} - ), - ] - c = _classified( - entities=[_make_entity("foo", 1, 5)], - set_2_groups=[["foo"]], - ) - plan = advise_file_limiter( - c, - "src/big.py", - CrispenConfig(file_limiter_retries=0), - existing_files=frozenset({"existing.py"}), - ) - assert plan.abort is True - - -# --------------------------------------------------------------------------- -# _group_summary: entity not in map -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_entity_not_in_entity_map(mock_key, mock_client, mock_call): - """Group contains name absent from entity list → falls back to name-only display.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _propose_ok("utils.py"), - _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), - ] - # "ghost" is not in entities list, so entity_map lookup fails. - c = _classified( - entities=[], - set_2_groups=[["ghost"]], - ) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - assert plan.abort is False - assert plan.placements[0].target_file == "utils.py" - - -# --------------------------------------------------------------------------- -# prev_failure feedback propagation -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_set3_prev_failure_appended_to_prompt(mock_key, mock_client, mock_call): - """prev_set3_failure is appended to the set3 advice prompt.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - {"decisions": [{"group_id": 0, "action": "stay"}]} - ) - - c = _classified( - entities=[_make_entity("bar", 1, 10)], - set_3_groups=[["bar"]], - ) - advise_file_limiter(c, "src/big.py", _CONFIG, prev_set3_failure="sentinel text") - - # messages is positional arg index 6 in call_with_tool - messages = mock_call.call_args[0][6] - assert "sentinel text" in messages[0]["content"] - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_placement_prev_failure_appended_to_prompt( - mock_key, mock_client, mock_call -): - """prev_placement_failure is appended to the assignment prompt (last call).""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _propose_ok("utils.py"), - _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), - ] - - c = _classified( - entities=[_make_entity("foo", 1, 10)], - set_2_groups=[["foo"]], - ) - advise_file_limiter( - c, "src/big.py", _CONFIG, prev_placement_failure="sentinel text" - ) - - assert mock_call.call_count == 2 # propose + assign - # The assign call is the last call; it receives prev_placement_failure. - messages = mock_call.call_args[0][6] - assert "sentinel text" in messages[0]["content"] - - -# --------------------------------------------------------------------------- -# Chunked placement calls (>_PLACEMENT_CHUNK_SIZE groups) -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_chunked_placement_makes_multiple_calls(mock_key, mock_client, mock_call): - """More than _PLACEMENT_CHUNK_SIZE groups → propose + multiple assign calls.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - - # Build _PLACEMENT_CHUNK_SIZE + 1 groups so two chunks are needed. - n = _PLACEMENT_CHUNK_SIZE + 1 - entities = [_make_entity(f"f{i}", i * 2 + 1, i * 2 + 2) for i in range(n)] - groups = [[f"f{i}"] for i in range(n)] - - # First chunk returns placements for group_ids 0..CHUNK_SIZE-1. - first_chunk_response = _make_llm_result( - { - "placements": [ - {"group_id": j, "target_file": "file_a.py"} - for j in range(_PLACEMENT_CHUNK_SIZE) - ] - } - ) - # Second chunk has 1 group (group_id 0) → goes to file_b.py (tiny, 2 lines). - second_chunk_response = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "file_b.py"}]} - ) - # Refinement: file_b.py is tiny (2 lines < 200), reassign to file_a.py. - refine_response = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "file_a.py"}]} - ) - - mock_call.side_effect = [ - _propose_ok("file_a.py", "file_b.py"), - first_chunk_response, - second_chunk_response, - refine_response, - ] - - c = _classified(entities=entities, set_2_groups=groups) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - - assert plan.abort is False - assert len(plan.placements) == n - # propose + chunk1 + chunk2 + refine = 4 calls. - assert mock_call.call_count == 4 - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_chunked_placement_second_chunk_fails_aborts( - mock_key, mock_client, mock_call, capsys -): - """Second chunk exhausts all per-chunk retries → placement returns None → abort.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - - n = _PLACEMENT_CHUNK_SIZE + 1 - entities = [_make_entity(f"f{i}", i * 2 + 1, i * 2 + 2) for i in range(n)] - groups = [[f"f{i}"] for i in range(n)] - - first_chunk_response = _make_llm_result( - { - "placements": [ - {"group_id": j, "target_file": "file_a.py"} - for j in range(_PLACEMENT_CHUNK_SIZE) - ] - } - ) - - cfg = CrispenConfig(file_limiter_retries=1) # 2 attempts per chunk - # propose + chunk 1 (1 call) + chunk 2 (2 failed attempts) = 4 calls. - mock_call.side_effect = [ - _propose_ok("file_a.py", "file_b.py"), - first_chunk_response, - _make_llm_result(None), - _make_llm_result(None), - ] - - c = _classified(entities=entities, set_2_groups=groups) - plan = advise_file_limiter(c, "src/big.py", cfg, verbose=True) - - assert plan.abort is True - assert "LLM failed to assign file placements" in plan.abort_reason - assert mock_call.call_count == 4 # propose + chunk1 + 2 failed chunk2 attempts - assert "failed to assign file placements" in capsys.readouterr().err - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_chunked_placement_chunk_retry_succeeds(mock_key, mock_client, mock_call): - """A chunk that fails once is retried; on success the full plan is returned.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - - n = _PLACEMENT_CHUNK_SIZE + 1 - entities = [_make_entity(f"f{i}", i * 2 + 1, i * 2 + 2) for i in range(n)] - groups = [[f"f{i}"] for i in range(n)] - - first_chunk_response = _make_llm_result( - { - "placements": [ - {"group_id": j, "target_file": "file_a.py"} - for j in range(_PLACEMENT_CHUNK_SIZE) - ] - } - ) - second_chunk_response = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "file_b.py"}]} - ) - # Refinement: file_b.py is tiny, reassign to file_a.py. - refine_response = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "file_a.py"}]} - ) - - cfg = CrispenConfig(file_limiter_retries=1) # 2 attempts per chunk - # propose + chunk1 + chunk2 (fail) + chunk2 (succeed) + refine = 5 calls. - mock_call.side_effect = [ - _propose_ok("file_a.py", "file_b.py"), - first_chunk_response, - _make_llm_result(None), - second_chunk_response, - refine_response, - ] - - c = _classified(entities=entities, set_2_groups=groups) - plan = advise_file_limiter(c, "src/big.py", cfg) - - assert plan.abort is False - assert len(plan.placements) == n - assert mock_call.call_count == 5 - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_plan_chunked_placement_zero_total_lines(mock_key, mock_client, mock_call): - """Groups whose names are absent from entity_map → total_lines==0 → fallback.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - - # Groups reference names not present in entity_map (entities=[]). - # Projected lines = 0 for all files → no tiny files → no refinement. - groups = [["orphan_a"], ["orphan_b"]] - mock_call.side_effect = [ - _propose_ok("a.py", "b.py"), - _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "a.py"}, - {"group_id": 1, "target_file": "b.py"}, - ] - } - ), - ] - - c = _classified(entities=[], set_2_groups=groups) - plan = advise_file_limiter(c, "src/big.py", _CONFIG) - - assert plan.abort is False - assert len(plan.placements) == 2 - assert mock_call.call_count == 2 # propose + assign (no refinement) - - -# --------------------------------------------------------------------------- -# _find_conflicting_placement_indices -# --------------------------------------------------------------------------- - - -def test_find_conflicting_idx_plan_vs_plan(): - """Flat file + subdir with same stem both appear → both indices returned.""" - placements = [ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="utils/io.py"), - GroupPlacement(group=["baz"], target_file="helpers.py"), - ] - idxs = _find_conflicting_placement_indices(placements, frozenset(), frozenset()) - assert idxs == [0, 1] - - -def test_find_conflicting_idx_file_vs_existing_dir(): - """Flat .py target whose stem matches an existing directory → index returned.""" - placements = [GroupPlacement(group=["foo"], target_file="models.py")] - idxs = _find_conflicting_placement_indices( - placements, frozenset(), frozenset({"models"}) - ) - assert idxs == [0] - - -def test_find_conflicting_idx_subdir_vs_existing_file(): - """Subdir target whose top matches an existing .py file → index returned.""" - placements = [GroupPlacement(group=["bar"], target_file="helpers/io.py")] - idxs = _find_conflicting_placement_indices( - placements, frozenset({"helpers.py"}), frozenset() - ) - assert idxs == [0] - - -def test_find_conflicting_idx_flat_target_in_existing_files(): - """Flat target in existing_files (e.g. conftest.py) → index returned.""" - placements = [ - GroupPlacement(group=["fix"], target_file="conftest.py"), - GroupPlacement(group=["bar"], target_file="helpers.py"), - ] - idxs = _find_conflicting_placement_indices( - placements, frozenset({"conftest.py"}), frozenset() - ) - assert idxs == [0] - - -def test_find_conflicting_idx_no_conflict(): - """Clean plan with no conflicts → empty list.""" - placements = [ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="helpers.py"), - ] - assert ( - _find_conflicting_placement_indices(placements, frozenset(), frozenset()) == [] - ) - - -# --------------------------------------------------------------------------- -# resolve_naming_conflicts — helpers shared by the block below -# --------------------------------------------------------------------------- - - -_CONFLICTING_PLACEMENTS = [ - GroupPlacement(group=["foo"], target_file="utils.py"), # plan-vs-plan conflict - GroupPlacement(group=["bar"], target_file="utils/io.py"), # plan-vs-plan conflict - GroupPlacement(group=["baz"], target_file="helpers.py"), # not conflicting -] - -_CLEAN_PLACEMENTS = [ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="helpers.py"), -] - - -# --------------------------------------------------------------------------- -# resolve_naming_conflicts — tests -# --------------------------------------------------------------------------- - - -def test_resolve_no_conflicts_returns_unchanged(): - """No conflicts → returns a copy of the input list; no LLM calls needed.""" - c = _classified(entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)]) - result = resolve_naming_conflicts( - _CLEAN_PLACEMENTS, c, "src/big.py", frozenset(), frozenset(), _CONFIG - ) - assert result == _CLEAN_PLACEMENTS - assert result is not _CLEAN_PLACEMENTS - - -def test_resolve_api_key_error_propagates(monkeypatch): - """Missing API key raises CrispenAPIError before any LLM call.""" - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - c = _classified(entities=[_make_entity("foo", 1, 5)]) - with pytest.raises(CrispenAPIError): - resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, c, "src/big.py", frozenset(), frozenset(), _CONFIG - ) - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_success(mock_key, mock_client, mock_call): - """Happy path: forbidden_dir_stems and existing_file_stems both non-empty; - prev_failure is False on the first (successful) attempt.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "models.py"}, - {"group_id": 1, "target_file": "services.py"}, - ] - } - ) - c = _classified( - entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)], - ) - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - existing_files=frozenset({"other.py"}), # non-empty → existing_file_stems - existing_dirs=frozenset({"mydir"}), # non-empty → forbidden_dir_stems - config=_CONFIG, - ) - assert result is not None - assert result[0].target_file == "models.py" - assert result[1].target_file == "services.py" - assert result[2].target_file == "helpers.py" # non-conflicting, unchanged - assert mock_call.call_count == 1 - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_llm_none_returns_none(mock_key, mock_client, mock_call): - """LLM returns None → resolve returns None.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result(None) - c = _classified() - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - frozenset(), - frozenset(), - CrispenConfig(file_limiter_retries=0), - ) - assert result is None - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_forbidden_target_returns_none(mock_key, mock_client, mock_call): - """LLM picks a target that is in forbidden_files → resolve returns None.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - # "helpers.py" is a non-conflicting target → included in forbidden_files. - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "helpers.py"}, - {"group_id": 1, "target_file": "services.py"}, - ] - } - ) - c = _classified() - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - frozenset(), - frozenset(), - CrispenConfig(file_limiter_retries=0), - ) - assert result is None - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_incomplete_response_returns_none(mock_key, mock_client, mock_call): - """LLM returns fewer placements than groups → len mismatch → None.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "models.py"}]} # only 1 of 2 - ) - c = _classified() - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - frozenset(), - frozenset(), - CrispenConfig(file_limiter_retries=0), - ) - assert result is None - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_retry_succeeds(mock_key, mock_client, mock_call): - """First attempt None, second succeeds; covers if prev_failure: True branch.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _make_llm_result(None), - _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "models.py"}, - {"group_id": 1, "target_file": "services.py"}, - ] - } - ), - ] - c = _classified() - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - frozenset(), - frozenset(), - CrispenConfig(file_limiter_retries=1), - ) - assert result is not None - assert result[0].target_file == "models.py" - assert mock_call.call_count == 2 - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_empty_forbidden_dir_stems(mock_key, mock_client, mock_call): - """existing_dirs empty → forbidden_dir_stems empty → branch False.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "models.py"}, - {"group_id": 1, "target_file": "services.py"}, - ] - } - ) - c = _classified() - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - existing_files=frozenset({"other.py"}), # file_stems non-empty - existing_dirs=frozenset(), # dir_stems empty - config=_CONFIG, - ) - assert result is not None - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_empty_existing_file_stems(mock_key, mock_client, mock_call): - """existing_files=frozenset() → file_stems empty → if existing_file_stems: False.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "models.py"}, - {"group_id": 1, "target_file": "services.py"}, - ] - } - ) - c = _classified() - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - existing_files=frozenset(), # file_stems empty - existing_dirs=frozenset({"mydir"}), # dir_stems non-empty - config=_CONFIG, - ) - assert result is not None - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_non_int_group_id(mock_key, mock_client, mock_call): - """Non-integer group_id → isinstance check fails → skipped → len mismatch → None.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": "zero", "target_file": "models.py"}, - {"group_id": 1, "target_file": "services.py"}, - ] - } - ) - c = _classified() - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - frozenset(), - frozenset(), - CrispenConfig(file_limiter_retries=0), - ) - assert result is None - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_out_of_range_group_id(mock_key, mock_client, mock_call): - """Out-of-range group_id → range check fails → skipped → len mismatch → None.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": 99, "target_file": "models.py"}, - {"group_id": 1, "target_file": "services.py"}, - ] - } - ) - c = _classified() - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - frozenset(), - frozenset(), - CrispenConfig(file_limiter_retries=0), - ) - assert result is None - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_duplicate_group_id(mock_key, mock_client, mock_call): - """Duplicate group_id → second entry skipped → len mismatch → None.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "models.py"}, - {"group_id": 0, "target_file": "other.py"}, # duplicate - ] - } - ) - c = _classified() - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - frozenset(), - frozenset(), - CrispenConfig(file_limiter_retries=0), - ) - assert result is None - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_empty_target(mock_key, mock_client, mock_call): - """Empty target_file → falsy check fails → skipped → len mismatch → None.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": ""}, - {"group_id": 1, "target_file": "services.py"}, - ] - } - ) - c = _classified() - result = resolve_naming_conflicts( - _CONFLICTING_PLACEMENTS, - c, - "src/big.py", - frozenset(), - frozenset(), - CrispenConfig(file_limiter_retries=0), - ) - assert result is None - - -# --------------------------------------------------------------------------- -# _group_summary — enriched descriptions -# --------------------------------------------------------------------------- - - -def test_group_summary_with_docstring_and_params(): - """Entity with docstring and params → both appear in summary.""" - ent = _make_entity( - "foo", - 1, - 10, - docstring="Parse the config file. More details here.", - params=["path: str", "strict: bool"], - ) - summary = _group_summary(["foo"], {"foo": ent}) - assert "foo (10 lines)" in summary - assert '"Parse the config file."' in summary - assert "params: path: str, strict: bool" in summary - - -def test_group_summary_with_params_only(): - """Entity with params but no docstring → params appear, no docstring quote.""" - ent = _make_entity("bar", 1, 5, params=["x: int", "y"]) - summary = _group_summary(["bar"], {"bar": ent}) - assert "params: x: int, y" in summary - assert '"' not in summary - - -def test_group_summary_docstring_no_period(): - """Docstring with no period → full text used as first sentence.""" - ent = _make_entity("baz", 1, 3, docstring="No period here") - summary = _group_summary(["baz"], {"baz": ent}) - assert '"No period here"' in summary - - -def test_group_summary_with_section_header(): - """Entity with section_header → section appears first in extras.""" - from crispen.file_limiter.entity_parser import Entity, EntityKind - - ent = Entity( - EntityKind.FUNCTION, - "foo", - 1, - 5, - ["foo"], - section_header="Helpers", - ) - summary = _group_summary(["foo"], {"foo": ent}) - assert 'section: "Helpers"' in summary - - -def test_group_summary_no_section_header(): - """Entity without section_header → no 'section:' in summary.""" - ent = _make_entity("bar", 1, 5) - summary = _group_summary(["bar"], {"bar": ent}) - assert "section:" not in summary - - -# --------------------------------------------------------------------------- -# _build_group_mermaid -# --------------------------------------------------------------------------- - - -def test_build_group_mermaid_no_edges(): - """Empty graph → no inter-group deps → returns empty string.""" - c = _classified(entities=[], set_2_groups=[["foo"], ["bar"]]) - result = _build_group_mermaid([["foo"], ["bar"]], c) - assert result == "" - - -def test_build_group_mermaid_with_inter_group_dep(): - """G0 depends on G1 → Mermaid text with that edge is returned.""" - c = _classified(graph={"foo": {"bar"}, "bar": set()}) - result = _build_group_mermaid([["foo"], ["bar"]], c) - assert "```mermaid" in result - assert "G0 --> G1" in result - - -def test_build_group_mermaid_dep_outside_chunk(): - """Dep to entity outside chunk → dep_gid is None → not added → empty.""" - c = _classified(graph={"foo": {"external"}, "bar": set()}) - result = _build_group_mermaid([["foo"], ["bar"]], c) - assert result == "" - - -def test_build_group_mermaid_intra_group_dep(): - """Dep within same SCC group → dep_gid == gid → not added as edge.""" - # foo and baz are in the same group; foo depends on baz (intra-SCC edge) - c = _classified(graph={"foo": {"baz"}, "baz": {"foo"}, "bar": set()}) - result = _build_group_mermaid([["foo", "baz"], ["bar"]], c) - assert result == "" - - -# --------------------------------------------------------------------------- -# Mermaid appears in assignment prompt when deps exist -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_placement_prompt_includes_mermaid_when_deps_exist( - mock_key, mock_client, mock_call -): - """Inter-group deps exist → Mermaid diagram included in the assignment prompt.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - mock_call.side_effect = [ - _propose_ok("utils.py", "models.py"), - _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "utils.py"}, - {"group_id": 1, "target_file": "models.py"}, - ] - } - ), - ] - c = _classified( - entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)], - set_2_groups=[["foo"], ["bar"]], - graph={"foo": {"bar"}, "bar": set()}, - ) - advise_file_limiter(c, "src/big.py", _CONFIG) - - # The assignment call is the last call; it has the Mermaid diagram. - messages = mock_call.call_args[0][6] - assert "```mermaid" in messages[0]["content"] - assert "G0 --> G1" in messages[0]["content"] - - -# --------------------------------------------------------------------------- -# verbose=True paths (covers print statements and _counter increment) -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_advise_verbose_set3_and_placement(mock_key, mock_client, mock_call, capsys): - """verbose=True exercises the print + _counter branches in _advise_set3, - _propose_files_step, and _assign_placements_chunk.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - # set3 call → propose call → assign call. - mock_call.side_effect = [ - _make_llm_result({"decisions": [{"group_id": 0, "action": "migrate"}]}), - _propose_ok("utils.py"), - _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), - ] - c = _classified( - entities=[_make_entity("bar", 1, 10)], - set_3_groups=[["bar"]], - ) - plan = advise_file_limiter(c, "src/big.py", _CONFIG, verbose=True) - - assert plan.abort is False - assert plan.llm_calls == 3 - err = capsys.readouterr().err - assert "set-3 group" in err - assert "file placements" in err - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_verbose(mock_key, mock_client, mock_call, capsys): - """verbose=True exercises the print + _counter branches in - _rename_conflicting_chunk (with _counter passed to cover the increment).""" - mock_key.return_value = "key" - # Both placements conflict (utils.py vs utils/io.py share stem "utils"), - # so the chunk sent to LLM has 2 groups; return both renamed. - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "models.py"}, - {"group_id": 1, "target_file": "helpers.py"}, - ] - } - ) - entity = _make_entity("foo", 1, 5) - c = _classified(entities=[entity]) - placements = [ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="utils/io.py"), # conflict - ] - acc = _LLMAccumulator() - result = resolve_naming_conflicts( - placements, - c, - "src/big.py", - frozenset(), - frozenset(), - _CONFIG, - verbose=True, - _acc=acc, - ) - - assert result is not None - assert acc.calls == 1 # one LLM call was counted - err = capsys.readouterr().err - assert "naming conflicts" in err - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_advise_verbose_detailed_timing_prints( - mock_key, mock_client, mock_call, capsys -): - """timing='detailed' prints per-call → done lines for set3, propose, and assign.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - # set3 call → propose call → assign call. - mock_call.side_effect = [ - _make_llm_result({"decisions": [{"group_id": 0, "action": "migrate"}]}), - _propose_ok("utils.py"), - _make_llm_result({"placements": [{"group_id": 0, "target_file": "utils.py"}]}), - ] - c = _classified( - entities=[_make_entity("bar", 1, 10)], - set_3_groups=[["bar"]], - ) - plan = advise_file_limiter( - c, "src/big.py", _CONFIG, verbose=True, timing="detailed" - ) - - assert plan.abort is False - err = capsys.readouterr().err - assert "→ done [" in err - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_resolve_verbose_detailed_timing_print( - mock_key, mock_client, mock_call, capsys -): - """timing='detailed' prints per-call → done line in resolve_naming_conflicts.""" - mock_key.return_value = "key" - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "models.py"}, - {"group_id": 1, "target_file": "helpers.py"}, - ] - } - ) - entity = _make_entity("foo", 1, 5) - c = _classified(entities=[entity]) - placements = [ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="utils/io.py"), # conflict - ] - acc = _LLMAccumulator() - result = resolve_naming_conflicts( - placements, - c, - "src/big.py", - frozenset(), - frozenset(), - _CONFIG, - verbose=True, - timing="detailed", - _acc=acc, - ) - - assert result is not None - err = capsys.readouterr().err - assert "→ done [" in err - - -@patch(_PATCH_CALL) -def test_advise_set3_no_counter(mock_call): - """_advise_set3 called without _counter covers the None-counter branch.""" - mock_call.return_value = _make_llm_result( - {"decisions": [{"group_id": 0, "action": "migrate"}]} - ) - c = _classified( - entities=[_make_entity("foo", 1, 5)], - set_3_groups=[["foo"]], - ) - result = _advise_set3(c, "big.py", MagicMock(), _CONFIG) - assert result == [["foo"]] - - -@patch(_PATCH_CALL) -def test_advise_set3_with_dep_graph(mock_call): - """_advise_set3 with inter-group dependencies includes mermaid graph in prompt.""" - mock_call.return_value = _make_llm_result( - {"decisions": [{"group_id": 0, "action": "migrate"}]} - ) - # graph["foo"] = {"bar"} means foo depends on bar → two groups have an edge - c = _classified( - entities=[_make_entity("foo", 1, 5), _make_entity("bar", 6, 10)], - graph={"foo": {"bar"}}, - set_3_groups=[["foo"], ["bar"]], - ) - result = _advise_set3(c, "big.py", MagicMock(), _CONFIG) - assert result == [["foo"]] - # Verify the mermaid graph was injected into the prompt. - prompt = mock_call.call_args[0][6][0]["content"] - assert "graph TD" in prompt - - -@patch(_PATCH_CALL) -def test_assign_placements_chunk_no_counter(mock_call): - """_assign_placements_chunk without _counter covers the None-counter branch.""" - mock_call.return_value = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "utils.py"}]} - ) - c = _classified(entities=[_make_entity("foo", 1, 5)]) - result = _assign_placements_chunk( - [["foo"]], c, "big.py", frozenset(), MagicMock(), _CONFIG - ) - assert result is not None - assert result[0].target_file == "utils.py" - - -@patch(_PATCH_CALL) -def test_assign_placements_chunk_subdir_name(mock_call): - """subdir_name is included in the prompt and suppresses the plain directory rule.""" - mock_call.return_value = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "detection_flow.py"}]} - ) - c = _classified(entities=[_make_entity("foo", 1, 5)]) - result = _assign_placements_chunk( - [["foo"]], - c, - "tests/test_duplicate_extractor.py", - frozenset(), - MagicMock(), - _CONFIG, - subdir_name="duplicate_extractor", - ) - assert result is not None - assert result[0].target_file == "detection_flow.py" - # The prompt should mention the subdirectory and warn against repeating its name. - prompt = mock_call.call_args[0][6][0]["content"] - assert "duplicate_extractor/" in prompt - assert "do not repeat" in prompt.lower() - - -@patch(_PATCH_CALL) -def test_assign_placements_chunk_strips_subdir_prefix(mock_call): - """LLM returns 'subdir/file.py' — the leading subdir/ should be stripped.""" - mock_call.return_value = _make_llm_result( - { - "placements": [ - {"group_id": 0, "target_file": "duplicate_extractor/detection_flow.py"} - ] - } - ) - c = _classified(entities=[_make_entity("foo", 1, 5)]) - result = _assign_placements_chunk( - [["foo"]], - c, - "tests/test_duplicate_extractor.py", - frozenset(), - MagicMock(), - _CONFIG, - subdir_name="duplicate_extractor", - ) - assert result is not None - assert result[0].target_file == "detection_flow.py" - - -# --------------------------------------------------------------------------- -# _assign_placements_chunk — constrained mode (proposed_files provided) -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -def test_assign_placements_chunk_constrained_success(mock_call): - """Constrained mode: target in proposed_filenames → placement accepted.""" - mock_call.return_value = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "utils.py"}]} - ) - c = _classified(entities=[_make_entity("foo", 1, 5)]) - proposed = [("utils.py", "general utilities"), ("models.py", "data models")] - result = _assign_placements_chunk( - [["foo"]], - c, - "src/big.py", - frozenset(), - MagicMock(), - _CONFIG, - proposed_files=proposed, - ) - assert result is not None - assert result[0].target_file == "utils.py" - # Prompt should list proposed files and instruct constrained choice. - prompt = mock_call.call_args[0][6][0]["content"] - assert "Proposed output files" in prompt - assert "utils.py" in prompt - assert "models.py" in prompt - - -@patch(_PATCH_CALL) -def test_assign_placements_chunk_constrained_invalid_target(mock_call): - """Constrained mode: target not in proposed_filenames → immediate None return.""" - mock_call.return_value = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "rogue_file.py"}]} - ) - c = _classified(entities=[_make_entity("foo", 1, 5)]) - proposed = [("utils.py", "general utilities")] - result = _assign_placements_chunk( - [["foo"]], - c, - "src/big.py", - frozenset(), - MagicMock(), - _CONFIG, - proposed_files=proposed, - ) - assert result is None - - -# --------------------------------------------------------------------------- -# _propose_files_step — unit tests -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -def test_propose_files_step_success(mock_call): - """Basic success: valid filenames are returned.""" - mock_call.return_value = _make_llm_result( - { - "files": [ - {"filename": "utils.py", "description": "utility functions"}, - {"filename": "models.py", "description": "data models"}, - ] - } - ) - c = _classified(entities=[_make_entity("foo", 1, 50)]) - result = _propose_files_step( - [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG - ) - assert result is not None - assert len(result) == 2 - assert result[0] == ("utils.py", "utility functions") - assert result[1] == ("models.py", "data models") - - -@patch(_PATCH_CALL) -def test_propose_files_step_llm_none(mock_call): - """call_with_tool returns None → _propose_files_step returns None.""" - mock_call.return_value = _make_llm_result(None) - c = _classified() - result = _propose_files_step( - [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG - ) - assert result is None - - -@patch(_PATCH_CALL) -def test_propose_files_step_empty_files_list(mock_call): - """LLM returns empty files list → returns None (not proposed).""" - mock_call.return_value = _make_llm_result({"files": []}) - c = _classified() - result = _propose_files_step( - [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG - ) - assert result is None - - -@patch(_PATCH_CALL) -def test_propose_files_step_strips_existing_files(mock_call): - """Filename in existing_files is stripped; remaining valid ones returned.""" - mock_call.return_value = _make_llm_result( - { - "files": [ - {"filename": "taken.py", "description": "already exists"}, - {"filename": "utils.py", "description": "new file"}, - ] - } - ) - c = _classified() - result = _propose_files_step( - [["foo"]], - c, - "src/big.py", - 2, - frozenset({"taken.py"}), - MagicMock(), - _CONFIG, - ) - assert result is not None - assert len(result) == 1 - assert result[0][0] == "utils.py" - - -@patch(_PATCH_CALL) -def test_propose_files_step_all_in_existing_files(mock_call): - """All proposed filenames are in existing_files → stripped → returns None.""" - mock_call.return_value = _make_llm_result( - {"files": [{"filename": "taken.py", "description": "existing"}]} - ) - c = _classified() - result = _propose_files_step( - [["foo"]], - c, - "src/big.py", - 2, - frozenset({"taken.py"}), - MagicMock(), - _CONFIG, - ) - assert result is None - - -@patch(_PATCH_CALL) -def test_propose_files_step_strips_duplicates(mock_call): - """Duplicate filenames are stripped; only first occurrence kept.""" - mock_call.return_value = _make_llm_result( - { - "files": [ - {"filename": "utils.py", "description": "first"}, - {"filename": "utils.py", "description": "duplicate"}, - ] - } - ) - c = _classified() - result = _propose_files_step( - [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG - ) - assert result is not None - assert len(result) == 1 - assert result[0] == ("utils.py", "first") - - -@patch(_PATCH_CALL) -def test_propose_files_step_strips_empty_filename(mock_call): - """Empty filename string is skipped; valid ones returned.""" - mock_call.return_value = _make_llm_result( - { - "files": [ - {"filename": "", "description": "empty"}, - {"filename": "utils.py", "description": "valid"}, - ] - } - ) - c = _classified() - result = _propose_files_step( - [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG - ) - assert result is not None - assert len(result) == 1 - assert result[0][0] == "utils.py" - - -@patch(_PATCH_CALL) -def test_propose_files_step_verbose(mock_call, capsys): - """verbose=True prints propose message to stderr and increments counter.""" - mock_call.return_value = _make_llm_result( - {"files": [{"filename": "utils.py", "description": "utilities"}]} - ) - c = _classified() - acc = _LLMAccumulator() - result = _propose_files_step( - [["foo"]], - c, - "src/big.py", - 2, - frozenset(), - MagicMock(), - _CONFIG, - verbose=True, - _acc=acc, - ) - assert result is not None - assert acc.calls == 1 - err = capsys.readouterr().err - assert "propose" in err.lower() - - -@patch(_PATCH_CALL) -def test_propose_files_step_no_counter(mock_call): - """_counter=None covers the None-counter branch (no increment).""" - mock_call.return_value = _make_llm_result( - {"files": [{"filename": "utils.py", "description": "utilities"}]} - ) - c = _classified() - result = _propose_files_step( - [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG - ) - assert result is not None - - -@patch(_PATCH_CALL) -def test_propose_files_step_subdir_name(mock_call): - """subdir_name triggers the subdir placement_rule branch in the prompt.""" - mock_call.return_value = _make_llm_result( - {"files": [{"filename": "handlers.py", "description": "request handlers"}]} - ) - c = _classified() - result = _propose_files_step( - [["foo"]], - c, - "src/service.py", - 2, - frozenset(), - MagicMock(), - _CONFIG, - subdir_name="service", - ) - assert result is not None - prompt = mock_call.call_args[0][6][0]["content"] - assert "service/" in prompt - - -@patch(_PATCH_CALL) -def test_propose_files_step_no_existing_files(mock_call): - """existing_files=frozenset() → exclude_section empty (branch False).""" - mock_call.return_value = _make_llm_result( - {"files": [{"filename": "utils.py", "description": "utilities"}]} - ) - c = _classified() - result = _propose_files_step( - [["foo"]], c, "src/big.py", 2, frozenset(), MagicMock(), _CONFIG - ) - prompt = mock_call.call_args[0][6][0]["content"] - assert "already exist" not in prompt - assert result is not None - - -@patch(_PATCH_CALL) -def test_propose_files_step_with_existing_files(mock_call): - """existing_files non-empty → exclude_section added to prompt (branch True).""" - mock_call.return_value = _make_llm_result( - {"files": [{"filename": "utils.py", "description": "utilities"}]} - ) - c = _classified() - result = _propose_files_step( - [["foo"]], - c, - "src/big.py", - 2, - frozenset({"other.py"}), - MagicMock(), - _CONFIG, - ) - prompt = mock_call.call_args[0][6][0]["content"] - assert "already exist" in prompt - assert result is not None - - -@patch(_PATCH_CALL) -def test_propose_files_step_prev_failure(mock_call): - """prev_failure is appended to the propose prompt.""" - mock_call.return_value = _make_llm_result( - {"files": [{"filename": "utils.py", "description": "utilities"}]} - ) - c = _classified() - _propose_files_step( - [["foo"]], - c, - "src/big.py", - 2, - frozenset(), - MagicMock(), - _CONFIG, - prev_failure="sentinel_propose_failure", - ) - prompt = mock_call.call_args[0][6][0]["content"] - assert "sentinel_propose_failure" in prompt - - -# --------------------------------------------------------------------------- -# _compute_projected_lines — unit tests -# --------------------------------------------------------------------------- - - -def test_compute_projected_lines_basic(): - """Entities found in map → lines counted per target file.""" - entity_a = _make_entity("func_a", 1, 50) # 50 lines - entity_b = _make_entity("func_b", 51, 100) # 50 lines - entity_map = {"func_a": entity_a, "func_b": entity_b} - placements = [ - GroupPlacement(group=["func_a"], target_file="utils.py"), - GroupPlacement(group=["func_b"], target_file="utils.py"), - ] - projected = _compute_projected_lines(placements, entity_map) - assert projected == {"utils.py": 100} - - -def test_compute_projected_lines_unknown_entity(): - """Entity name not in map → no lines added for that entity (skipped).""" - entity_map = {} # nothing in the map - placements = [GroupPlacement(group=["ghost"], target_file="utils.py")] - projected = _compute_projected_lines(placements, entity_map) - assert projected == {} - - -def test_compute_projected_lines_multiple_files(): - """Entities across multiple target files → separate line counts.""" - entity_a = _make_entity("func_a", 1, 100) # 100 lines - entity_b = _make_entity("func_b", 101, 200) # 100 lines - entity_map = {"func_a": entity_a, "func_b": entity_b} - placements = [ - GroupPlacement(group=["func_a"], target_file="module_a.py"), - GroupPlacement(group=["func_b"], target_file="module_b.py"), - ] - projected = _compute_projected_lines(placements, entity_map) - assert projected == {"module_a.py": 100, "module_b.py": 100} - - -# --------------------------------------------------------------------------- -# _refine_merge_tiny — unit tests -# --------------------------------------------------------------------------- - - -def test_refine_merge_tiny_no_tiny_files(): - """All projected files are above the tiny threshold → no merge, return unchanged.""" - # Entity with 300 lines is well above min_size (200 for 1000-line limit). - entity = _make_entity("large_func", 1, 300) - c = _classified(entities=[entity]) - placements = [GroupPlacement(group=["large_func"], target_file="utils.py")] - proposed_files = [("utils.py", "large functions"), ("models.py", "models")] - - result = _refine_merge_tiny( - placements, proposed_files, c, "src/big.py", MagicMock(), _CONFIG - ) - assert result == placements - assert result is not placements - - -def test_refine_merge_tiny_no_ok_proposed(): - """All proposed files are tiny → ok_proposed is empty → return unchanged.""" - # Two tiny entities, both below threshold. - entity_a = _make_entity("tiny_a", 1, 10) - entity_b = _make_entity("tiny_b", 11, 20) - c = _classified(entities=[entity_a, entity_b]) - placements = [ - GroupPlacement(group=["tiny_a"], target_file="a.py"), - GroupPlacement(group=["tiny_b"], target_file="b.py"), - ] - proposed_files = [("a.py", "tiny a"), ("b.py", "tiny b")] - # Both files are tiny (10 and 10 lines < 200); no ok_proposed → no merge. - - result = _refine_merge_tiny( - placements, proposed_files, c, "src/big.py", MagicMock(), _CONFIG - ) - assert result == placements - - -@patch(_PATCH_CALL) -def test_refine_merge_tiny_success(mock_call): - """Tiny file group is merged into a larger file successfully.""" - # LLM reassigns the tiny group to the large file. - mock_call.return_value = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "large.py"}]} - ) - - entity_small = _make_entity("small_func", 1, 10) # 10 lines (tiny) - entity_large = _make_entity("large_func", 11, 310) # 300 lines (not tiny) - c = _classified(entities=[entity_small, entity_large]) - - placements = [ - GroupPlacement(group=["small_func"], target_file="small.py"), - GroupPlacement(group=["large_func"], target_file="large.py"), - ] - proposed_files = [("small.py", "small"), ("large.py", "large")] - # small.py: 10 lines (tiny <200); large.py: 300 lines (not tiny). - # ok_proposed = [("large.py", "large")]; refinement merges small.py into large.py. - - result = _refine_merge_tiny( - placements, proposed_files, c, "src/big.py", MagicMock(), _CONFIG - ) - assert len(result) == 2 - # small_func should now be in large.py. - small_placement = next(r for r in result if "small_func" in r.group) - assert small_placement.target_file == "large.py" - # large_func remains in large.py. - large_placement = next(r for r in result if "large_func" in r.group) - assert large_placement.target_file == "large.py" - - -@patch(_PATCH_CALL) -def test_refine_merge_tiny_llm_fails(mock_call): - """Reassignment LLM returns None → original placements returned (best-effort).""" - mock_call.return_value = _make_llm_result(None) # LLM fails - - entity_small = _make_entity("small_func", 1, 10) - entity_large = _make_entity("large_func", 11, 310) - c = _classified(entities=[entity_small, entity_large]) - - placements = [ - GroupPlacement(group=["small_func"], target_file="small.py"), - GroupPlacement(group=["large_func"], target_file="large.py"), - ] - proposed_files = [("small.py", "small"), ("large.py", "large")] - - result = _refine_merge_tiny( - placements, proposed_files, c, "src/big.py", MagicMock(), _CONFIG - ) - # Best-effort: return original placements unchanged. - assert len(result) == 2 - assert result[0].target_file == "small.py" - assert result[1].target_file == "large.py" - - -@patch(_PATCH_CALL) -def test_refine_merge_tiny_verbose(mock_call, capsys): - """verbose=True prints the refining message to stderr.""" - mock_call.return_value = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "large.py"}]} - ) - - entity_small = _make_entity("small_func", 1, 10) - entity_large = _make_entity("large_func", 11, 310) - c = _classified(entities=[entity_small, entity_large]) - - placements = [ - GroupPlacement(group=["small_func"], target_file="small.py"), - GroupPlacement(group=["large_func"], target_file="large.py"), - ] - proposed_files = [("small.py", "small"), ("large.py", "large")] - - _refine_merge_tiny( - placements, - proposed_files, - c, - "src/big.py", - MagicMock(), - _CONFIG, - verbose=True, - ) - err = capsys.readouterr().err - assert "refining" in err.lower() - - -# --------------------------------------------------------------------------- -# Coverage gap: free-form _assign_placements_chunk with existing_files -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -def test_assign_placements_chunk_existing_files_exclude_section(mock_call): - """Free-form mode with non-empty existing_files builds the exclude section.""" - mock_call.return_value = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "helpers.py"}]} - ) - entity = _make_entity("foo", 1, 10) - c = _classified(entities=[entity], set_2_groups=[["foo"]]) - result = _assign_placements_chunk( - [["foo"]], - c, - "src/big.py", - frozenset({"existing.py"}), # non-empty existing_files - MagicMock(), - _CONFIG, - proposed_files=None, # free-form mode - ) - assert result is not None - assert result[0].target_file == "helpers.py" - - -@patch(_PATCH_CALL) -def test_assign_placements_chunk_target_in_existing_files_returns_none(mock_call): - """Free-form mode: target_file in existing_files → return None (line 589).""" - mock_call.return_value = _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "existing.py"}]} - ) - entity = _make_entity("foo", 1, 10) - c = _classified(entities=[entity], set_2_groups=[["foo"]]) - result = _assign_placements_chunk( - [["foo"]], - c, - "src/big.py", - frozenset({"existing.py"}), # target collides with existing file - MagicMock(), - _CONFIG, - proposed_files=None, # free-form mode - ) - assert result is None - - -# --------------------------------------------------------------------------- -# Coverage gap: propose retry loop in _assign_placements -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_propose_retry_succeeds_on_second_attempt(mock_key, mock_client, mock_call): - """Propose returns None once, then succeeds on retry (lines 862->882, 878).""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - entity = _make_entity("foo", 1, 50) - c = _classified(entities=[entity], set_2_groups=[["foo"]]) - mock_call.side_effect = [ - _make_llm_result(None), # propose fails first attempt - _propose_ok("helpers.py"), # propose succeeds on retry - _make_llm_result( - {"placements": [{"group_id": 0, "target_file": "helpers.py"}]} - ), # assign - # no refinement: 50 lines is not tiny (>= 200 is fine, 50 < 200 but only file) - ] - plan = advise_file_limiter( - c, - "src/big.py", - CrispenConfig(file_limiter_retries=1), # allow 1 retry - ) - assert plan.abort is False - assert plan.placements[0].target_file == "helpers.py" - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_propose_all_retries_exhausted_aborts(mock_key, mock_client, mock_call): - """All propose retries fail → _assign_placements returns None → abort (line 883).""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - entity = _make_entity("foo", 1, 50) - c = _classified(entities=[entity], set_2_groups=[["foo"]]) - mock_call.return_value = _make_llm_result(None) # propose always fails - plan = advise_file_limiter( - c, - "src/big.py", - CrispenConfig(file_limiter_retries=0), # no retries - ) - assert plan.abort is True - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_propose_no_tool_call_verbose(mock_key, mock_client, mock_call, capsys): - """tool_input=None + verbose=True → logs 'no tool call in response'.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - entity = _make_entity("foo", 1, 50) - c = _classified(entities=[entity], set_2_groups=[["foo"]]) - mock_call.return_value = _make_llm_result(None) - plan = advise_file_limiter( - c, "src/big.py", CrispenConfig(file_limiter_retries=0), verbose=True - ) - assert plan.abort is True - assert "no tool call" in capsys.readouterr().err - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_propose_empty_files_list_verbose(mock_key, mock_client, mock_call, capsys): - """tool_input={"files": []} + verbose=True → logs 'empty files list'.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - entity = _make_entity("foo", 1, 50) - c = _classified(entities=[entity], set_2_groups=[["foo"]]) - mock_call.return_value = _make_llm_result({"files": []}) - plan = advise_file_limiter( - c, "src/big.py", CrispenConfig(file_limiter_retries=0), verbose=True - ) - assert plan.abort is True - assert "empty files list" in capsys.readouterr().err - - -@patch(_PATCH_CALL) -@patch(_PATCH_CLIENT) -@patch(_PATCH_KEY) -def test_propose_all_filenames_filtered_verbose( - mock_key, mock_client, mock_call, capsys -): - """All proposed filenames in existing_files + verbose=True → logs filtered names.""" - mock_key.return_value = "key" - mock_client.return_value = MagicMock() - entity = _make_entity("foo", 1, 50) - c = _classified(entities=[entity], set_2_groups=[["foo"]]) - # Propose "taken.py" which is already in existing_files. - mock_call.return_value = _make_llm_result( - {"files": [{"filename": "taken.py", "description": "existing"}]} - ) - plan = advise_file_limiter( - c, - "src/big.py", - CrispenConfig(file_limiter_retries=0), - existing_files=frozenset({"taken.py"}), - verbose=True, - ) - assert plan.abort is True - assert "filtered" in capsys.readouterr().err diff --git a/tests/test_code_gen.py b/tests/test_code_gen.py index 9f8296d..2cc5605 100644 --- a/tests/test_code_gen.py +++ b/tests/test_code_gen.py @@ -2,6492 +2,10 @@ from __future__ import annotations -import textwrap -from crispen.file_limiter.advisor import FileLimiterPlan, GroupPlacement -from crispen.file_limiter.classifier import ClassifiedEntities -from crispen.import_sort import _sort_imports_pep8 -from crispen.file_limiter.code_gen import ( - ImportInfo, - _abs_package_for_dir, - _add_re_exports, - _bump_relative_imports, - _class_has_test_methods, - _collect_external_imported_names, - _collect_name_loads, - _collect_quoted_annotation_names, - _collect_name_stores, - _inject_module_level_imports, - _inject_type_checking_imports, - _test_names_in_decorators, - _extract_import_info, - _extract_module_docstring, - _extract_shared_helpers, - _find_cross_file_imports, - _find_cross_file_type_checking_imports, - _module_import_stmt, - _rewrite_module_level_stores, - _rewrite_module_var_names, - _find_main_block_entity, - _find_main_direct_callees, - _find_needed_imports, - _find_type_checking_needed_imports, - _narrow_import_source, - _find_project_root, - _import_derived_names, - _import_line_numbers, - _inject_inline_imports, - _inject_inline_test_imports_original, - _file_has_only_fixtures, - _is_pytest_fixture, - _is_test_name, - _merge_conftest_sources, - _merge_from_imports, - _module_path_from_file, - _prune_inline_redundant_imports, - _prune_unused_imports, - _relative_import_prefix, - _remove_entity_lines, - _split_cross_imports_by_test, - _source_is_only_docstring, - _strip_module_docstring, - _multiline_string_ranges, - _normalize_blank_lines, - _sub_skip_strings, - _strip_orphaned_indented_comments, - _strip_orphaned_section_headers, - _strip_top_level_import_lines, - _target_module_name, - _topo_depth, - generate_file_splits, -) -from crispen.file_limiter.entity_parser import Entity, EntityKind - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_entity(name: str, start: int, end: int, defines=None) -> Entity: - return Entity(EntityKind.FUNCTION, name, start, end, defines or [name]) - - -def _classified( - *, entities=None, set_2_groups=None, set_3_groups=None -) -> ClassifiedEntities: - return ClassifiedEntities( - entities=entities or [], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=set_2_groups or [], - set_3_groups=set_3_groups or [], - abort=False, - ) - - -def _plan(placements=None) -> FileLimiterPlan: - return FileLimiterPlan(set3_migrate=[], placements=placements or [], abort=False) - - -def _abort_plan() -> FileLimiterPlan: - return FileLimiterPlan(set3_migrate=[], placements=[], abort=True) - - -# --------------------------------------------------------------------------- -# _collect_name_loads -# --------------------------------------------------------------------------- - - -def test_collect_name_loads_basic(): - source = "x = foo + bar" - names = _collect_name_loads(source) - assert "foo" in names - assert "bar" in names - - -def test_collect_name_loads_store_not_included(): - source = "x = 1" - names = _collect_name_loads(source) - # x is a Store, not a Load - assert "x" not in names - - -def test_collect_name_loads_syntax_error(): - assert _collect_name_loads("def (invalid") == set() - - -def test_collect_name_loads_excludes_function_params(): - # 'client' is a parameter of test_foo — excluded from loads inside the body. - source = "def test_foo(client):\n client.call()\n" - names = _collect_name_loads(source) - assert "client" not in names - - -def test_collect_name_loads_includes_non_param_name(): - # 'helper' is not a parameter of test_foo — still counted as a load. - source = "def test_foo(client):\n helper(client)\n" - names = _collect_name_loads(source) - assert "helper" in names - assert "client" not in names - - -def test_collect_name_loads_excludes_nested_function_params(): - # Inner function params are excluded only within that function's own body. - source = textwrap.dedent( - """\ - def outer(x): - def inner(y): - return y + x - return inner - """ - ) - names = _collect_name_loads(source) - assert "y" not in names # param of inner — excluded inside inner body - assert "x" not in names # param of outer — excluded inside outer body - - -def test_collect_name_loads_includes_annotation_names(): - # Type annotations are in the outer scope — their names are included. - source = "def f(x: MyType) -> ReturnType:\n pass\n" - names = _collect_name_loads(source) - assert "MyType" in names - assert "ReturnType" in names - assert "x" not in names # param name itself, not counted - - -def test_collect_name_loads_includes_decorator_names(): - # Decorator expressions are in the outer scope. - source = "@pytest.fixture\ndef client():\n pass\n" - names = _collect_name_loads(source) - assert "pytest" in names - - -def test_collect_name_loads_kw_defaults_none_skipped(): - # kw_defaults may contain None for keyword-only args without defaults. - # None entries must not cause a crash and are simply skipped. - source = "def f(*, a, b=DEFAULT):\n pass\n" - names = _collect_name_loads(source) - assert "DEFAULT" in names - assert "a" not in names - assert "b" not in names - - -def test_collect_name_loads_annotated_vararg_kwarg(): - # *args: T and **kwargs: T annotations are in the outer scope. - source = "def f(*args: VarType, **kwargs: KwType):\n pass\n" - names = _collect_name_loads(source) - assert "VarType" in names - assert "KwType" in names - - -def test_collect_name_loads_excludes_local_variable_assignments(): - # A name assigned in the function body is a local variable — not an import. - # Loads of that name (e.g. attribute access) must not generate cross-file imports. - source = textwrap.dedent( - """\ - def test_foo(tmp_path): - helpers = tmp_path / "helpers.py" - helpers.write_text("X = 1", encoding="utf-8") - assert str(helpers.resolve()) == "x" - """ - ) - names = _collect_name_loads(source) - assert "helpers" not in names - assert "tmp_path" not in names # also a param — still excluded - - -def test_collect_name_loads_local_store_does_not_suppress_outer_loads(): - # A local assignment in an inner function must not suppress the outer scope's load. - source = textwrap.dedent( - """\ - def outer(): - use(helper) - def inner(): - helper = 1 - use(helper) - """ - ) - names = _collect_name_loads(source) - # outer() loads 'helper' (not locally defined there); inner() assigns it locally. - assert "helper" in names - - -# --------------------------------------------------------------------------- -# _collect_quoted_annotation_names -# --------------------------------------------------------------------------- - - -def test_collect_quoted_annotation_names_basic(): - # "MyType" in a string annotation → detected. - source = 'def f(x: "MyType") -> None:\n pass\n' - names = _collect_quoted_annotation_names(source) - assert "MyType" in names - - -def test_collect_quoted_annotation_names_optional(): - # Optional["_LLMAccumulator"] — the inner string is parsed. - source = 'def f(x: Optional["_LLMAccumulator"]) -> None:\n pass\n' - names = _collect_quoted_annotation_names(source) - assert "_LLMAccumulator" in names - - -def test_collect_quoted_annotation_names_return(): - # Quoted return annotation. - source = 'def f() -> "ReturnType":\n pass\n' - names = _collect_quoted_annotation_names(source) - assert "ReturnType" in names - - -def test_collect_quoted_annotation_names_annassign(): - # Variable annotation: x: "MyClass" - source = 'x: "MyClass"\n' - names = _collect_quoted_annotation_names(source) - assert "MyClass" in names - - -def test_collect_quoted_annotation_names_unquoted_not_included(): - # Normal (unquoted) annotation names are NOT returned by this function. - source = "def f(x: MyType) -> None:\n pass\n" - names = _collect_quoted_annotation_names(source) - assert "MyType" not in names - - -def test_collect_quoted_annotation_names_syntax_error(): - # Unparseable source returns empty set (no crash). - assert _collect_quoted_annotation_names("def (invalid") == set() - - -def test_collect_quoted_annotation_names_inner_syntax_error(): - # A string annotation that isn't valid Python is silently ignored. - source = 'def f(x: "not valid python !!") -> None:\n pass\n' - names = _collect_quoted_annotation_names(source) - assert names == set() - - -def test_collect_quoted_annotation_names_vararg_kwarg(): - # *args and **kwargs with quoted annotations. - source = 'def f(*args: "VarType", **kwargs: "KwType") -> None:\n pass\n' - names = _collect_quoted_annotation_names(source) - assert "VarType" in names - assert "KwType" in names - - -def test_collect_quoted_annotation_names_annassign_with_value(): - # x: "MyClass" = SomeFactory() — annotation has quoted name AND there is a value. - # The _walk branch for AnnAssign with node.value must execute. - source = 'x: "MyClass" = object()\n' - names = _collect_quoted_annotation_names(source) - assert "MyClass" in names - - -# --------------------------------------------------------------------------- -# _collect_name_stores -# --------------------------------------------------------------------------- - - -def test_collect_name_stores_simple_assign(): - assert _collect_name_stores("X = 1\n") == {"X"} - - -def test_collect_name_stores_multiple_assigns(): - src = "X = 1\nY = 2\n" - assert _collect_name_stores(src) == {"X", "Y"} - - -def test_collect_name_stores_augassign(): - assert _collect_name_stores("X += 1\n") == {"X"} - - -def test_collect_name_stores_annotated_assign_with_value(): - assert _collect_name_stores("X: int = 42\n") == {"X"} - - -def test_collect_name_stores_annotated_assign_without_value(): - # Declaration only (no assignment) — not a store. - assert _collect_name_stores("X: int\n") == set() - - -def test_collect_name_stores_function_body_not_included(): - # Assignments inside function bodies are not module-level stores. - src = "def f():\n X = 1\n" - assert _collect_name_stores(src) == set() - - -def test_collect_name_stores_load_not_included(): - assert _collect_name_stores("y = X\n") == {"y"} - assert "X" not in _collect_name_stores("y = X\n") - - -def test_collect_name_stores_syntax_error(): - assert _collect_name_stores("def (broken:\n") == set() - - -def test_collect_name_stores_empty(): - assert _collect_name_stores("") == set() - - -def test_collect_name_stores_non_name_assign_target(): - # Tuple-unpacking targets are not plain Name nodes — must not crash. - src = "a, b = 1, 2\n" - result = _collect_name_stores(src) - assert "a" not in result # tuple target, not a plain Name store - assert "b" not in result - - -def test_collect_name_stores_non_name_augassign_target(): - # Attribute augmented assignment — target is Attribute, not Name. - src = "obj.x += 1\n" - result = _collect_name_stores(src) - assert result == set() - - -# --------------------------------------------------------------------------- -# _inject_module_level_imports -# --------------------------------------------------------------------------- - - -def test_inject_module_level_imports_docstring_only(): - # Source with only a docstring and no imports — insert after the docstring. - src = '"""Module doc."""\n\nx = 1\n' - result = _inject_module_level_imports(src, ["from . import converters"]) - assert '"""Module doc."""' in result - assert "from . import converters" in result - doc_pos = result.index('"""Module doc."""') - imp_pos = result.index("from . import converters") - assert doc_pos < imp_pos - - -def test_inject_module_level_imports_empty_list(): - src = "x = 1\n" - assert _inject_module_level_imports(src, []) == src - - -def test_inject_module_level_imports_after_imports(): - src = "import os\n\nx = 1\n" - result = _inject_module_level_imports(src, ["from . import converters"]) - assert result == "import os\nfrom . import converters\n\nx = 1\n" - - -def test_inject_module_level_imports_no_existing_imports(): - src = "x = 1\n" - result = _inject_module_level_imports(src, ["from . import converters"]) - # Prepended before non-import content - assert "from . import converters" in result - assert result.index("from . import converters") < result.index("x = 1") - - -def test_inject_module_level_imports_sorted(): - src = "import os\n\nx = 1\n" - result = _inject_module_level_imports( - src, ["from . import z_mod", "from . import a_mod"] - ) - lines = result.splitlines() - import_lines = [ln for ln in lines if "import" in ln] - assert import_lines.index("from . import a_mod") < import_lines.index( - "from . import z_mod" - ) - - -def test_inject_module_level_imports_syntax_error_prepends(): - src = "def (broken:\n" - result = _inject_module_level_imports(src, ["import os"]) - assert result.startswith("import os\n") - - -# --------------------------------------------------------------------------- -# _inject_type_checking_imports -# --------------------------------------------------------------------------- - - -def test_inject_type_checking_imports_empty_list(): - src = "import os\n" - assert _inject_type_checking_imports(src, []) == src - - -def test_inject_type_checking_imports_syntax_error(): - src = "def (broken:\n" - assert _inject_type_checking_imports(src, ["from .config import Cfg"]) == src - - -def test_inject_type_checking_imports_all_already_present(): - # If every requested import is already in an existing TC block, no change. - src = ( - "from typing import TYPE_CHECKING\n" - "if TYPE_CHECKING:\n" - " from .config import Cfg\n" - "\n" - "x = 1\n" - ) - result = _inject_type_checking_imports(src, ["from .config import Cfg"]) - assert result == src - - -def test_inject_type_checking_imports_appends_to_existing_block(): - # New import should be appended inside the existing TYPE_CHECKING block. - src = ( - "from typing import TYPE_CHECKING\n" - "if TYPE_CHECKING:\n" - " from .config import Cfg\n" - "\n" - "x = 1\n" - ) - result = _inject_type_checking_imports(src, ["from .models import MyModel"]) - assert "from .models import MyModel" in result - tc_start = result.index("if TYPE_CHECKING:") - assert result.index("from .models import MyModel") > tc_start - assert "x = 1" in result - - -def test_inject_type_checking_imports_creates_block_with_typing_import(): - # No existing TC block and TYPE_CHECKING not imported → add both. - src = "from typing import List\n\ndef foo(x: 'Cfg') -> None:\n pass\n" - result = _inject_type_checking_imports(src, ["from .config import Cfg"]) - assert "from typing import TYPE_CHECKING" in result - assert "if TYPE_CHECKING:" in result - assert " from .config import Cfg" in result - - -def test_inject_type_checking_imports_creates_block_type_checking_already_imported(): - # TYPE_CHECKING already in typing import → don't add it again. - src = ( - "from typing import List, TYPE_CHECKING\n" - "\n" - "def foo(x: 'Cfg') -> None:\n" - " pass\n" - ) - result = _inject_type_checking_imports(src, ["from .config import Cfg"]) - assert result.count("TYPE_CHECKING") == 2 # one in import, one in if-block - assert "if TYPE_CHECKING:" in result - assert " from .config import Cfg" in result - - -def test_inject_type_checking_imports_block_after_last_import(): - # The new block should appear after the last import, before other code. - src = "import os\nimport sys\n\nx = 1\n" - result = _inject_type_checking_imports(src, ["from .config import Cfg"]) - lines = result.splitlines() - sys_line = next(i for i, l in enumerate(lines) if "import sys" in l) - if_line = next(i for i, l in enumerate(lines) if "if TYPE_CHECKING" in l) - x_line = next(i for i, l in enumerate(lines) if "x = 1" in l) - assert sys_line < if_line < x_line - - -# --------------------------------------------------------------------------- -# _test_names_in_decorators -# --------------------------------------------------------------------------- - - -def test_test_names_in_decorators_finds_name_in_decorator(): - src = ( - "@pytest.mark.parametrize('x', TestFixture.PARAMS)\ndef test_fn(x):\n pass\n" - ) - assert _test_names_in_decorators(src, {"TestFixture"}) == {"TestFixture"} - - -def test_test_names_in_decorators_name_only_in_body_not_found(): - src = "def test_fn():\n TestFixture.setup()\n" - assert _test_names_in_decorators(src, {"TestFixture"}) == set() - - -def test_test_names_in_decorators_syntax_error_returns_empty(): - assert _test_names_in_decorators("def (invalid", {"TestFixture"}) == set() - - -def test_test_names_in_decorators_class_decorator(): - src = "@TestFixture.mark\nclass TestSomething:\n pass\n" - assert _test_names_in_decorators(src, {"TestFixture"}) == {"TestFixture"} - - -# --------------------------------------------------------------------------- -# _extract_import_info -# --------------------------------------------------------------------------- - - -def test_extract_import_info_syntax_error(): - assert _extract_import_info("def (invalid") == [] - - -def test_extract_import_info_plain_import(): - infos = _extract_import_info("import os\n") - assert len(infos) == 1 - assert "os" in infos[0].names - assert infos[0].is_future is False - - -def test_extract_import_info_import_with_asname(): - infos = _extract_import_info("import os as operating_system\n") - assert infos[0].names == ["operating_system"] - - -def test_extract_import_info_dotted_import(): - infos = _extract_import_info("import os.path\n") - assert infos[0].names == ["os"] - - -def test_extract_import_info_from_import(): - infos = _extract_import_info("from pathlib import Path\n") - assert "Path" in infos[0].names - assert infos[0].is_future is False - - -def test_extract_import_info_from_import_with_asname(): - infos = _extract_import_info("from pathlib import Path as P\n") - assert infos[0].names == ["P"] - - -def test_extract_import_info_future_import(): - infos = _extract_import_info("from __future__ import annotations\n") - assert infos[0].is_future is True - assert "annotations" in infos[0].names - - -def test_extract_import_info_skips_non_imports(): - infos = _extract_import_info("def foo():\n pass\n") - assert infos == [] - - -def test_extract_import_info_multiple(): - source = "import os\nfrom pathlib import Path\n" - infos = _extract_import_info(source) - assert len(infos) == 2 - - -def test_extract_import_info_multiline_parens_normalized(): - # Multi-line parenthesized from-import must be normalized to a single line - # so that _merge_from_imports can process it without producing malformed output. - source = "from pathlib import (\n Path,\n PurePath,\n)\n" - infos = _extract_import_info(source) - assert len(infos) == 1 - assert infos[0].source == "from pathlib import Path, PurePath" - assert "\n" not in infos[0].source - assert "Path" in infos[0].names - assert "PurePath" in infos[0].names - - -def test_extract_import_info_type_checking_from_import(): - # Imports inside `if TYPE_CHECKING:` are extracted with is_type_checking=True. - source = ( - "from typing import TYPE_CHECKING\n" - "if TYPE_CHECKING:\n" - " from .config import MyConfig\n" - ) - infos = _extract_import_info(source) - tc = [i for i in infos if i.is_type_checking] - assert len(tc) == 1 - assert "MyConfig" in tc[0].names - assert tc[0].source == "from .config import MyConfig" - assert tc[0].is_future is False - - -def test_extract_import_info_type_checking_plain_import(): - # Plain `import` inside `if TYPE_CHECKING:` is also captured. - source = "if TYPE_CHECKING:\n import sys\n" - infos = _extract_import_info(source) - tc = [i for i in infos if i.is_type_checking] - assert len(tc) == 1 - assert "sys" in tc[0].names - assert tc[0].is_type_checking is True - - -def test_extract_import_info_type_checking_not_is_future(): - # TYPE_CHECKING block imports must not be marked as is_future. - source = "if TYPE_CHECKING:\n from .foo import Bar\n" - infos = _extract_import_info(source) - tc = [i for i in infos if i.is_type_checking] - assert all(not i.is_future for i in tc) - - -def test_extract_import_info_type_checking_skips_non_import_children(): - # Non-import statements inside a TYPE_CHECKING block (rare but valid) - # must not cause errors and must be silently skipped. - source = "if TYPE_CHECKING:\n from .foo import Bar\n x = 1\n" - infos = _extract_import_info(source) - tc = [i for i in infos if i.is_type_checking] - assert len(tc) == 1 - assert "Bar" in tc[0].names - - -# --------------------------------------------------------------------------- -# _find_needed_imports -# --------------------------------------------------------------------------- - - -def test_find_needed_imports_referenced_name(): - # Entity references "os"; import for "os" should be included. - entity_src_map = {"foo": "def foo():\n os.getcwd()\n"} - infos = [ImportInfo(names=["os"], source="import os", is_future=False)] - result = _find_needed_imports(["foo"], entity_src_map, infos, {"foo"}) - assert "import os" in result - - -def test_find_needed_imports_unreferenced_name(): - # Entity doesn't reference "sys"; import should be excluded. - entity_src_map = {"foo": "def foo():\n pass\n"} - infos = [ImportInfo(names=["sys"], source="import sys", is_future=False)] - result = _find_needed_imports(["foo"], entity_src_map, infos, {"foo"}) - assert result == [] - - -def test_find_needed_imports_future_always_included(): - # __future__ import is always included regardless of entity references. - entity_src_map = {"foo": "def foo():\n pass\n"} - infos = [ - ImportInfo( - names=["annotations"], - source="from __future__ import annotations", - is_future=True, - ) - ] - result = _find_needed_imports(["foo"], entity_src_map, infos, {"foo"}) - assert "from __future__ import annotations" in result - - -def test_find_needed_imports_deduplicates(): - # Two ImportInfo entries with the same source string → only one included. - entity_src_map = {"foo": "def foo():\n os.getcwd()\n"} - infos = [ - ImportInfo(names=["os"], source="import os", is_future=False), - ImportInfo(names=["os"], source="import os", is_future=False), # duplicate - ] - result = _find_needed_imports(["foo"], entity_src_map, infos, {"foo"}) - assert result.count("import os") == 1 - - -def test_find_needed_imports_entity_not_in_map(): - # Entity name not in entity_source_map → treated as empty source. - infos = [ImportInfo(names=["os"], source="import os", is_future=False)] - result = _find_needed_imports(["ghost"], {}, infos, set()) - assert result == [] - - -def test_find_needed_imports_skips_type_checking(): - # is_type_checking imports must not appear as regular imports. - entity_src_map = {"foo": 'def foo(x: "MyConfig") -> None:\n pass\n'} - infos = [ - ImportInfo( - names=["MyConfig"], - source="from .config import MyConfig", - is_future=False, - is_type_checking=True, - ) - ] - result = _find_needed_imports(["foo"], entity_src_map, infos, {"foo"}) - assert result == [] - - -# --------------------------------------------------------------------------- -# _find_type_checking_needed_imports -# --------------------------------------------------------------------------- - - -def test_find_type_checking_needed_imports_quoted_only(): - # "MyType" appears only in a quoted annotation, not a runtime load. - entity_src_map = {"foo": 'def foo(x: Optional["MyType"]) -> None:\n pass\n'} - infos = [ - ImportInfo( - names=["MyType"], source="from models import MyType", is_future=False - ) - ] - result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) - assert "from models import MyType" in result - - -def test_find_type_checking_needed_imports_runtime_excluded(): - # When the name is used at runtime (not just in a quoted annotation), - # it should NOT appear in the TYPE_CHECKING-only list. - # annotation_only = quoted - runtime excludes runtime names directly. - entity_src_map = {"foo": "def foo():\n return MyType()\n"} - infos = [ - ImportInfo( - names=["MyType"], source="from models import MyType", is_future=False - ) - ] - result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) - assert result == [] - - -def test_find_type_checking_needed_imports_no_annotations(): - # No quoted annotations → result is empty. - entity_src_map = {"foo": "def foo():\n pass\n"} - infos = [ - ImportInfo( - names=["MyType"], source="from models import MyType", is_future=False - ) - ] - result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) - assert result == [] - - -def test_find_type_checking_needed_imports_future_excluded(): - # __future__ imports are never returned (they're always in regular imports). - entity_src_map = {"foo": 'def foo(x: "MyType") -> None:\n pass\n'} - infos = [ - ImportInfo( - names=["annotations"], - source="from __future__ import annotations", - is_future=True, - ), - ImportInfo( - names=["MyType"], source="from models import MyType", is_future=False - ), - ] - result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) - assert "from __future__ import annotations" not in result - assert "from models import MyType" in result - - -def test_find_type_checking_needed_imports_deduplicates(): - # Two ImportInfo entries with the same source → only one returned. - entity_src_map = {"foo": 'def foo(x: "MyType") -> None:\n pass\n'} - infos = [ - ImportInfo( - names=["MyType"], source="from models import MyType", is_future=False - ), - ImportInfo( - names=["MyType"], source="from models import MyType", is_future=False - ), - ] - result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) - assert result.count("from models import MyType") == 1 - - -def test_find_type_checking_needed_imports_import_names_no_match(): - # annotation_only has "MyType" but the ImportInfo names do not include it → - # the tc_names check returns False → import is skipped. - entity_src_map = {"foo": 'def foo(x: "MyType") -> None:\n pass\n'} - infos = [ - ImportInfo( - names=["OtherType"], source="from models import OtherType", is_future=False - ) - ] - result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) - assert result == [] - - -def test_find_type_checking_needed_imports_partial_multi_name_import(): - # From a multi-name import, only the annotation-only name should appear in - # the TYPE_CHECKING block; the other name (not referenced at all) must not. - entity_src_map = { - "foo": 'def foo(x: "MyResult") -> None:\n pass\n', - } - infos = [ - ImportInfo( - names=["MyResult", "run_thing"], - source="from mymod import MyResult, run_thing", - is_future=False, - ) - ] - result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) - assert len(result) == 1 - assert "MyResult" in result[0] - assert "run_thing" not in result[0] - - -def test_find_type_checking_needed_imports_narrowed_src_dedup(): - # When two ImportInfo entries produce the same narrowed source after - # filtering, only one copy should appear in the result (line 535 branch). - entity_src_map = {"foo": 'def foo(x: "MyResult") -> None:\n pass\n'} - infos = [ - ImportInfo( - names=["MyResult", "run_thing"], - source="from mymod import MyResult, run_thing", - is_future=False, - ), - # A second entry with the same source (e.g. two entities requested it). - ImportInfo( - names=["MyResult", "run_thing"], - source="from mymod import MyResult, run_thing", - is_future=False, - ), - ] - result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) - assert result.count("from mymod import MyResult") == 1 - - -def test_find_type_checking_needed_imports_shared_import_with_runtime_peer(): - # Regression: when an import line covers both a runtime name and an - # annotation-only name, the annotation-only name must still get a - # TYPE_CHECKING import even though the import source appears in the - # regular imports (where _prune_unused_imports will later drop it). - entity_src_map = { - "foo": ( - 'def foo(_acc: Optional["_LLMAccumulator"] = None) -> None:\n' - " call_with_tool(_PLACEMENT_TOOL)\n" - ) - } - infos = [ - ImportInfo( - names=["_LLMAccumulator", "_PLACEMENT_TOOL"], - source="from .llm_schemas import _LLMAccumulator, _PLACEMENT_TOOL", - is_future=False, - ) - ] - result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) - # _LLMAccumulator is only in a quoted annotation → must be in TC block - assert any("_LLMAccumulator" in r for r in result) - # _PLACEMENT_TOOL is a runtime reference → must NOT be in TC block - assert not any("_PLACEMENT_TOOL" in r for r in result) - - -def test_find_type_checking_needed_imports_uses_is_type_checking_infos(): - # is_type_checking=True ImportInfo entries are used for TC distribution; - # the function should return them for entities that use the name in a - # quoted annotation. - entity_src_map = {"foo": 'def foo(config: "MyConfig") -> None:\n pass\n'} - infos = [ - ImportInfo( - names=["MyConfig"], - source="from .config import MyConfig", - is_future=False, - is_type_checking=True, - ) - ] - result = _find_type_checking_needed_imports(["foo"], entity_src_map, infos) - assert "from .config import MyConfig" in result - - -# --------------------------------------------------------------------------- -# _narrow_import_source -# --------------------------------------------------------------------------- - - -def test_narrow_import_source_syntax_error(): - # Invalid Python → original string returned unchanged. - bad = "from ??? import Foo" - assert _narrow_import_source(bad, {"Foo"}) == bad - - -def test_narrow_import_source_plain_import(): - # Non-ImportFrom statement (bare `import X`) → returned unchanged. - src = "import os" - assert _narrow_import_source(src, {"os"}) == src - - -def test_narrow_import_source_empty_keep(): - # keep_names matches nothing → alias_strs is empty → return original. - src = "from mymod import A, B" - assert _narrow_import_source(src, {"C"}) == src - - -# --------------------------------------------------------------------------- -# _target_module_name -# --------------------------------------------------------------------------- - - -def test_target_module_name_simple(): - assert _target_module_name("utils.py") == "utils" - - -def test_target_module_name_nested(): - assert _target_module_name("helpers/io.py") == "helpers.io" - - -def test_target_module_name_init(): - # __init__.py represents the package, not a "__init__" submodule. - assert _target_module_name("pkg/__init__.py") == "pkg" - - -# --------------------------------------------------------------------------- -# _remove_entity_lines -# --------------------------------------------------------------------------- - - -def test_remove_entity_lines_removes_range(): - source = "line1\nline2\nline3\nline4\n" - entity = _make_entity("foo", 2, 3) - entity_map = {"foo": entity} - result = _remove_entity_lines(source, {"foo"}, entity_map, {}) - assert "line1" in result - assert "line2" not in result - assert "line3" not in result - assert "line4" in result - - -def test_remove_entity_lines_name_not_in_map(): - # Name not in entity_map → nothing removed. - source = "line1\nline2\n" - result = _remove_entity_lines(source, {"ghost"}, {}, {}) - assert result == source - - -def test_remove_entity_lines_top_level_preserves_import_lines(): - # When a TOP_LEVEL entity containing both imports and assignments is - # migrated, the import lines must be kept in the original file so that - # the remaining functions still have access to those names. - source = "import os\n_CONST = 1\n\ndef foo():\n return os.getcwd()\n" - entity_src = "import os\n_CONST = 1\n" - entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 2, ["os", "_CONST"]) - entity_map = {"_block_1": entity} - entity_source_map = {"_block_1": entity_src} - result = _remove_entity_lines(source, {"_block_1"}, entity_map, entity_source_map) - assert "import os" in result # import line preserved - assert "_CONST" not in result # assignment line removed - assert "def foo():" in result # function untouched - - -def test_remove_entity_lines_top_level_no_source_map_removes_all(): - # Empty entity_source_map → no imports can be identified, all lines removed. - source = "import os\n_CONST = 1\n\ndef foo():\n pass\n" - entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 2, ["os", "_CONST"]) - entity_map = {"_block_1": entity} - result = _remove_entity_lines(source, {"_block_1"}, entity_map, {}) - assert "import os" not in result - assert "_CONST" not in result - - -# --------------------------------------------------------------------------- -# _import_derived_names -# --------------------------------------------------------------------------- - - -def test_import_derived_names_plain_import(): - src = "import os\nimport sys\n" - assert _import_derived_names(src) == {"os", "sys"} - - -def test_import_derived_names_from_import(): - src = "from typing import Dict, List\n" - assert _import_derived_names(src) == {"Dict", "List"} - - -def test_import_derived_names_aliased(): - src = "import libcst as cst\nfrom dataclasses import dataclass\n" - assert _import_derived_names(src) == {"cst", "dataclass"} - - -def test_import_derived_names_ignores_assignments(): - src = "_MODEL = 'x'\n_MIN = 3\n" - assert _import_derived_names(src) == set() - - -def test_import_derived_names_syntax_error(): - assert _import_derived_names("def (\n") == set() - - -# --------------------------------------------------------------------------- -# _import_line_numbers -# --------------------------------------------------------------------------- - - -def test_import_line_numbers_basic(): - src = "import os\n_CONST = 1\n" - entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 5, 6, []) - # Entity starts at line 5; "import os" is relative line 1 → absolute line 5. - result = _import_line_numbers(entity, src) - assert result == {5} - - -def test_import_line_numbers_no_imports(): - src = "_CONST = 1\n_OTHER = 2\n" - entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 2, []) - assert _import_line_numbers(entity, src) == set() - - -def test_import_line_numbers_syntax_error(): - entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, []) - assert _import_line_numbers(entity, "def (\n") == set() - - -# --------------------------------------------------------------------------- -# _add_re_exports — import-derived name filtering -# --------------------------------------------------------------------------- - - -def test_add_re_exports_top_level_import_derived_names_not_re_exported(): - # A TOP_LEVEL entity that includes import statements: the names introduced - # by those imports must NOT appear in re-exports because they are preserved - # in the original file by _remove_entity_lines, not moved to the new file. - source = "import os\n\nMY_CONST\n" # MY_CONST still loaded - entity_src = "from typing import Dict\n\nMY_CONST = 42\n" - entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["Dict", "MY_CONST"]) - placement = GroupPlacement(group=["_block_1"], target_file="constants.py") - result = _add_re_exports( - source, [placement], {"_block_1": entity}, {"_block_1": entity_src} - ) - assert "MY_CONST" in result # assignment-defined name re-exported - assert "Dict" not in result # import-derived name suppressed - - -# --------------------------------------------------------------------------- -# _add_re_exports -# --------------------------------------------------------------------------- - - -def test_add_re_exports_all_private_no_change(): - # Private name not called anywhere in remaining source → no import added. - source = "import os\n\ndef _helper():\n pass\n" - entity = _make_entity("_helper", 3, 4) - placement = GroupPlacement(group=["_helper"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"_helper": entity}, {}) - assert result == source - - -def test_add_re_exports_private_referenced_in_source(): - # Private name still called in remaining source → import is added. - source = "import os\n\n_helper()\n" - entity = _make_entity("_helper", 3, 3) - placement = GroupPlacement(group=["_helper"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"_helper": entity}, {}) - assert "from .utils import _helper" in result - - -def test_add_re_exports_public_inserted_after_imports(): - source = "import os\n\ndef foo():\n pass\n" - entity = _make_entity("foo", 3, 4) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"foo": entity}, {}) - assert "from .utils import foo" in result - # Re-export line should come after "import os" - lines = result.splitlines() - import_idx = next(i for i, l in enumerate(lines) if "import os" in l) - reexport_idx = next(i for i, l in enumerate(lines) if "from .utils import foo" in l) - assert reexport_idx > import_idx - - -def test_add_re_exports_no_import_in_source(): - # No imports and no docstring → re-export inserted at beginning. - source = "\ndef foo():\n pass\n" - entity = _make_entity("foo", 2, 3) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"foo": entity}, {}) - assert "from .utils import foo" in result - - -def test_add_re_exports_no_import_with_module_docstring(): - # No imports but module docstring present → re-export inserted after docstring, - # not before it, so the docstring remains the first statement. - source = '"""Module docstring."""\n\n\ndef foo():\n pass\n' - entity = _make_entity("foo", 4, 5) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"foo": entity}, {}) - lines = result.splitlines() - docstring_idx = next( - i for i, l in enumerate(lines) if '"""Module docstring."""' in l - ) - reexport_idx = next(i for i, l in enumerate(lines) if "from .utils import foo" in l) - assert docstring_idx == 0 - assert reexport_idx > docstring_idx - - -def test_add_re_exports_from_import_line(): - # "from pathlib import Path" should be detected as an import line. - source = "from pathlib import Path\n\ndef foo():\n pass\n" - entity = _make_entity("foo", 3, 4) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"foo": entity}, {}) - lines = result.splitlines() - from_import_idx = next( - i for i, l in enumerate(lines) if "from pathlib import Path" in l - ) - reexport_idx = next(i for i, l in enumerate(lines) if "from .utils import foo" in l) - assert reexport_idx > from_import_idx - - -def test_add_re_exports_multiple_targets_sorted(): - source = "import os\n" - e1 = _make_entity("foo", 1, 2) - e2 = _make_entity("bar", 3, 4) - placements = [ - GroupPlacement(group=["foo"], target_file="b_module.py"), - GroupPlacement(group=["bar"], target_file="a_module.py"), - ] - result = _add_re_exports(source, placements, {"foo": e1, "bar": e2}, {}) - # a_module comes before b_module (sorted) - a_idx = result.index("a_module") - b_idx = result.index("b_module") - assert a_idx < b_idx - - -def test_add_re_exports_mixed_public_private(): - source = "import os\n" - entity_map = { - "pub": _make_entity("pub", 1, 2), - "_priv": _make_entity("_priv", 3, 4), - } - placement = GroupPlacement(group=["pub", "_priv"], target_file="utils.py") - result = _add_re_exports(source, [placement], entity_map, {}) - # Only "pub" in re-export, not "_priv" - assert "pub" in result - assert "_priv" not in result - - -def test_add_re_exports_test_function_not_re_exported(): - # test_ functions must never get a proxy import — pytest would discover and - # run them twice (once from the original file, once from the new file). - source = "import os\n" - entity = _make_entity("test_something", 1, 3) - placement = GroupPlacement(group=["test_something"], target_file="tests/helpers.py") - result = _add_re_exports(source, [placement], {"test_something": entity}, {}) - assert result == source - - -def test_add_re_exports_test_function_never_re_exported_even_when_referenced(): - # test_* names are never re-exported at module level even when the - # remaining source references them — _inject_inline_test_imports_original - # handles those cases inline to prevent pytest double-discovery. - source = "import os\n\ntest_something()\n" - entity = _make_entity("test_something", 1, 3) - placement = GroupPlacement(group=["test_something"], target_file="tests/helpers.py") - result = _add_re_exports(source, [placement], {"test_something": entity}, {}) - assert "from .tests.helpers import test_something" not in result - - -def test_class_has_test_methods_true(): - src = "class TestFoo:\n def test_bar(self): pass\n" - assert _class_has_test_methods(src) is True - - -def test_class_has_test_methods_false(): - src = "class Helper:\n def run(self): pass\n" - assert _class_has_test_methods(src) is False - - -def test_class_has_test_methods_syntax_error(): - assert _class_has_test_methods("def (") is False - - -def test_add_re_exports_test_class_not_re_exported(): - # A class that contains test_ methods must not be re-exported — pytest - # would discover it via the original file and the new file, running every - # test twice. - source = "import os\n" - entity = Entity(EntityKind.CLASS, "TestFoo", 1, 5, ["TestFoo"]) - entity_src = "class TestFoo:\n def test_bar(self): pass\n" - placement = GroupPlacement(group=["TestFoo"], target_file="tests/helpers.py") - result = _add_re_exports( - source, [placement], {"TestFoo": entity}, {"TestFoo": entity_src} - ) - assert result == source - - -def test_add_re_exports_test_class_never_re_exported_even_when_referenced(): - # Test-named symbols are never re-exported at module level even when - # referenced in remaining source — _inject_inline_test_imports_original - # handles them inline to prevent pytest double-discovery. - source = "import os\n\nTestFoo()\n" - entity = Entity(EntityKind.CLASS, "TestFoo", 1, 5, ["TestFoo"]) - entity_src = "class TestFoo:\n def test_bar(self): pass\n" - placement = GroupPlacement(group=["TestFoo"], target_file="tests/helpers.py") - result = _add_re_exports( - source, [placement], {"TestFoo": entity}, {"TestFoo": entity_src} - ) - assert "from .tests.helpers import TestFoo" not in result - - -def test_add_re_exports_top_level_block_private_names_referenced(): - # TOP_LEVEL block entity name (_block_1) differs from its defined names. - # Both defined names are still loaded in remaining source → re-imported. - source = "import os\n\n_DUP_SOURCE\n_DUP_RANGES\n" - entity = Entity( - EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_DUP_SOURCE", "_DUP_RANGES"] - ) - placement = GroupPlacement(group=["_block_1"], target_file="test_helpers.py") - result = _add_re_exports(source, [placement], {"_block_1": entity}, {}) - assert "from .test_helpers import _DUP_RANGES, _DUP_SOURCE" in result - - -def test_add_re_exports_entity_not_in_map_falls_back_to_entity_name(): - # Entity name in group is missing from entity_map → falls back to entity name. - source = "import os\n\nghost()\n" # 'ghost' is still referenced - placement = GroupPlacement(group=["ghost"], target_file="utils.py") - result = _add_re_exports(source, [placement], {}, {}) - assert "from .utils import ghost" in result - - -def test_add_re_exports_top_level_block_private_names_not_referenced(): - # TOP_LEVEL block entity whose defined name is private and not used → no import. - source = "import os\n" - entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) - placement = GroupPlacement(group=["_block_1"], target_file="constants.py") - result = _add_re_exports(source, [placement], {"_block_1": entity}, {}) - assert result == source - - -def test_add_re_exports_indented_local_import_not_treated_as_last_import(): - # Functions with local (indented) imports must not cause re-exports to be - # inserted inside the function body. The re-export should appear after the - # top-level "import os" line, not after the indented "from x import y". - source = ( - "import os\n" - "\n" - "def foo():\n" - " from unittest.mock import MagicMock\n" - " MagicMock()\n" - "\n" - "def bar():\n" - " pass\n" - ) - entity = _make_entity("baz", 7, 8) - placement = GroupPlacement(group=["baz"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"baz": entity}, {}) - # Re-export must appear immediately after "import os", not inside foo(). - lines = result.splitlines() - os_idx = next(i for i, l in enumerate(lines) if l == "import os") - reexport_idx = next(i for i, l in enumerate(lines) if "from .utils import baz" in l) - assert reexport_idx == os_idx + 1 - # The function body must remain intact (local import line must still be there). - assert " from unittest.mock import MagicMock" in result - - -def test_add_re_exports_syntax_error_returns_source_unchanged(): - # If the source has a SyntaxError, _add_re_exports cannot determine where - # to insert re-exports and must return the source unchanged. - source = "import os\ndef (invalid\n" - entity = _make_entity("baz", 1, 1) - placement = GroupPlacement(group=["baz"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"baz": entity}, {}) - assert result == source - - -def test_add_re_exports_abs_pkg_package_prefix(): - # abs_pkg="tests" → absolute import: "from tests.utils import foo" - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"foo": entity}, {}, abs_pkg="tests") - assert "from tests.utils import foo" in result - assert "from .utils import foo" not in result - - -def test_add_re_exports_abs_pkg_root_level(): - # abs_pkg="" → root-level absolute import: "from utils import foo" - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"foo": entity}, {}, abs_pkg="") - assert "from utils import foo" in result - assert "from .utils import foo" not in result - - -# --------------------------------------------------------------------------- -# generate_file_splits -# --------------------------------------------------------------------------- - - -def test_generate_abort_plan(): - plan = _abort_plan() - c = _classified() - result = generate_file_splits(c, plan, "def foo():\n pass\n", "big.py") - assert result.abort is True - assert result.new_files == {} - assert result.original_source == "def foo():\n pass\n" - - -def test_generate_empty_placements(): - plan = _plan() # placements=[] - c = _classified() - source = "def foo():\n pass\n" - result = generate_file_splits(c, plan, source, "big.py") - assert result.abort is False - assert result.new_files == {} - assert result.original_source == source - - -def test_generate_single_entity_migration(): - source = "import os\n\ndef foo():\n os.getcwd()\n" - entity = _make_entity("foo", 3, 4) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["foo"], target_file="utils.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - assert result.abort is False - assert "utils.py" in result.new_files - new_src = result.new_files["utils.py"] - assert "import os" in new_src - assert "def foo():" in new_src - # Original should not have foo's def anymore - assert "def foo():" not in result.original_source - # But should have a re-export - assert "from .utils import foo" in result.original_source - - -def test_generate_private_entity_no_reexport(): - source = "def _helper():\n pass\n" - entity = _make_entity("_helper", 1, 2) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["_helper"], target_file="private.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - assert "from .private import" not in result.original_source - - -def test_generate_entity_not_in_source_map(): - # Group has entity name not in classified.entities → entity skipped in new file. - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - c = _classified(entities=[entity]) - # "ghost" is in the group but has no matching entity - plan = _plan([GroupPlacement(group=["foo", "ghost"], target_file="utils.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - assert "utils.py" in result.new_files - # "ghost" produces no source so only "foo" appears - new_src = result.new_files["utils.py"] - assert "def foo():" in new_src - - -def test_generate_no_imports_needed(): - # Entity uses no imports → no import section in new file. - source = "def add(a, b):\n return a + b\n" - entity = _make_entity("add", 1, 2) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["add"], target_file="math_utils.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - new_src = result.new_files["math_utils.py"] - # No "import" prefix expected - assert not new_src.startswith("import") - assert "def add" in new_src - - -def test_generate_multiple_groups_same_file(): - source = textwrap.dedent( - """\ - import os - - def foo(): - pass - - def bar(): - pass - """ - ) - e_foo = _make_entity("foo", 3, 4) - e_bar = _make_entity("bar", 6, 7) - c = _classified(entities=[e_foo, e_bar]) - plan = _plan( - [ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="utils.py"), - ] - ) - result = generate_file_splits(c, plan, source, "big.py") - - new_src = result.new_files["utils.py"] - assert "def foo():" in new_src - assert "def bar():" in new_src - - -def test_generate_multiple_different_target_files(): - source = "def foo():\n pass\n\ndef bar():\n pass\n" - e_foo = _make_entity("foo", 1, 2) - e_bar = _make_entity("bar", 4, 5) - c = _classified(entities=[e_foo, e_bar]) - plan = _plan( - [ - GroupPlacement(group=["foo"], target_file="foo_module.py"), - GroupPlacement(group=["bar"], target_file="bar_module.py"), - ] - ) - result = generate_file_splits(c, plan, source, "big.py") - - assert "foo_module.py" in result.new_files - assert "bar_module.py" in result.new_files - assert "def foo():" in result.new_files["foo_module.py"] - assert "def bar():" in result.new_files["bar_module.py"] - assert "from .bar_module import bar" in result.original_source - assert "from .foo_module import foo" in result.original_source - - -def test_generate_future_import_not_duplicated_when_in_entity_source(): - # Entity source itself contains `from __future__ import annotations` - # (e.g. the _block_1 TOP_LEVEL entity which IS the file's import block). - # It must appear only once at the top of the new file, not again inside - # the entity source, which would cause a SyntaxError. - source = textwrap.dedent( - """\ - from __future__ import annotations - - \"\"\"Module docstring.\"\"\" - - from __future__ import annotations - - import os - - _CONST = 42 - """ - ) - # _block_1 spans the whole file and contains the future import + constants. - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 9, ["_CONST"]) - c = _classified(entities=[e_block]) - plan = _plan([GroupPlacement(group=["_block_1"], target_file="constants.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - new_src = result.new_files["constants.py"] - assert new_src.count("from __future__ import annotations") == 1 - # Must be at the very start of the file (before any other code). - first_non_blank = next(line for line in new_src.splitlines() if line.strip()) - assert first_non_blank == "from __future__ import annotations" - - -def test_generate_future_import_always_included(): - source = "from __future__ import annotations\n\ndef foo():\n pass\n" - entity = _make_entity("foo", 3, 4) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["foo"], target_file="utils.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - new_src = result.new_files["utils.py"] - assert "from __future__ import annotations" in new_src - - -# --------------------------------------------------------------------------- -# _find_cross_file_imports -# --------------------------------------------------------------------------- - - -def test_find_cross_file_imports_basic(): - # fn_a references _MODEL which is defined in block_1.py - entity_source_map = {"fn_a": "def fn_a():\n return _MODEL\n"} - name_to_target_file = {"_MODEL": "block_1.py"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["fn_a"], entity_source_map, name_to_target_file, "llm_extract.py" - ) - assert from_imports == ["from .block_1 import _MODEL"] - assert module_imports == [] - assert rewrites == {} - - -def test_find_cross_file_imports_same_file_excluded(): - # _MODEL goes to the same file as fn_a → no cross-file import needed - entity_source_map = {"fn_a": "def fn_a():\n return _MODEL\n"} - name_to_target_file = {"_MODEL": "llm_extract.py"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["fn_a"], entity_source_map, name_to_target_file, "llm_extract.py" - ) - assert from_imports == [] - assert module_imports == [] - assert rewrites == {} - - -def test_find_cross_file_imports_no_match(): - # Referenced name not in name_to_target_file → no cross-file import - entity_source_map = {"fn_a": "def fn_a():\n return os.getcwd()\n"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["fn_a"], entity_source_map, {}, "utils.py" - ) - assert from_imports == [] - assert module_imports == [] - assert rewrites == {} - - -def test_find_cross_file_imports_entity_not_in_map(): - # Entity name not in entity_source_map → treated as empty source, no imports - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["ghost"], {}, {"x": "other.py"}, "utils.py" - ) - assert from_imports == [] - assert module_imports == [] - assert rewrites == {} - - -# --------------------------------------------------------------------------- -# _relative_import_prefix -# --------------------------------------------------------------------------- - - -def test_relative_import_prefix_same_directory(): - # Both files at the root level → single dot. - assert _relative_import_prefix("a.py", "b.py") == ".b" - - -def test_relative_import_prefix_sibling_subdir(): - # from_file is in sub/, to_file is in helpers/ → go up one, then down. - assert _relative_import_prefix("sub/a.py", "helpers/b.py") == "..helpers.b" - - -def test_relative_import_prefix_same_subdir(): - # Both in the same subdirectory → single dot. - assert _relative_import_prefix("sub/a.py", "sub/b.py") == ".b" - - -def test_relative_import_prefix_to_nested(): - # to_file is in a subdirectory of root while from_file is at root. - assert _relative_import_prefix("a.py", "helpers/b.py") == ".helpers.b" - - -def test_relative_import_prefix_to_init_same_dir(): - # to_file is __init__.py in the same directory → "." (the package itself). - assert _relative_import_prefix("a.py", "__init__.py") == "." - - -def test_relative_import_prefix_to_init_same_subdir(): - # Both in sub/, to_file is sub/__init__.py → "." (the package itself). - assert _relative_import_prefix("sub/a.py", "sub/__init__.py") == "." - - -def test_find_cross_file_imports_cross_directory(): - # fn_a is in tests/test.py; helper is in helpers/entities.py. - # Cross-directory import needs ".." to go up from tests/ to root. - entity_source_map = {"fn_a": "def fn_a():\n return _helper()\n"} - name_to_target_file = {"_helper": "helpers/entities.py"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["fn_a"], entity_source_map, name_to_target_file, "tests/test.py" - ) - assert from_imports == ["from ..helpers.entities import _helper"] - assert module_imports == [] - assert rewrites == {} - - -def test_find_cross_file_imports_top_level_var_uses_module_import(): - # SAFE_MODE is a TOP_LEVEL variable in conversion.py; runtime.py references it. - # Should produce a module-level import (from . import conversion) in - # module_imports, not a direct name import, so that later mutations to the - # variable propagate correctly. - entity_source_map = { - "create_lua_runtime": ( - "def create_lua_runtime(safe_mode=None):\n" - " if safe_mode is None:\n" - " safe_mode = SAFE_MODE\n" - ) - } - name_to_target_file = {"SAFE_MODE": "conversion.py"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["create_lua_runtime"], - entity_source_map, - name_to_target_file, - "runtime.py", - top_level_var_names={"SAFE_MODE"}, - ) - assert from_imports == [] - assert module_imports == ["from . import conversion"] - assert rewrites == {"SAFE_MODE": "conversion.SAFE_MODE"} - - -def test_find_cross_file_imports_top_level_var_abs_pkg(): - # Same as above but with abs_pkg set (test-file context). - # Uses "import pkg.module as local" syntax to avoid test-name misclassification. - entity_source_map = {"fn_a": "def fn_a():\n return SAFE_MODE\n"} - name_to_target_file = {"SAFE_MODE": "conversion.py"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["fn_a"], - entity_source_map, - name_to_target_file, - "test_fn.py", - abs_pkg="mylib", - top_level_var_names={"SAFE_MODE"}, - ) - assert from_imports == [] - assert module_imports == ["import mylib.conversion as conversion"] - assert rewrites == {"SAFE_MODE": "conversion.SAFE_MODE"} - - -def test_find_cross_file_imports_top_level_var_abs_pkg_empty(): - # abs_pkg="" (root-level test) — no package prefix, plain "import conversion". - entity_source_map = {"fn_a": "def fn_a():\n return SAFE_MODE\n"} - name_to_target_file = {"SAFE_MODE": "conversion.py"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["fn_a"], - entity_source_map, - name_to_target_file, - "test_fn.py", - abs_pkg="", - top_level_var_names={"SAFE_MODE"}, - ) - assert from_imports == [] - assert module_imports == ["import conversion"] - assert rewrites == {"SAFE_MODE": "conversion.SAFE_MODE"} - - -def test_find_cross_file_imports_top_level_var_cross_directory(): - # TOP_LEVEL var in sub/constants.py, referenced from runtime.py at root. - entity_source_map = {"fn_a": "def fn_a():\n return TIMEOUT\n"} - name_to_target_file = {"TIMEOUT": "sub/constants.py"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["fn_a"], - entity_source_map, - name_to_target_file, - "runtime.py", - top_level_var_names={"TIMEOUT"}, - ) - assert from_imports == [] - assert module_imports == ["from .sub import constants"] - assert rewrites == {"TIMEOUT": "constants.TIMEOUT"} - - -def test_find_cross_file_imports_mixed_top_level_and_function(): - # SAFE_MODE is a TOP_LEVEL var; _helper is a function — mixed case. - entity_source_map = { - "fn_a": ("def fn_a():\n" " if SAFE_MODE:\n" " return _helper()\n") - } - name_to_target_file = {"SAFE_MODE": "conversion.py", "_helper": "helpers.py"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["fn_a"], - entity_source_map, - name_to_target_file, - "runtime.py", - top_level_var_names={"SAFE_MODE"}, - ) - assert from_imports == ["from .helpers import _helper"] - assert module_imports == ["from . import conversion"] - assert rewrites == {"SAFE_MODE": "conversion.SAFE_MODE"} - - -# --------------------------------------------------------------------------- -# _module_import_stmt -# --------------------------------------------------------------------------- - - -def test_module_import_stmt_sibling_relative(): - stmt, local = _module_import_stmt("runtime.py", "conversion.py", abs_pkg=None) - assert stmt == "from . import conversion" - assert local == "conversion" - - -def test_module_import_stmt_cross_directory_relative(): - stmt, local = _module_import_stmt("runtime.py", "sub/constants.py", abs_pkg=None) - assert stmt == "from .sub import constants" - assert local == "constants" - - -def test_module_import_stmt_parent_directory_relative(): - # svc/test_fns.py importing from test_svc.py (parent dir) - stmt, local = _module_import_stmt("svc/test_fns.py", "test_svc.py", abs_pkg=None) - assert stmt == "from .. import test_svc" - assert local == "test_svc" - - -def test_module_import_stmt_abs_pkg_with_prefix(): - # Uses "import pkg.module as local" to avoid test-name collision. - stmt, local = _module_import_stmt("test_fn.py", "conversion.py", abs_pkg="mylib") - assert stmt == "import mylib.conversion as conversion" - assert local == "conversion" - - -def test_module_import_stmt_abs_pkg_empty(): - # No package prefix → plain "import conversion". - stmt, local = _module_import_stmt("test_fn.py", "conversion.py", abs_pkg="") - assert stmt == "import conversion" - assert local == "conversion" - - -def test_module_import_stmt_abs_pkg_nested_module(): - # source_file has a nested path within the package - stmt, local = _module_import_stmt("test_fn.py", "sub/constants.py", abs_pkg="mylib") - assert stmt == "import mylib.sub.constants as constants" - assert local == "constants" - - -# --------------------------------------------------------------------------- -# _rewrite_module_level_stores -# --------------------------------------------------------------------------- - - -def test_rewrite_module_level_stores_simple(): - src = "_CONST = int('99')\n" - result = _rewrite_module_level_stores(src, {"_CONST": "constants._CONST"}) - assert result == "constants._CONST = int('99')\n" - - -def test_rewrite_module_level_stores_augassign(): - src = "X += 1\n" - result = _rewrite_module_level_stores(src, {"X": "mod.X"}) - assert result == "mod.X += 1\n" - - -def test_rewrite_module_level_stores_annassign_with_value(): - src = "X: int = 42\n" - result = _rewrite_module_level_stores(src, {"X": "mod.X"}) - assert result == "mod.X: int = 42\n" - - -def test_rewrite_module_level_stores_annassign_without_value_skipped(): - # Declaration only — no value, so nothing to rewrite. - src = "X: int\n" - result = _rewrite_module_level_stores(src, {"X": "mod.X"}) - assert result == src - - -def test_rewrite_module_level_stores_function_body_not_rewritten(): - # Assignments inside function bodies must not be touched. - src = "def f():\n X = 1\n" - result = _rewrite_module_level_stores(src, {"X": "mod.X"}) - assert result == src - - -def test_rewrite_module_level_stores_empty_rewrites(): - src = "X = 1\n" - assert _rewrite_module_level_stores(src, {}) == src - - -def test_rewrite_module_level_stores_syntax_error(): - src = "def (broken:\n" - assert _rewrite_module_level_stores(src, {"X": "mod.X"}) == src - - -def test_rewrite_module_level_stores_name_not_in_rewrites(): - src = "Y = 1\n" - result = _rewrite_module_level_stores(src, {"X": "mod.X"}) - assert result == src - - -def test_rewrite_module_level_stores_augassign_non_name_target(): - # Attribute augmented assignment — target is Attribute, not Name; must be skipped. - src = "obj.x += 1\n" - result = _rewrite_module_level_stores(src, {"x": "mod.x"}) - assert result == src - - -# --------------------------------------------------------------------------- -# _rewrite_module_var_names -# --------------------------------------------------------------------------- - - -def test_rewrite_module_var_names_basic(): - src = "def fn():\n if SAFE_MODE:\n pass\n" - result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) - assert "conversion.SAFE_MODE" in result - # bare SAFE_MODE no longer appears as a standalone Name - import ast - - tree = ast.parse(result) - bare = [ - n for n in ast.walk(tree) if isinstance(n, ast.Name) and n.id == "SAFE_MODE" - ] - assert bare == [] - - -def test_rewrite_module_var_names_skips_attribute_access(): - # obj.SAFE_MODE must NOT become obj.conversion.SAFE_MODE — the regex approach - # would corrupt this; the AST approach correctly skips it because 'SAFE_MODE' - # is the attr string of an Attribute node, not an ast.Name load. - src = "def fn():\n return obj.SAFE_MODE\n" - result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) - assert result == src - - -def test_rewrite_module_var_names_skips_strings(): - src = 'x = "SAFE_MODE"\n' - result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) - assert result == src - - -def test_rewrite_module_var_names_skips_comments(): - src = "# use SAFE_MODE here\nx = 1\n" - result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) - assert result == src - - -def test_rewrite_module_var_names_no_partial_name_match(): - # SAFE_MODE_EXTRA is a different identifier and must not be rewritten - src = "x = SAFE_MODE_EXTRA\ny = SAFE_MODE\n" - result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) - assert "SAFE_MODE_EXTRA" in result - assert "y = conversion.SAFE_MODE" in result - - -def test_rewrite_module_var_names_empty_rewrites(): - src = "def fn():\n return SAFE_MODE\n" - result = _rewrite_module_var_names(src, {}) - assert result == src - - -def test_rewrite_module_var_names_initial_syntax_error_returns_original(): - # Unparseable source at the start → return unchanged (first ast.parse fails) - src = "def fn(\n" - result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) - assert result == src - - -def test_rewrite_module_var_names_no_name_nodes_returns_original(): - # Source has no Name nodes for the given key → return unchanged - src = "x = 1\n" - result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) - assert result == src - - -def test_rewrite_module_var_names_verify_bare_name_survives_returns_original(): - # If a rewrite introduces a new bare Name that itself appears in rewrites, - # verification catches it and returns the original source. - # rewrites={"A": "mod.A", "mod": "pkg.mod"}: rewriting "A" → "mod.A" leaves - # "mod" as a bare Name load, which is in rewrites → verification fails. - src = "x = A\n" - result = _rewrite_module_var_names(src, {"A": "mod.A", "mod": "pkg.mod"}) - assert result == src - - -def test_rewrite_module_var_names_verify_syntax_error_returns_original(monkeypatch): - # If re-parsing the rewritten result raises SyntaxError (defensive guard), - # the original source is returned unchanged. - import crispen.file_limiter.code_gen as _code_gen - import ast as _ast - - call_count = [0] - real_parse = _ast.parse - - def patched_parse(src, *args, **kwargs): - call_count[0] += 1 - if call_count[0] >= 2: # fail on the verification parse - raise SyntaxError("synthetic verify failure") - return real_parse(src, *args, **kwargs) - - monkeypatch.setattr(_code_gen.ast, "parse", patched_parse) - src = "x = SAFE_MODE\n" - result = _rewrite_module_var_names(src, {"SAFE_MODE": "conversion.SAFE_MODE"}) - assert result == src - - -# --------------------------------------------------------------------------- -# _merge_from_imports -# --------------------------------------------------------------------------- - - -def test_merge_from_imports_no_overlap(): - imports = ["from .a import x", "from .b import y"] - assert _merge_from_imports(imports) == ["from .a import x", "from .b import y"] - - -def test_merge_from_imports_overlapping(): - imports = ["from .conv import A, C", "from .conv import B, C"] - result = _merge_from_imports(imports) - assert result == ["from .conv import A, B, C"] - - -def test_merge_from_imports_deduplicates_names(): - imports = ["from .m import foo, bar", "from .m import bar, baz"] - result = _merge_from_imports(imports) - assert result == ["from .m import bar, baz, foo"] - - -def test_merge_from_imports_preserves_plain_imports(): - imports = ["import os", "from .m import x", "import sys"] - result = _merge_from_imports(imports) - assert result == ["from .m import x", "import os", "import sys"] - - -def test_merge_from_imports_empty(): - assert _merge_from_imports([]) == [] - - -# --------------------------------------------------------------------------- -# _sort_imports_pep8 -# --------------------------------------------------------------------------- - - -def test_sort_imports_pep8_basic_ordering(): - # Third-party plain import after relative from-import → should be reordered. - imports = [ - "from typing import Any", - "from .conversion import foo", - "import lupa", - ] - result = _sort_imports_pep8(imports) - assert result == [ - "from typing import Any", - "import lupa", - "from .conversion import foo", - ] - - -def test_sort_imports_pep8_future_first(): - imports = ["import os", "from __future__ import annotations", "from .x import y"] - result = _sort_imports_pep8(imports) - assert result[0] == "from __future__ import annotations" - - -def test_sort_imports_pep8_preserves_within_group_order(): - imports = ["from .b import y", "from .a import x"] - result = _sort_imports_pep8(imports) - # Both are local; original order preserved - assert result == ["from .b import y", "from .a import x"] - - -def test_sort_imports_pep8_empty(): - assert _sort_imports_pep8([]) == [] - - -def test_sort_imports_pep8_all_stdlib(): - imports = ["import os", "import sys", "from pathlib import Path"] - result = _sort_imports_pep8(imports) - assert result == imports # already ordered, stable sort keeps original order - - -# --------------------------------------------------------------------------- -# generate_file_splits — cross-file import integration -# --------------------------------------------------------------------------- - - -def test_generate_cross_file_import(): - # fn_a goes to fn_module.py; _block_1 (defining _CONST) goes to constants.py. - # _CONST is a TOP_LEVEL variable that is never reassigned → fn_module.py uses - # a plain "from .constants import _CONST" (idiomatic Python; no module alias). - source = "_CONST = 42\n\ndef fn_a():\n return _CONST\n" - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) - e_fn = _make_entity("fn_a", 3, 4) - c = _classified(entities=[e_block, e_fn]) - plan = _plan( - [ - GroupPlacement(group=["_block_1"], target_file="constants.py"), - GroupPlacement(group=["fn_a"], target_file="fn_module.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - fn_src = result.new_files["fn_module.py"] - assert "from .constants import _CONST" in fn_src - assert "from . import constants" not in fn_src - assert "constants._CONST" not in fn_src - # constants.py should NOT have a cross-import (it defines _CONST, not uses it) - const_src = result.new_files["constants.py"] - assert "from .fn_module" not in const_src - - -def test_generate_cross_file_import_no_duplicate_names(): - # Two entities (fn_a and fn_b) migrate to the same new file. - # fn_a uses X and Z from helpers; fn_b uses Y and Z from helpers. - # X, Y, Z are TOP_LEVEL variables that are never reassigned → the new file - # gets ONE "from .constants import X, Y, Z" (no module alias needed). - source = textwrap.dedent( - """\ - X = 1 - Y = 2 - Z = 3 - - def fn_a(): - return X + Z - - def fn_b(): - return Y + Z - """ - ) - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["X", "Y", "Z"]) - e_a = _make_entity("fn_a", 5, 6) - e_b = _make_entity("fn_b", 8, 9) - c = _classified(entities=[e_block, e_a, e_b]) - plan = _plan( - [ - GroupPlacement(group=["_block_1"], target_file="constants.py"), - GroupPlacement(group=["fn_a", "fn_b"], target_file="funcs.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - funcs_src = result.new_files["funcs.py"] - # Both fn_a and fn_b are present - assert "def fn_a" in funcs_src - assert "def fn_b" in funcs_src - # Plain from-import (no module alias) since none of X/Y/Z are reassigned - assert "from .constants import" in funcs_src - assert "from . import constants" not in funcs_src - # Variables are referenced by their bare names, not as module attributes - assert "constants.X" not in funcs_src - assert "constants.Y" not in funcs_src - assert "constants.Z" not in funcs_src - - -def test_generate_cross_file_import_reassigned_uses_module_alias(): - # _CONST is defined by _block_1 (→ constants.py) AND reassigned by _block_2 - # (non-migrated, stays in big.py). Because _CONST is stored by a different - # entity, fn_module.py must use the module-alias form so that any mutation of - # _CONST propagates through the module reference rather than a stale copy. - source = textwrap.dedent( - """\ - _CONST = 42 - _CONST = int("99") - - def fn_a(): - return _CONST - """ - ) - e_block1 = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) - e_block2 = Entity(EntityKind.TOP_LEVEL, "_block_2", 2, 2, ["_CONST"]) - e_fn = _make_entity("fn_a", 4, 5) - c = _classified(entities=[e_block1, e_block2, e_fn]) - plan = _plan( - [ - GroupPlacement(group=["_block_1"], target_file="constants.py"), - GroupPlacement(group=["fn_a"], target_file="fn_module.py"), - # _block_2 stays (non-migrated) — its store makes _CONST "reassigned" - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - fn_src = result.new_files["fn_module.py"] - # _CONST is reassigned → module-alias import so mutations propagate. - assert "from . import constants" in fn_src - assert "constants._CONST" in fn_src - assert "from .constants import _CONST" not in fn_src - - -def test_generate_cross_file_reassigned_original_file_uses_module_alias(): - # _CONST is defined by _block_1 (migrated) and reassigned by _block_2 - # (non-migrated). - # The original file must rewrite both the load in fn_a AND the module-level - # store in _block_2 to constants._CONST so that the reassignment updates the - # value in constants.py rather than creating an orphaned local binding. - source = textwrap.dedent( - """\ - _CONST = 42 - _CONST = int("99") - - def fn_a(): - return _CONST - """ - ) - e_block1 = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) - e_block2 = Entity(EntityKind.TOP_LEVEL, "_block_2", 2, 2, ["_CONST"]) - e_fn = _make_entity("fn_a", 4, 5) - c = _classified(entities=[e_block1, e_block2, e_fn]) - plan = _plan( - [ - GroupPlacement(group=["_block_1"], target_file="constants.py"), - # _block_2 and fn_a stay (non-migrated) - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - assert not result.abort - orig = result.original_source - # Module-level import added for the module alias. - assert "from . import constants" in orig - # Both the store (_block_2) and the load (fn_a) are rewritten. - assert 'constants._CONST = int("99")' in orig - assert "return constants._CONST" in orig - # Must NOT bind _CONST as a bare name via from-import (would shadow the rewrite) - assert "from .constants import _CONST" not in orig - - -def test_generate_reassigned_all_entities_migrated_no_original_processing(): - # When ALL entities are migrated, non_migrated_entity_names is empty and the - # original-file module-alias processing block must be skipped without error. - source = "_CONST = 42\n_CONST = 99\n\ndef fn_a():\n return _CONST\n" - e_block1 = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) - e_block2 = Entity(EntityKind.TOP_LEVEL, "_block_2", 2, 2, ["_CONST"]) - e_fn = _make_entity("fn_a", 4, 5) - c = _classified(entities=[e_block1, e_block2, e_fn]) - plan = _plan( - [ - GroupPlacement(group=["_block_1"], target_file="constants.py"), - GroupPlacement(group=["_block_2", "fn_a"], target_file="funcs.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - # Does not abort or crash; original source may be minimal. - assert not result.abort - - -def test_generate_reassigned_two_entities_same_file_single_module_import(): - # Two entities in the same new file both reference a reassigned constant. - # The same "from . import constants" import must appear only once - # (seen_top_cross deduplication). - source = textwrap.dedent( - """\ - _CONST = 42 - _CONST = 99 - - def fn_a(): - return _CONST - - def fn_b(): - return _CONST - """ - ) - e_block1 = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) - e_block2 = Entity(EntityKind.TOP_LEVEL, "_block_2", 2, 2, ["_CONST"]) - e_fn_a = _make_entity("fn_a", 4, 5) - e_fn_b = _make_entity("fn_b", 7, 8) - c = _classified(entities=[e_block1, e_block2, e_fn_a, e_fn_b]) - plan = _plan( - [ - GroupPlacement(group=["_block_1"], target_file="constants.py"), - GroupPlacement(group=["fn_a", "fn_b"], target_file="funcs.py"), - # _block_2 stays non-migrated → makes _CONST "reassigned" - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - funcs_src = result.new_files["funcs.py"] - # The module import must appear exactly once despite two entities needing it. - import_lines = [ln for ln in funcs_src.splitlines() if "import constants" in ln] - assert len(import_lines) == 1 - - -def test_generate_aborts_when_test_class_used_in_decorator(): - # TestFixture (a Test* class) provides PARAMS used in a parametrize decorator - # on test_fn. If they are split into different files, TestFixture would need - # to be imported inline (to avoid pytest duplicate collection), but that - # import would not be in scope when the decorator is evaluated. - source = textwrap.dedent( - """\ - import pytest - - class TestFixture: - PARAMS = [1, 2, 3] - - @pytest.mark.parametrize("x", TestFixture.PARAMS) - def test_fn(x): - assert x - """ - ) - e_fixture = Entity(EntityKind.CLASS, "TestFixture", 3, 4, ["TestFixture"]) - e_fn = _make_entity("test_fn", 6, 8) - c = _classified(entities=[e_fixture, e_fn]) - plan = _plan( - [ - GroupPlacement(group=["TestFixture"], target_file="test_fixture.py"), - GroupPlacement(group=["test_fn"], target_file="test_fns.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "tests/test_original.py") - - assert result.abort - assert "TestFixture" in result.abort_reason - assert "decorator" in result.abort_reason - - -def test_generate_non_migrated_helper_extracted_to_new_file(): - # _run is non-migrated; test_fn is migrated and references _run. - # _run is extracted into test_helpers.py to prevent an O→F→O cycle. - source = textwrap.dedent( - """\ - import textwrap - - def _run(x): - return x - - def test_fn(): - return _run(1) - """ - ) - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["textwrap"]) - e_run = _make_entity("_run", 3, 4) - e_test = _make_entity("test_fn", 6, 7) - c = _classified(entities=[e_block, e_run, e_test]) - plan = _plan([GroupPlacement(group=["test_fn"], target_file="test_helpers.py")]) - - result = generate_file_splits(c, plan, source, "original.py") - - new_src = result.new_files["test_helpers.py"] - # _run is defined in the new file (extracted), not imported from original - assert "def _run" in new_src - assert "from .original import _run" not in new_src - # import textwrap is not referenced by either entity - assert "from .original import textwrap" not in new_src - - -def test_generate_self_referential_placement_dropped(): - # LLM names a target file the same as the original → would create a - # circular import. The placement must be silently dropped so the entity - # stays in the original file and no self-import is added. - source = "class Foo:\n pass\n\nclass Bar:\n pass\n" - e_foo = _make_entity("Foo", 1, 2) - e_bar = _make_entity("Bar", 4, 5) - c = _classified(entities=[e_foo, e_bar]) - # "mymodule.py" is also the original filename → self-referential - plan = _plan( - [ - GroupPlacement(group=["Foo"], target_file="mymodule.py"), - GroupPlacement(group=["Bar"], target_file="helpers.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "mymodule.py") - - # Foo stays in the original — no circular self-import - assert "from .mymodule import Foo" not in result.original_source - assert "mymodule.py" not in result.new_files - # Bar is still moved normally - assert "helpers.py" in result.new_files - assert "class Bar" in result.new_files["helpers.py"] - # Foo remains in the original source (not removed) - assert "class Foo" in result.original_source - - -def test_generate_all_placements_self_referential(): - # All placements target the original file → nothing is moved. - source = "def foo():\n pass\n" - e_foo = _make_entity("foo", 1, 2) - c = _classified(entities=[e_foo]) - plan = _plan([GroupPlacement(group=["foo"], target_file="original.py")]) - - result = generate_file_splits(c, plan, source, "original.py") - - assert result.new_files == {} - assert "from .original import foo" not in result.original_source - assert "def foo" in result.original_source - - -def test_generate_aborts_on_cross_file_import_cycle(): - # fn_a references fn_b (in b.py) and fn_b references fn_a (in a.py). - # This creates a circular import a.py ↔ b.py that Python cannot load. - # generate_file_splits must detect the cycle and abort rather than emit - # broken code. - source = "def fn_a():\n return fn_b()\n\ndef fn_b():\n return fn_a()\n" - e_a = _make_entity("fn_a", 1, 2) - e_b = _make_entity("fn_b", 4, 5) - c = _classified(entities=[e_a, e_b]) - plan = _plan( - [ - GroupPlacement(group=["fn_a"], target_file="a.py"), - GroupPlacement(group=["fn_b"], target_file="b.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - assert result.abort is True - assert result.new_files == {} - - -def test_generate_aborts_on_cycle_through_original(): - # _CONST is a TOP_LEVEL constant (stays in original). - # _worker is migrated to helpers.py and references _CONST. - # main() (non-migrated) calls _worker → original will re-export _worker. - # Cycle: original → helpers.py (re-export of _worker) - # → original (via `from .original import _CONST`). - source = textwrap.dedent( - """\ - _CONST = "value" - - def _worker(): - return _CONST - - def main(): - return _worker() - """ - ) - e_const = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) - e_worker = _make_entity("_worker", 3, 4) - e_main = _make_entity("main", 6, 7) - c = _classified(entities=[e_const, e_worker, e_main]) - plan = _plan([GroupPlacement(group=["_worker"], target_file="helpers.py")]) - - result = generate_file_splits(c, plan, source, "original.py") - - # helpers.py would need `from .original import _CONST` while original - # re-exports _worker from helpers.py → circular import → must abort. - assert result.abort is True - assert result.new_files == {} - - -def test_generate_aborts_on_cycle_through_original_test_subdir(): - # In a test-file subdir split non_migrated_home ("test_svc.py") differs - # from original_basename ("svc/__init__.py"). The cycle detection must - # treat the original test file as its own graph node: - # - # _CONFIG stays in test_svc.py (TOP_LEVEL, non-migrated). - # _helper is migrated to svc/test_helpers.py and references _CONFIG. - # test_fn (non-migrated) calls _helper → test_svc.py re-exports _helper. - # Cycle: test_svc.py → svc/test_helpers.py (re-export of _helper) - # → test_svc.py (via `from ..test_svc import _CONFIG`). - source = textwrap.dedent( - """\ - _CONFIG = "value" - - def _helper(): - return _CONFIG - - def test_fn(): - return _helper() - """ - ) - e_config = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONFIG"]) - e_helper = _make_entity("_helper", 3, 4) - e_test = _make_entity("test_fn", 6, 7) - c = _classified(entities=[e_config, e_helper, e_test]) - plan = _plan([GroupPlacement(group=["_helper"], target_file="svc/test_helpers.py")]) - - result = generate_file_splits( - c, plan, source, "tests/test_svc.py", subdir_name="svc" - ) - - # svc/test_helpers.py imports _CONFIG from test_svc.py, and test_svc.py - # re-exports _helper from svc/test_helpers.py → circular import → abort. - assert result.abort is True - assert result.new_files == {} - - -# --------------------------------------------------------------------------- -# generate_file_splits — TYPE_CHECKING imports for quoted annotations -# --------------------------------------------------------------------------- - - -def test_generate_file_splits_type_checking_for_quoted_annotation(): - # _advise_set3 uses Optional["_LLMAccumulator"] (quoted annotation). - # _LLMAccumulator is migrated to models.py; _advise_set3 goes to placements.py. - # placements.py must get: - # from typing import TYPE_CHECKING - # if TYPE_CHECKING: - # from .models import _LLMAccumulator - source = textwrap.dedent( - """\ - from typing import Optional - - class _LLMAccumulator: - pass - - def _advise_set3(acc: Optional["_LLMAccumulator"]) -> None: - pass - """ - ) - e_acc = Entity(EntityKind.CLASS, "_LLMAccumulator", 3, 4, ["_LLMAccumulator"]) - e_fn = Entity(EntityKind.FUNCTION, "_advise_set3", 6, 7, ["_advise_set3"]) - c = _classified(entities=[e_acc, e_fn]) - plan = _plan( - [ - GroupPlacement(group=["_LLMAccumulator"], target_file="models.py"), - GroupPlacement(group=["_advise_set3"], target_file="placements.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "advisor.py") - - placements_src = result.new_files["placements.py"] - # TYPE_CHECKING may be merged into an existing "from typing import ..." line. - assert "TYPE_CHECKING" in placements_src - assert "if TYPE_CHECKING:" in placements_src - assert "from .models import _LLMAccumulator" in placements_src - - -def test_generate_file_splits_type_checking_deduplication(): - # Two functions in the same target file both reference "_LLMAccumulator" - # in quoted annotations. The TYPE_CHECKING import should appear only once - # even though both entities trigger _find_cross_file_type_checking_imports. - source = textwrap.dedent( - """\ - from typing import Optional - - class _LLMAccumulator: - pass - - def _fn_a(x: Optional["_LLMAccumulator"]) -> None: - pass - - def _fn_b(y: Optional["_LLMAccumulator"]) -> None: - pass - """ - ) - e_acc = Entity(EntityKind.CLASS, "_LLMAccumulator", 3, 4, ["_LLMAccumulator"]) - e_fna = Entity(EntityKind.FUNCTION, "_fn_a", 6, 7, ["_fn_a"]) - e_fnb = Entity(EntityKind.FUNCTION, "_fn_b", 9, 10, ["_fn_b"]) - c = _classified(entities=[e_acc, e_fna, e_fnb]) - plan = _plan( - [ - GroupPlacement(group=["_LLMAccumulator"], target_file="models.py"), - GroupPlacement(group=["_fn_a", "_fn_b"], target_file="placements.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "advisor.py") - - placements_src = result.new_files["placements.py"] - assert placements_src.count("from .models import _LLMAccumulator") == 1 - - -def test_generate_file_splits_tc_dedup_drops_when_already_in_regular(): - # Entity A uses _Acc at runtime (unquoted annotation → regular cross-file import). - # Entity B uses _Acc only in a quoted annotation → would normally get a TC import. - # Both go to workers.py. The dedup step must remove the TC import entirely since - # _Acc is already covered by the regular import. - source = textwrap.dedent( - """\ - from typing import Optional - - class _Acc: - pass - - def fn_runtime(x) -> None: - a: _Acc = x - - def fn_quoted(x: Optional["_Acc"]) -> None: - pass - """ - ) - e_acc = Entity(EntityKind.CLASS, "_Acc", 3, 4, ["_Acc"]) - e_rt = Entity(EntityKind.FUNCTION, "fn_runtime", 6, 7, ["fn_runtime"]) - e_qt = Entity(EntityKind.FUNCTION, "fn_quoted", 9, 10, ["fn_quoted"]) - c = _classified(entities=[e_acc, e_rt, e_qt]) - plan = _plan( - [ - GroupPlacement(group=["_Acc"], target_file="models.py"), - GroupPlacement(group=["fn_runtime", "fn_quoted"], target_file="workers.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "advisor.py") - - workers_src = result.new_files["workers.py"] - # Regular import must be present, TYPE_CHECKING block must NOT be. - assert "from .models import _Acc" in workers_src - assert "if TYPE_CHECKING:" not in workers_src - - -def test_generate_file_splits_tc_dedup_plain_import_branches(): - # Covers the non-from-import branches in the dedup loop: - # • "import sys" in needed → _FROM_IMPORT_RE does not match (2633->2631 branch) - # • "import typing_extensions" in needed_tc (annotation-only) → TC import is a - # plain import statement, not a from-import (2655 branch) - source = textwrap.dedent( - """\ - import sys - import typing_extensions - from typing import Optional - - class _Acc: - pass - - def fn(x: Optional["_Acc"]) -> None: - sys.exit(0) - - def fn2() -> "typing_extensions.Literal": - pass - """ - ) - e_acc = Entity(EntityKind.CLASS, "_Acc", 5, 6, ["_Acc"]) - e_fn = Entity(EntityKind.FUNCTION, "fn", 8, 9, ["fn"]) - e_fn2 = Entity(EntityKind.FUNCTION, "fn2", 11, 12, ["fn2"]) - c = _classified(entities=[e_acc, e_fn, e_fn2]) - plan = _plan( - [ - GroupPlacement(group=["_Acc"], target_file="models.py"), - GroupPlacement(group=["fn", "fn2"], target_file="workers.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "advisor.py") - - workers_src = result.new_files["workers.py"] - # TC import for _Acc (cross-file, quoted annotation) must still be present. - assert "if TYPE_CHECKING:" in workers_src - assert "_Acc" in workers_src - # Plain import for typing_extensions preserved in TC block. - assert "typing_extensions" in workers_src - - -# --------------------------------------------------------------------------- -# _extract_shared_helpers -# --------------------------------------------------------------------------- - - -def _make_classified(entities, migrated_names=None): - migrated = set(migrated_names or []) - return ( - ClassifiedEntities( - entities=entities, - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[], - set_3_groups=[], - abort=False, - ), - migrated, - ) - - -# --------------------------------------------------------------------------- -# _topo_depth -# --------------------------------------------------------------------------- - - -def test_topo_depth_empty(): - assert _topo_depth({}) == {} - - -def test_topo_depth_dag(): - # Linear chain: a → b → c. c is the leaf (depth 0), b has depth 1, a depth 2. - # The outer loop visits a first, which recurses into b then c, memoising both. - # When the outer loop reaches b and c they are already in depths (True branch). - graph = {"a": {"b"}, "b": {"c"}, "c": set()} - assert _topo_depth(graph) == {"a": 2, "b": 1, "c": 0} - - -def test_topo_depth_cycle(): - graph = {"a": {"b"}, "b": {"a"}} - assert _topo_depth(graph) == {"a": 0, "b": 0} - - -# --------------------------------------------------------------------------- -# _extract_shared_helpers -# --------------------------------------------------------------------------- - - -def test_extract_shared_helpers_extracts_referenced_function(): - # _helper is non-migrated, test_fn (migrated to helpers.py) references it. - e_helper = Entity(EntityKind.FUNCTION, "_helper", 1, 2, ["_helper"]) - e_test = Entity(EntityKind.FUNCTION, "test_fn", 4, 6, ["test_fn"]) - classified, migrated_names = _make_classified([e_helper, e_test], ["test_fn"]) - entity_map = {"_helper": e_helper, "test_fn": e_test} - entity_source_map = { - "_helper": "def _helper():\n pass", - "test_fn": "def test_fn():\n return _helper()", - } - file_entity_names = {"helpers.py": ["test_fn"]} - name_to_target_file = {"_helper": "original.py", "test_fn": "helpers.py"} - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - # _helper extracted into helpers.py (prepended before test_fn) - assert file_entity_names["helpers.py"] == ["_helper", "test_fn"] - assert "_helper" in migrated_names - assert name_to_target_file["_helper"] == "helpers.py" - assert len(synthetic) == 1 - assert synthetic[0].group == ["_helper"] - assert synthetic[0].target_file == "helpers.py" - - -def test_extract_shared_helpers_skips_top_level_entities(): - # TOP_LEVEL entities are not extracted (only FUNCTION/CLASS). - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) - e_test = Entity(EntityKind.FUNCTION, "test_fn", 3, 4, ["test_fn"]) - classified, migrated_names = _make_classified([e_block, e_test], ["test_fn"]) - entity_map = {"_block_1": e_block, "test_fn": e_test} - entity_source_map = { - "_block_1": "_CONST = 42", - "test_fn": "def test_fn():\n return _CONST", - } - file_entity_names = {"helpers.py": ["test_fn"]} - name_to_target_file = {"_CONST": "original.py", "test_fn": "helpers.py"} - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - assert "_block_1" not in migrated_names - assert file_entity_names["helpers.py"] == ["test_fn"] - assert synthetic == [] - - -def test_extract_shared_helpers_extracts_only_once_for_multiple_refs(): - # _helper referenced twice in the same migrated entity → extracted once. - e_helper = Entity(EntityKind.FUNCTION, "_helper", 1, 2, ["_helper"]) - e_test = Entity(EntityKind.FUNCTION, "test_fn", 4, 6, ["test_fn"]) - classified, migrated_names = _make_classified([e_helper, e_test], ["test_fn"]) - entity_map = {"_helper": e_helper, "test_fn": e_test} - entity_source_map = { - "_helper": "def _helper():\n pass", - "test_fn": "def test_fn():\n _helper()\n _helper()", - } - file_entity_names = {"helpers.py": ["test_fn"]} - name_to_target_file = {"_helper": "original.py", "test_fn": "helpers.py"} - - _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - assert file_entity_names["helpers.py"].count("_helper") == 1 - - -def test_extract_shared_helpers_skips_name_already_pointing_to_other_target(): - # A non-migrated FUNCTION entity whose defined name already points to a - # non-original target in name_to_target_file (e.g. a migrated entity also - # defines it) should not be added to defined_to_entity. - e_helper = Entity(EntityKind.FUNCTION, "_helper", 1, 2, ["_helper"]) - e_test = Entity(EntityKind.FUNCTION, "test_fn", 4, 5, ["test_fn"]) - classified, migrated_names = _make_classified([e_helper, e_test], ["test_fn"]) - entity_map = {"_helper": e_helper, "test_fn": e_test} - entity_source_map = { - "_helper": "def _helper(): pass", - "test_fn": "def test_fn(): return _helper()", - } - file_entity_names = {"helpers.py": ["test_fn"]} - # _helper already points to helpers.py (not original) — skip it - name_to_target_file = {"_helper": "helpers.py", "test_fn": "helpers.py"} - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - assert "_helper" not in migrated_names - assert synthetic == [] - - -def test_extract_shared_helpers_no_extraction_when_no_original_dep(): - # test_fn references other_fn which is also migrated → no extraction needed. - e_other = Entity(EntityKind.FUNCTION, "other_fn", 1, 2, ["other_fn"]) - e_test = Entity(EntityKind.FUNCTION, "test_fn", 4, 5, ["test_fn"]) - classified, migrated_names = _make_classified( - [e_other, e_test], ["test_fn", "other_fn"] - ) - entity_map = {"other_fn": e_other, "test_fn": e_test} - entity_source_map = { - "other_fn": "def other_fn():\n pass", - "test_fn": "def test_fn():\n return other_fn()", - } - file_entity_names = {"helpers.py": ["test_fn", "other_fn"]} - name_to_target_file = {"other_fn": "helpers.py", "test_fn": "helpers.py"} - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - assert synthetic == [] - assert file_entity_names["helpers.py"] == ["test_fn", "other_fn"] - - -def test_extract_shared_helpers_transitive_pull_in(): - # _helper_a is directly wanted by fn_a (in f1.py). - # _helper_a's source calls _helper_b (non-migrated, in original). - # _helper_b must be transitively extracted into f1.py to prevent an - # O→f1.py cycle (f1.py imports _helper_a which calls _helper_b in original; - # original re-exports _helper_a from f1.py → cycle). - e_a = Entity(EntityKind.FUNCTION, "_helper_a", 1, 2, ["_helper_a"]) - e_b = Entity(EntityKind.FUNCTION, "_helper_b", 3, 4, ["_helper_b"]) - e_fn = Entity(EntityKind.FUNCTION, "fn_a", 6, 7, ["fn_a"]) - classified, migrated_names = _make_classified([e_a, e_b, e_fn], ["fn_a"]) - entity_map = {"_helper_a": e_a, "_helper_b": e_b, "fn_a": e_fn} - entity_source_map = { - "_helper_a": "def _helper_a():\n _helper_b()", - "_helper_b": "def _helper_b():\n pass", - "fn_a": "def fn_a():\n _helper_a()", - } - file_entity_names = {"f1.py": ["fn_a"]} - name_to_target_file = { - "_helper_a": "original.py", - "_helper_b": "original.py", - "fn_a": "f1.py", - } - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - # Both helpers extracted into f1.py. - assert "_helper_a" in file_entity_names["f1.py"] - assert "_helper_b" in file_entity_names["f1.py"] - assert "_helper_a" in migrated_names - assert "_helper_b" in migrated_names - assert name_to_target_file["_helper_a"] == "f1.py" - assert name_to_target_file["_helper_b"] == "f1.py" - assert len(synthetic) == 2 - - -def test_extract_shared_helpers_scc_prevents_new_to_new_cycle(): - # helper_a is wanted by f1.py; helper_b is wanted by f2.py. - # They mutually reference each other → one SCC → must go to the same file - # to prevent the F1→F2→F1 import cycle. - e_a = Entity(EntityKind.FUNCTION, "helper_a", 1, 2, ["helper_a"]) - e_b = Entity(EntityKind.FUNCTION, "helper_b", 3, 4, ["helper_b"]) - e_fn1 = Entity(EntityKind.FUNCTION, "fn_1", 6, 7, ["fn_1"]) - e_fn2 = Entity(EntityKind.FUNCTION, "fn_2", 9, 10, ["fn_2"]) - classified = ClassifiedEntities( - entities=[e_a, e_b, e_fn1, e_fn2], - entity_class={}, - graph={ - "helper_a": {"helper_b"}, - "helper_b": {"helper_a"}, - "fn_1": set(), - "fn_2": set(), - }, - set_1=[], - set_2_groups=[], - set_3_groups=[], - abort=False, - ) - migrated_names = {"fn_1", "fn_2"} - entity_map = {"helper_a": e_a, "helper_b": e_b, "fn_1": e_fn1, "fn_2": e_fn2} - entity_source_map = { - "helper_a": "def helper_a():\n helper_b()", - "helper_b": "def helper_b():\n helper_a()", - "fn_1": "def fn_1():\n helper_a()", - "fn_2": "def fn_2():\n helper_b()", - } - file_entity_names = {"f1.py": ["fn_1"], "f2.py": ["fn_2"]} - name_to_target_file = { - "helper_a": "original.py", - "helper_b": "original.py", - "fn_1": "f1.py", - "fn_2": "f2.py", - } - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - # Both helpers must land in the same file (f1.py is first in plan order). - assert name_to_target_file["helper_a"] == name_to_target_file["helper_b"] - chosen = name_to_target_file["helper_a"] - assert "helper_a" in file_entity_names[chosen] - assert "helper_b" in file_entity_names[chosen] - assert "helper_a" in migrated_names - assert "helper_b" in migrated_names - # One synthetic placement covering both (single SCC). - assert len(synthetic) == 1 - assert set(synthetic[0].group) == {"helper_a", "helper_b"} - - -def test_extract_shared_helpers_transitive_dep_already_wanted(): - # helper_a is directly wanted by f1.py; helper_b is directly wanted by f2.py. - # helper_a's source also references helper_b (transitive), so helper_b's - # wanting-set grows from {f2.py} to {f1.py, f2.py} — True branch of the - # transitive update condition. - e_a = Entity(EntityKind.FUNCTION, "helper_a", 1, 2, ["helper_a"]) - e_b = Entity(EntityKind.FUNCTION, "helper_b", 3, 4, ["helper_b"]) - e_fn1 = Entity(EntityKind.FUNCTION, "fn_1", 6, 7, ["fn_1"]) - e_fn2 = Entity(EntityKind.FUNCTION, "fn_2", 9, 10, ["fn_2"]) - classified, migrated_names = _make_classified( - [e_a, e_b, e_fn1, e_fn2], ["fn_1", "fn_2"] - ) - entity_map = {"helper_a": e_a, "helper_b": e_b, "fn_1": e_fn1, "fn_2": e_fn2} - entity_source_map = { - "helper_a": "def helper_a():\n helper_b()", - "helper_b": "def helper_b():\n pass", - "fn_1": "def fn_1():\n helper_a()", - "fn_2": "def fn_2():\n helper_b()", - } - file_entity_names = {"f1.py": ["fn_1"], "f2.py": ["fn_2"]} - name_to_target_file = { - "helper_a": "original.py", - "helper_b": "original.py", - "fn_1": "f1.py", - "fn_2": "f2.py", - } - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - # Both helpers are extracted (as separate SCCs since no mutual cycle in graph). - assert "helper_a" in migrated_names - assert "helper_b" in migrated_names - # Two synthetic placements — one for each singleton SCC. - assert len(synthetic) == 2 - - -def test_extract_shared_helpers_transitive_dep_no_new_targets(): - # fn_1 directly references both helper_a and helper_b. - # helper_a's source also references helper_b (transitive dep). - # When the transitive loop processes helper_a, helper_b already has the same - # wanting-set {f1.py} → new_targets is empty → False branch of update condition. - e_a = Entity(EntityKind.FUNCTION, "helper_a", 1, 2, ["helper_a"]) - e_b = Entity(EntityKind.FUNCTION, "helper_b", 3, 4, ["helper_b"]) - e_fn = Entity(EntityKind.FUNCTION, "fn_1", 6, 7, ["fn_1"]) - classified, migrated_names = _make_classified([e_a, e_b, e_fn], ["fn_1"]) - entity_map = {"helper_a": e_a, "helper_b": e_b, "fn_1": e_fn} - entity_source_map = { - "helper_a": "def helper_a():\n helper_b()", - "helper_b": "def helper_b():\n pass", - "fn_1": "def fn_1():\n helper_a()\n helper_b()", - } - file_entity_names = {"f1.py": ["fn_1"]} - name_to_target_file = { - "helper_a": "original.py", - "helper_b": "original.py", - "fn_1": "f1.py", - } - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - # Both helpers are still extracted; the transitive dep on helper_b is a no-op - # because helper_b already has {f1.py} in its wanting-set (direct want). - assert "helper_a" in migrated_names - assert "helper_b" in migrated_names - assert len(synthetic) == 2 - - -def test_extract_shared_helpers_avoids_cycle_by_choosing_downstream_file(): - # _run is wanted by both test_skip.py and test_transformers.py. - # test_skip.py already imports from test_transformers.py (_RaisingTransformer). - # Placing _run in test_skip.py would force test_transformers.py to import from - # test_skip.py → cycle. The cycle-aware logic must pick test_transformers.py - # (the downstream file) instead. - e_raise = Entity( - EntityKind.FUNCTION, "_RaisingTransformer", 1, 3, ["_RaisingTransformer"] - ) - e_run = Entity(EntityKind.FUNCTION, "_run", 4, 5, ["_run"]) - e_skip = Entity(EntityKind.FUNCTION, "fn_skip", 7, 9, ["fn_skip"]) - e_transform = Entity(EntityKind.FUNCTION, "fn_transform", 11, 13, ["fn_transform"]) - classified, migrated_names = _make_classified( - [e_raise, e_run, e_skip, e_transform], - ["fn_skip", "fn_transform", "_RaisingTransformer"], - ) - entity_map = { - "_RaisingTransformer": e_raise, - "_run": e_run, - "fn_skip": e_skip, - "fn_transform": e_transform, - } - entity_source_map = { - "_RaisingTransformer": "def _RaisingTransformer():\n pass", - "_run": "def _run(x):\n return x", - # fn_skip refs _RaisingTransformer (migrated to test_transformers.py) AND - # _run (non-migrated) → _run is wanted by test_skip.py. - "fn_skip": "def fn_skip():\n _RaisingTransformer()\n _run(1)", - # fn_transform also refs _run → _run is wanted by test_transformers.py too. - "fn_transform": "def fn_transform():\n _run(2)", - } - file_entity_names = { - "test_skip.py": ["fn_skip"], - "test_transformers.py": ["fn_transform", "_RaisingTransformer"], - } - name_to_target_file = { - "_RaisingTransformer": "test_transformers.py", - "_run": "original.py", - "fn_skip": "test_skip.py", - "fn_transform": "test_transformers.py", - } - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - # _run must go to test_transformers.py, not test_skip.py. - assert name_to_target_file["_run"] == "test_transformers.py" - assert "_run" in file_entity_names["test_transformers.py"] - assert "_run" not in file_entity_names["test_skip.py"] - assert "_run" in migrated_names - assert len(synthetic) == 1 - assert synthetic[0].group == ["_run"] - assert synthetic[0].target_file == "test_transformers.py" - - -def test_extract_shared_helpers_skips_scc_when_no_cycle_free_placement(): - # fn_1 (in f1.py) refs fn_2 (in f2.py) and fn_2 refs fn_1 → pre-existing - # cycle in file_deps. fn_1 also refs helper_h (non-migrated), which itself - # refs fn_2. The only candidate for helper_h is f1.py; placing it there - # would still result in a cycle (f1.py→f2.py→f1.py already exists). - # Since no cycle-free placement exists, the SCC is skipped entirely. - e_fn1 = Entity(EntityKind.FUNCTION, "fn_1", 1, 2, ["fn_1"]) - e_fn2 = Entity(EntityKind.FUNCTION, "fn_2", 4, 5, ["fn_2"]) - e_h = Entity(EntityKind.FUNCTION, "helper_h", 7, 8, ["helper_h"]) - classified, migrated_names = _make_classified([e_fn1, e_fn2, e_h], ["fn_1", "fn_2"]) - entity_map = {"fn_1": e_fn1, "fn_2": e_fn2, "helper_h": e_h} - entity_source_map = { - "fn_1": "def fn_1():\n fn_2()\n helper_h()", - "fn_2": "def fn_2():\n fn_1()", - "helper_h": "def helper_h():\n fn_2()", - } - file_entity_names = {"f1.py": ["fn_1"], "f2.py": ["fn_2"]} - name_to_target_file = { - "fn_1": "f1.py", - "fn_2": "f2.py", - "helper_h": "original.py", - } - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - # helper_h is skipped — no placement avoids the pre-existing cycle. - assert "helper_h" not in migrated_names - assert synthetic == [] - - -def test_extract_shared_helpers_helper_refs_migrated_entity_in_other_file(): - # helper_a (non-migrated) references fn_2 (migrated to f2.py). - # When placed in f1.py the trial and apply phases must account for the - # resulting f1.py → f2.py dependency edge. - e_fn1 = Entity(EntityKind.FUNCTION, "fn_1", 1, 2, ["fn_1"]) - e_fn2 = Entity(EntityKind.FUNCTION, "fn_2", 4, 5, ["fn_2"]) - e_helper = Entity(EntityKind.FUNCTION, "helper_a", 7, 8, ["helper_a"]) - classified, migrated_names = _make_classified( - [e_fn1, e_fn2, e_helper], ["fn_1", "fn_2"] - ) - entity_map = {"fn_1": e_fn1, "fn_2": e_fn2, "helper_a": e_helper} - entity_source_map = { - "fn_1": "def fn_1():\n helper_a()", - "fn_2": "def fn_2():\n pass", - "helper_a": "def helper_a():\n fn_2()", - } - file_entity_names = {"f1.py": ["fn_1"], "f2.py": ["fn_2"]} - name_to_target_file = { - "fn_1": "f1.py", - "fn_2": "f2.py", - "helper_a": "original.py", - } - - synthetic = _extract_shared_helpers( - file_entity_names, - entity_source_map, - entity_map, - classified, - name_to_target_file, - migrated_names, - "original.py", - ) - - # helper_a is extracted to f1.py; its dep on fn_2 (f2.py) is tracked in - # both the trial and apply dep-file branches. - assert "helper_a" in migrated_names - assert name_to_target_file["helper_a"] == "f1.py" - assert len(synthetic) == 1 - assert synthetic[0].target_file == "f1.py" - - -def test_generate_no_circular_import_when_helper_referenced_by_migrated(): - # Integration test: _run stays in original and is used by test_fn (migrated). - # Without the fix: original → helpers.py (re-export) and helpers.py → original. - # With the fix: _run is moved into helpers.py; original imports _run from helpers. - source = textwrap.dedent( - """\ - def _run(x): - return x - - def test_fn(tmp_path): - return _run(tmp_path) - """ - ) - e_run = _make_entity("_run", 1, 2) - e_test = _make_entity("test_fn", 4, 5) - c = _classified(entities=[e_run, e_test]) - plan = _plan([GroupPlacement(group=["test_fn"], target_file="helpers.py")]) - - result = generate_file_splits(c, plan, source, "original.py") - - helpers_src = result.new_files["helpers.py"] - # _run is defined in helpers.py (extracted), not imported from original - assert "def _run" in helpers_src - assert "from .original import _run" not in helpers_src - # original re-imports _run from helpers.py (since it's still used there via - # non-migrated code — but in this minimal example there's nothing left) - # At minimum, no circular self-import exists - assert "from .original import" not in helpers_src - - -# --------------------------------------------------------------------------- -# _prune_unused_imports -# --------------------------------------------------------------------------- - - -def test_prune_unused_imports_syntax_error(): - # Unparseable source → returned unchanged. - source = "def (invalid syntax" - assert _prune_unused_imports(source) == source - - -def test_prune_unused_imports_no_replacements_needed(): - # All imports are fully used → fast-path returns source unchanged. - source = "import os\n\ndef f():\n os.getcwd()\n" - assert _prune_unused_imports(source) == source - - -def test_prune_unused_imports_preserves_future_import(): - # __future__ imports are always kept, even when the name isn't referenced. - source = "from __future__ import annotations\n\ndef f():\n pass\n" - result = _prune_unused_imports(source) - assert "from __future__ import annotations" in result - - -def test_prune_unused_imports_preserves_star_import(): - # Star imports cannot be pruned — kept as-is. - source = "from os.path import *\n\ndef f():\n pass\n" - result = _prune_unused_imports(source) - assert "from os.path import *" in result - - -def test_prune_unused_imports_removes_fully_unused_plain_import(): - # import whose name is never referenced is dropped entirely. - source = "import sys\n\ndef f():\n pass\n" - result = _prune_unused_imports(source) - assert "import sys" not in result - - -def test_prune_unused_imports_removes_fully_unused_from_import(): - # from-import whose names are never referenced is dropped entirely. - source = "from typing import Dict\n\ndef f():\n return 1\n" - result = _prune_unused_imports(source) - assert "from typing import" not in result - - -def test_prune_unused_imports_narrows_partial_from_import(): - # Only List is used — import narrowed to just List. - source = "from typing import Dict, List\n\ndef f(x: List):\n return x\n" - result = _prune_unused_imports(source) - assert "from typing import List" in result - assert "Dict" not in result - - -def test_prune_unused_imports_narrows_plain_import(): - # import x, y where only y is used → narrowed to import y. - source = "import os, sys\n\ndef f():\n sys.exit()\n" - result = _prune_unused_imports(source) - assert "import sys" in result - assert "os" not in result - - -def test_prune_unused_imports_multiline_import_collapsed(): - # Multi-line parenthesised import is collapsed to a single line. - source = textwrap.dedent( - """\ - from typing import ( - Dict, - List, - ) - - def f(x: List): - return x - """ - ) - result = _prune_unused_imports(source) - assert "from typing import List" in result - assert "Dict" not in result - assert "(\n" not in result - - -def test_prune_unused_imports_relative_import_narrowed(): - # Relative from-import is reconstructed with dots preserved. - source = "from .utils import foo, bar\n\ndef f():\n return foo()\n" - result = _prune_unused_imports(source) - assert "from .utils import foo" in result - assert "bar" not in result - - -def test_prune_unused_imports_preserves_noqa_f401(): - # Imports marked "# noqa: F401" are intentional re-export stubs and must - # never be pruned, even when the name is unused in the file body. - source = ( - "from .utils import _helper # fmt: skip # noqa: F401, E501\n" - "\n" - "def f():\n" - " pass\n" - ) - result = _prune_unused_imports(source) - assert "from .utils import _helper" in result - - -def test_prune_unused_imports_prunes_unused_without_noqa(): - # Without noqa, unused imports are still removed. - source = "from .utils import _helper\n\ndef f():\n pass\n" - result = _prune_unused_imports(source) - assert "from .utils import _helper" not in result - - -# --------------------------------------------------------------------------- -# generate_file_splits — import pruning integration -# --------------------------------------------------------------------------- - - -def test_generate_prunes_unused_names_from_multiname_import(): - # foo uses only List, not Dict; the new file's import should be narrowed. - source = "from typing import Dict, List\n\ndef foo(x: List):\n return x\n" - entity = _make_entity("foo", 3, 4) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["foo"], target_file="utils.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - new_src = result.new_files["utils.py"] - assert "from typing import List" in new_src - assert "Dict" not in new_src - - -def test_generate_prunes_fully_unused_import_from_original(): - # import os is only used by foo; after foo migrates the original no longer - # needs os, so the import should be removed. - source = "import os\n\ndef foo():\n os.getcwd()\n\ndef bar():\n return 1\n" - e_foo = _make_entity("foo", 3, 4) - e_bar = _make_entity("bar", 6, 7) - c = _classified(entities=[e_foo, e_bar]) - plan = _plan([GroupPlacement(group=["foo"], target_file="utils.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - assert "from .utils import foo" in result.original_source - assert "import os" not in result.original_source - assert "def bar():" in result.original_source - - -def test_generate_narrows_partial_unused_import_in_original(): - # foo uses Dict; bar uses List. After foo migrates, Dict should be - # stripped from the original's import while List is kept. - source = ( - "from typing import Dict, List\n\n" - "def foo(x: Dict):\n return x\n\n" - "def bar(x: List):\n return x\n" - ) - e_foo = _make_entity("foo", 3, 4) - e_bar = _make_entity("bar", 6, 7) - c = _classified(entities=[e_foo, e_bar]) - plan = _plan([GroupPlacement(group=["foo"], target_file="utils.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - assert "from typing import List" in result.original_source - assert "Dict" not in result.original_source - - -def test_generate_migrated_top_level_import_names_not_in_cross_file_imports(): - # Regression: when a TOP_LEVEL entity containing "from dataclasses import - # dataclass" is migrated, the name "dataclass" must NOT be added to the - # name→target-file map. A FUNCTION entity in a separate new file that also - # uses dataclass should get "from dataclasses import dataclass" (via - # _find_needed_imports) rather than "from .constants import dataclass" (a - # spurious cross-file import that would fail at runtime because constants.py - # never exports dataclass). - source = ( - "from dataclasses import dataclass\n\n" - "_CONST = 42\n\n" - "def make():\n return dataclass\n" - ) - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["dataclass", "_CONST"]) - e_make = _make_entity("make", 5, 6) - c = _classified(entities=[e_block, e_make]) - plan = _plan( - [ - GroupPlacement(group=["_block_1"], target_file="constants.py"), - GroupPlacement(group=["make"], target_file="utils.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - utils_src = result.new_files["utils.py"] - # Must import dataclass from the stdlib, not from constants.py - assert "from dataclasses import dataclass" in utils_src - assert "from .constants import dataclass" not in utils_src - - -# --------------------------------------------------------------------------- -# _find_project_root -# --------------------------------------------------------------------------- - - -def test_find_project_root_finds_pyproject_toml(tmp_path): - (tmp_path / "pyproject.toml").write_text("") - sub = tmp_path / "pkg" / "module.py" - sub.parent.mkdir() - sub.write_text("x = 1\n") - assert _find_project_root(sub) == tmp_path - - -def test_find_project_root_finds_git(tmp_path): - (tmp_path / ".git").mkdir() - sub = tmp_path / "module.py" - sub.write_text("x = 1\n") - assert _find_project_root(sub) == tmp_path - - -def test_find_project_root_called_with_directory(tmp_path): - (tmp_path / "pyproject.toml").write_text("") - assert _find_project_root(tmp_path) == tmp_path - - -def test_find_project_root_not_found(tmp_path): - # tmp_path is under /tmp which has no project markers → None. - sub = tmp_path / "module.py" - sub.write_text("x = 1\n") - result = _find_project_root(sub) - # If the test runner is inside a project that happens to include tmp_path - # (unlikely but possible with in-tree pytest), just ensure the function - # returns without crashing. The important coverage is the happy path above. - assert result is None or result.exists() - - -# --------------------------------------------------------------------------- -# _module_path_from_file -# --------------------------------------------------------------------------- - - -def test_module_path_from_file_success(tmp_path): - f = tmp_path / "pkg" / "utils.py" - f.parent.mkdir() - f.write_text("") - assert _module_path_from_file(tmp_path, f) == "pkg.utils" - - -def test_module_path_from_file_top_level(tmp_path): - f = tmp_path / "module.py" - f.write_text("") - assert _module_path_from_file(tmp_path, f) == "module" - - -def test_module_path_from_file_not_under_root(tmp_path): - other = tmp_path.parent / "other.py" - assert _module_path_from_file(tmp_path, other) is None - - -# --------------------------------------------------------------------------- -# _collect_external_imported_names -# --------------------------------------------------------------------------- - - -def test_collect_external_imported_names_relative_path(): - # Non-absolute path → empty set (no scan). - assert _collect_external_imported_names("relative/path.py") == set() - - -def test_collect_external_imported_names_nonexistent_file(tmp_path): - # Absolute but non-existent → empty set. - assert _collect_external_imported_names(str(tmp_path / "ghost.py")) == set() - - -def test_collect_external_imported_names_no_project_root(tmp_path): - # File exists but no pyproject.toml/.git above it → empty set. - f = tmp_path / "module.py" - f.write_text("x = 1\n") - # tmp_path is under /tmp which typically has no project markers. - result = _collect_external_imported_names(str(f)) - # May or may not find a root depending on environment; we just verify no crash. - assert isinstance(result, set) - - -def test_collect_external_imported_names_absolute_import(tmp_path): - (tmp_path / "pyproject.toml").write_text("") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("") - mod = pkg / "utils.py" - mod.write_text("def _helper():\n pass\n") - caller = tmp_path / "tests" / "test_utils.py" - caller.parent.mkdir() - caller.write_text("from mypkg.utils import _helper\n") - result = _collect_external_imported_names(str(mod)) - assert "_helper" in result - - -def test_collect_external_imported_names_relative_import(tmp_path): - (tmp_path / "pyproject.toml").write_text("") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("") - mod = pkg / "utils.py" - mod.write_text("def _helper():\n pass\n") - sibling = pkg / "other.py" - sibling.write_text("from .utils import _helper\n") - result = _collect_external_imported_names(str(mod)) - assert "_helper" in result - - -def test_collect_external_imported_names_self_excluded(tmp_path): - # The file being scanned is excluded from the search. - (tmp_path / "pyproject.toml").write_text("") - mod = tmp_path / "module.py" - mod.write_text("from module import _x\n") # self-referential (ignored) - result = _collect_external_imported_names(str(mod)) - assert "_x" not in result - - -def test_collect_external_imported_names_syntax_error_skipped(tmp_path): - (tmp_path / "pyproject.toml").write_text("") - mod = tmp_path / "module.py" - mod.write_text("def _helper(): pass\n") - bad = tmp_path / "bad.py" - bad.write_text("def (invalid\n") - good = tmp_path / "good.py" - good.write_text("from module import _helper\n") - result = _collect_external_imported_names(str(mod)) - assert "_helper" in result - - -def test_collect_external_imported_names_non_matching_import_ignored(tmp_path): - (tmp_path / "pyproject.toml").write_text("") - mod = tmp_path / "module.py" - mod.write_text("def _helper(): pass\n") - other = tmp_path / "other.py" - other.write_text("from different_module import _helper\n") - result = _collect_external_imported_names(str(mod)) - assert "_helper" not in result - - -def test_collect_external_imported_names_non_importfrom_nodes_skipped(tmp_path): - # Caller file contains a plain `import` statement (not ImportFrom) mixed - # with a matching `from … import`. The plain import must be skipped without - # crashing, and the matching ImportFrom still contributes to the result. - (tmp_path / "pyproject.toml").write_text("") - mod = tmp_path / "module.py" - mod.write_text("def _helper(): pass\n") - caller = tmp_path / "caller.py" - caller.write_text("import os\nfrom module import _helper\n") - result = _collect_external_imported_names(str(mod)) - assert "_helper" in result - - -def test_collect_external_imported_names_deep_relative_import(tmp_path): - # Two-level relative import: `from ..utils import _helper` - (tmp_path / "pyproject.toml").write_text("") - mod = tmp_path / "utils.py" - mod.write_text("def _helper(): pass\n") - sub = tmp_path / "pkg" / "sub" / "caller.py" - sub.parent.mkdir(parents=True) - sub.write_text("from ...utils import _helper\n") - result = _collect_external_imported_names(str(mod)) - assert "_helper" in result - - -def test_collect_external_imported_names_init_py_at_root(tmp_path): - # A bare __init__.py at the project root has no package prefix, so no - # external caller can import from it by package path — returns empty set. - (tmp_path / "pyproject.toml").write_text("") - init_py = tmp_path / "__init__.py" - init_py.write_text("class Foo: pass\n") - result = _collect_external_imported_names(str(init_py)) - assert result == set() - - -def test_collect_external_imported_names_init_py(tmp_path): - # When original_path is an __init__.py, callers import from the package - # name (e.g. "mypkg.sub"), not "mypkg.sub.__init__". - (tmp_path / "pyproject.toml").write_text("") - pkg = tmp_path / "mypkg" / "sub" - pkg.mkdir(parents=True) - init_py = pkg / "__init__.py" - init_py.write_text("class Foo: pass\n") - caller = tmp_path / "caller.py" - caller.write_text("from mypkg.sub import Foo\n") - result = _collect_external_imported_names(str(init_py)) - assert "Foo" in result - - -def test_collect_external_imported_names_init_py_relative_caller(tmp_path): - # Relative import from sibling module targeting a package __init__.py. - (tmp_path / "pyproject.toml").write_text("") - pkg = tmp_path / "mypkg" - pkg.mkdir() - sub = pkg / "sub" - sub.mkdir() - (sub / "__init__.py").write_text("def _helper(): pass\n") - sibling = pkg / "other.py" - sibling.write_text("from .sub import _helper\n") - result = _collect_external_imported_names(str(sub / "__init__.py")) - assert "_helper" in result - - -def test_collect_external_imported_names_relative_level_too_deep(tmp_path): - # Relative import that goes above the project root → skipped without crash. - (tmp_path / "pyproject.toml").write_text("") - mod = tmp_path / "utils.py" - mod.write_text("def _helper(): pass\n") - # A file at the top level trying to go up 5 packages (impossible). - top = tmp_path / "top.py" - top.write_text("from .....utils import _helper\n") - result = _collect_external_imported_names(str(mod)) - # The over-deep import is silently skipped; no crash. - assert isinstance(result, set) - - -# --------------------------------------------------------------------------- -# _add_re_exports — external_loads parameter -# --------------------------------------------------------------------------- - - -def test_add_re_exports_private_in_external_loads(): - # Private name not referenced in remaining source but present in external_loads - # → re-export proxy IS added so the external caller continues to work. - source = "import os\n" - entity = _make_entity("_helper", 1, 2) - placement = GroupPlacement(group=["_helper"], target_file="utils.py") - result = _add_re_exports( - source, [placement], {"_helper": entity}, {}, external_loads={"_helper"} - ) - assert "from .utils import _helper" in result - - -def test_add_re_exports_test_function_in_external_loads_not_re_exported(): - # test_ functions must never get a proxy even when listed in external_loads, - # because pytest would discover and run them twice. - source = "import os\n" - entity = _make_entity("test_something", 1, 2) - placement = GroupPlacement(group=["test_something"], target_file="helpers.py") - result = _add_re_exports( - source, - [placement], - {"test_something": entity}, - {}, - external_loads={"test_something"}, - ) - assert result == source - - -# --------------------------------------------------------------------------- -# _add_re_exports — reexport_mode parameter -# --------------------------------------------------------------------------- - - -def test_add_re_exports_mode_always_public_always_reexported(): - # "always" mode: public names are unconditionally re-exported (current behaviour). - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports( - source, [placement], {"foo": entity}, {}, reexport_mode="always" - ) - assert "from .utils import foo" in result - - -def test_add_re_exports_mode_application_non_test_public_reexported(): - # "application" mode + non-test file: public names are re-exported. - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports( - source, - [placement], - {"foo": entity}, - {}, - reexport_mode="application", - is_test_file=False, - ) - assert "from .utils import foo" in result - - -def test_add_re_exports_mode_application_test_file_public_not_reexported(): - # "application" mode + test file: public names are NOT unconditionally re-exported. - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports( - source, - [placement], - {"foo": entity}, - {}, - reexport_mode="application", - is_test_file=True, - ) - assert result == source - - -def test_add_re_exports_mode_application_test_file_in_external_loads_reexported(): - # "application" mode + test file: public name IS re-exported when in external_loads. - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports( - source, - [placement], - {"foo": entity}, - {}, - external_loads={"foo"}, - reexport_mode="application", - is_test_file=True, - ) - assert "from .utils import foo" in result - - -def test_add_re_exports_mode_application_test_file_public_in_still_loaded_reexported(): - # "application" mode + test file: public name IS re-exported when still referenced. - source = "import os\n\nfoo()\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports( - source, - [placement], - {"foo": entity}, - {}, - reexport_mode="application", - is_test_file=True, - ) - assert "from .utils import foo" in result - - -def test_add_re_exports_mode_imported_public_not_in_external_loads_not_reexported(): - # "imported" mode: public name is NOT re-exported if absent from external_loads. - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports( - source, [placement], {"foo": entity}, {}, reexport_mode="imported" - ) - assert result == source - - -def test_add_re_exports_mode_imported_public_in_external_loads_reexported(): - # "imported" mode: public name IS re-exported when in external_loads. - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports( - source, - [placement], - {"foo": entity}, - {}, - external_loads={"foo"}, - reexport_mode="imported", - ) - assert "from .utils import foo" in result - - -def test_add_re_exports_mode_imported_public_in_still_loaded_reexported(): - # "imported" mode: public name IS re-exported when still referenced in source. - source = "import os\n\nfoo()\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports( - source, [placement], {"foo": entity}, {}, reexport_mode="imported" - ) - assert "from .utils import foo" in result - - -def test_add_re_exports_mode_imported_private_in_external_loads_reexported(): - # "imported" mode: private names still follow the same rule (external_loads). - source = "import os\n" - entity = _make_entity("_helper", 1, 2) - placement = GroupPlacement(group=["_helper"], target_file="utils.py") - result = _add_re_exports( - source, - [placement], - {"_helper": entity}, - {}, - external_loads={"_helper"}, - reexport_mode="imported", - ) - assert "from .utils import _helper" in result - - -# --------------------------------------------------------------------------- -# generate_file_splits — private entity re-exported for external caller -# --------------------------------------------------------------------------- - - -def test_generate_private_entity_reexported_when_external_caller(tmp_path): - # Private entity is re-exported when an external file imports it. - (tmp_path / "pyproject.toml").write_text("") - pkg = tmp_path / "mypkg" - pkg.mkdir() - mod = pkg / "big.py" - mod.write_text("def _helper():\n pass\n") - caller = tmp_path / "tests" / "test_big.py" - caller.parent.mkdir() - caller.write_text("from mypkg.big import _helper\n") - - source = "def _helper():\n pass\n" - entity = _make_entity("_helper", 1, 2) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["_helper"], target_file="private.py")]) - - result = generate_file_splits(c, plan, source, str(mod)) - - assert "from .private import _helper" in result.original_source - - -def test_generate_file_splits_reexport_imported_public_not_reexported_without_caller( - tmp_path, -): - # "imported" mode: public entity not imported elsewhere → no re-export stub. - (tmp_path / "pyproject.toml").write_text("") - pkg = tmp_path / "mypkg" - pkg.mkdir() - mod = pkg / "big.py" - mod.write_text("def foo():\n pass\n") - # No external callers import foo. - - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["foo"], target_file="helpers.py")]) - - result = generate_file_splits(c, plan, source, str(mod), reexport_mode="imported") - - assert "from .helpers import foo" not in result.original_source - - -def test_generate_file_splits_reexport_mode_imported_public_reexported_with_caller( - tmp_path, -): - # "imported" mode: public entity imported elsewhere → re-export stub is added. - (tmp_path / "pyproject.toml").write_text("") - pkg = tmp_path / "mypkg" - pkg.mkdir() - mod = pkg / "big.py" - mod.write_text("def foo():\n pass\n") - caller = tmp_path / "other.py" - caller.write_text("from mypkg.big import foo\n") - - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["foo"], target_file="helpers.py")]) - - result = generate_file_splits(c, plan, source, str(mod), reexport_mode="imported") - - assert "from .helpers import foo" in result.original_source - - -def test_generate_file_splits_reexport_mode_always_public_reexported_without_caller( - tmp_path, -): - # "always" mode: public entity re-exported even when no external callers exist. - (tmp_path / "pyproject.toml").write_text("") - pkg = tmp_path / "mypkg" - pkg.mkdir() - mod = pkg / "big.py" - mod.write_text("def foo():\n pass\n") - - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["foo"], target_file="helpers.py")]) - - result = generate_file_splits(c, plan, source, str(mod), reexport_mode="always") - - assert "from .helpers import foo" in result.original_source - - -# --------------------------------------------------------------------------- -# _add_re_exports — # fmt: skip # noqa: F401, E501 for pure re-export imports -# --------------------------------------------------------------------------- - - -def test_add_re_exports_private_external_only_gets_noqa(): - # Private name in external_loads but NOT in remaining source → fmt: skip # noqa comment. - source = "import os\n" - entity = _make_entity("_helper", 1, 2) - placement = GroupPlacement(group=["_helper"], target_file="utils.py") - result = _add_re_exports( - source, [placement], {"_helper": entity}, {}, external_loads={"_helper"} - ) - assert "from .utils import _helper # fmt: skip # noqa: F401, E501" in result - - -def test_add_re_exports_private_in_still_loaded_no_noqa(): - # Private name referenced in remaining source but NOT in external_loads - # → re-export without noqa (it is actively used; no future-pruning risk). - source = "import os\n\n_helper()\n" - entity = _make_entity("_helper", 3, 3) - placement = GroupPlacement(group=["_helper"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"_helper": entity}, {}) - assert "from .utils import _helper\n" in result - assert "# noqa" not in result - - -def test_add_re_exports_private_in_still_loaded_and_external_loads_gets_noqa(): - # Private name referenced in remaining source AND in external_loads → noqa - # marker is added even though it is currently "used", because the non-migrated - # entity that uses it may itself be migrated in a later recursive split, at - # which point _prune_unused_imports would silently drop an un-annotated stub. - source = "import os\n\n_helper()\n" - entity = _make_entity("_helper", 3, 3) - placement = GroupPlacement(group=["_helper"], target_file="utils.py") - result = _add_re_exports( - source, [placement], {"_helper": entity}, {}, external_loads={"_helper"} - ) - assert "from .utils import _helper # fmt: skip # noqa: F401, E501" in result - - -def test_add_re_exports_public_not_in_still_loaded_gets_noqa(): - # Public name migrated but not referenced in remaining source → fmt: skip # noqa. - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"foo": entity}, {}) - assert "from .utils import foo # fmt: skip # noqa: F401, E501" in result - - -def test_add_re_exports_public_in_still_loaded_no_noqa(): - # Public name still referenced in remaining source → re-export without noqa. - source = "import os\n\nfoo()\n" - entity = _make_entity("foo", 3, 3) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"foo": entity}, {}) - assert "from .utils import foo\n" in result - assert "# noqa" not in result - - -def test_add_re_exports_multiple_noqa_each_on_own_line(): - # Two names both need noqa → one import line each so Black can't break the comment. - source = "import os\n" - entity = _make_entity("_block", 3, 4, ["_a", "_b"]) - placement = GroupPlacement(group=["_block"], target_file="utils.py") - result = _add_re_exports( - source, - [placement], - {"_block": entity}, - {}, - external_loads={"_a", "_b"}, - ) - lines = result.splitlines() - noqa_lines = [line for line in lines if "# fmt: skip # noqa: F401, E501" in line] - assert len(noqa_lines) == 2 - names = {line.split("import")[1].split("#")[0].strip() for line in noqa_lines} - assert names == {"_a", "_b"} - - -def test_add_re_exports_mixed_splits_into_two_lines(): - # One entity defines two names: one in still_loaded, one purely re-exported. - # Both are in external_loads, so both get # noqa: F401 to protect them from - # being pruned if the non-migrated entity that currently uses _used is itself - # migrated in a later recursive split. - source = "import os\n\n_used()\n" - entity = _make_entity("_block", 3, 4, ["_used", "_reexport"]) - placement = GroupPlacement(group=["_block"], target_file="utils.py") - result = _add_re_exports( - source, - [placement], - {"_block": entity}, - {}, - external_loads={"_used", "_reexport"}, - ) - lines = result.splitlines() - noqa_lines = [line for line in lines if "# fmt: skip # noqa: F401, E501" in line] - assert len(noqa_lines) == 2 - names = {line.split("import")[1].split("#")[0].strip() for line in noqa_lines} - assert names == {"_used", "_reexport"} - - -def test_add_re_exports_mixed_only_still_loaded_in_external_loads_gets_noqa(): - # When only the used name is in external_loads (not the purely re-exported one), - # verify external_loads membership drives noqa independently of still_loaded. - source = "import os\n\n_used()\n" - entity = _make_entity("_block", 3, 4, ["_used", "_reexport"]) - placement = GroupPlacement(group=["_block"], target_file="utils.py") - result = _add_re_exports( - source, - [placement], - {"_block": entity}, - {}, - external_loads={"_used"}, # only _used is externally imported - ) - lines = result.splitlines() - noqa_lines = [line for line in lines if "# fmt: skip # noqa: F401, E501" in line] - # _used is in still_loaded AND external_loads → gets noqa - assert len(noqa_lines) == 1 - assert "_used" in noqa_lines[0] - # _reexport is not in still_loaded and not in external_loads → not re-exported - assert "_reexport" not in result - - -def test_add_re_exports_is_test_file_adds_comment_before_first_noqa(): - # is_test_file=True → single explanatory comment inserted before the first - # F401 import; non-test files and test files with no noqa imports get no comment. - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports( - source, [placement], {"foo": entity}, {}, is_test_file=True - ) - lines = result.splitlines() - comment_idx = next( - ( - i - for i, l in enumerate(lines) - if "Re-exported for backwards compatibility" in l - ), - None, - ) - noqa_idx = next( - (i for i, l in enumerate(lines) if "# noqa: F401" in l), - None, - ) - assert comment_idx is not None - assert noqa_idx is not None - assert comment_idx == noqa_idx - 1 - - -def test_add_re_exports_is_test_file_false_no_comment(): - # is_test_file=False (default) → no explanatory comment added. - source = "import os\n" - entity = _make_entity("foo", 1, 2) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports(source, [placement], {"foo": entity}, {}) - assert "Re-exported for backwards compatibility" not in result - - -def test_add_re_exports_is_test_file_no_noqa_imports_no_comment(): - # is_test_file=True but all re-exports are already referenced in source - # (no noqa imports) → comment is not added. - source = "import os\n\nfoo()\n" - entity = _make_entity("foo", 3, 3) - placement = GroupPlacement(group=["foo"], target_file="utils.py") - result = _add_re_exports( - source, [placement], {"foo": entity}, {}, is_test_file=True - ) - assert "Re-exported for backwards compatibility" not in result - - -def test_add_re_exports_is_test_file_comment_added_once_for_multiple_noqa(): - # Multiple noqa imports → comment appears exactly once, before the first one. - source = "import os\n" - entity = _make_entity("_block", 1, 2, ["foo", "bar"]) - placement = GroupPlacement(group=["_block"], target_file="utils.py") - result = _add_re_exports( - source, [placement], {"_block": entity}, {}, is_test_file=True - ) - comment_count = result.count("Re-exported for backwards compatibility") - assert comment_count == 1 - - -def test_add_re_exports_is_test_file_comment_before_noqa_when_mixed(): - # is_test_file=True with a mix of used (no noqa) and pure re-export (noqa) - # imports: the comment must appear before the noqa line, not before the used line. - source = "import os\n\n_used()\n" - entity = _make_entity("_block", 3, 4, ["_used", "pub"]) - placement = GroupPlacement(group=["_block"], target_file="utils.py") - result = _add_re_exports( - source, [placement], {"_block": entity}, {}, is_test_file=True - ) - lines = result.splitlines() - comment_idx = next( - i for i, l in enumerate(lines) if "Re-exported for backwards" in l - ) - noqa_idx = next(i for i, l in enumerate(lines) if "# noqa: F401" in l) - used_idx = next(i for i, l in enumerate(lines) if "import _used" in l) - assert used_idx < comment_idx - assert comment_idx == noqa_idx - 1 - - -# --------------------------------------------------------------------------- -# _prune_inline_redundant_imports -# --------------------------------------------------------------------------- - - -def test_prune_inline_syntax_error(): - # Unparseable source → returned unchanged. - source = "def (invalid syntax" - assert _prune_inline_redundant_imports(source) == source - - -def test_prune_inline_no_top_level_imports(): - # No module-level imports → nothing can be redundant, return unchanged. - source = "def f():\n from os import path\n path.join('a', 'b')\n" - assert _prune_inline_redundant_imports(source) == source - - -def test_prune_inline_no_inner_imports(): - # Only top-level imports, no function-body imports → unchanged. - source = "import os\n\ndef f():\n return os.getcwd()\n" - assert _prune_inline_redundant_imports(source) == source - - -def test_prune_inline_no_redundancy(): - # Inner import brings in a different name than the top-level import. - source = "import os\n\ndef f():\n from sys import argv\n return argv\n" - assert _prune_inline_redundant_imports(source) == source - - -def test_prune_inline_removes_fully_redundant_from_import(): - # Top-level import covers all names in the inner from-import → remove it. - source = textwrap.dedent( - """\ - from unittest.mock import patch - from mymod import Foo - - def test_thing(): - from mymod import Foo - assert Foo() - """ - ) - result = _prune_inline_redundant_imports(source) - assert result.count("from mymod import Foo") == 1 - assert "assert Foo()" in result - - -def test_prune_inline_narrows_partially_redundant_from_import(): - # Only one of two inner names is already at top level → narrow the inner import. - source = textwrap.dedent( - """\ - from mymod import Foo - - def test_thing(): - from mymod import Foo, Bar - assert Foo() and Bar() - """ - ) - result = _prune_inline_redundant_imports(source) - lines = result.splitlines() - inner = [ln for ln in lines if "from mymod import" in ln and ln.startswith(" ")] - assert len(inner) == 1 - assert "Bar" in inner[0] - assert "Foo" not in inner[0] - - -def test_prune_inline_removes_fully_redundant_plain_import(): - # Inner ``import x`` where x is already available at top level → removed. - source = textwrap.dedent( - """\ - import os - - def f(): - import os - return os.getcwd() - """ - ) - result = _prune_inline_redundant_imports(source) - assert result.count("import os") == 1 - - -def test_prune_inline_narrows_partially_redundant_plain_import(): - # ``import os, sys`` inside function where os is already top-level → narrows to sys. - source = textwrap.dedent( - """\ - import os - - def f(): - import os, sys - return sys.argv - """ - ) - result = _prune_inline_redundant_imports(source) - inner = [ - ln - for ln in result.splitlines() - if ln.strip().startswith("import") and ln.startswith(" ") - ] - assert len(inner) == 1 - assert "sys" in inner[0] - assert "os" not in inner[0] - - -def test_prune_inline_preserves_indentation(): - # The narrowed replacement line must preserve the original indentation. - source = textwrap.dedent( - """\ - from mymod import Foo - - def test_thing(): - if True: - from mymod import Foo, Bar - assert Bar() - """ - ) - result = _prune_inline_redundant_imports(source) - inner = [ - ln - for ln in result.splitlines() - if "from mymod import" in ln and ln.startswith(" ") - ] - assert len(inner) == 1 - assert inner[0].startswith(" from mymod import Bar") - - -def test_prune_inline_preserves_type_checking_block(): - # Imports inside 'if TYPE_CHECKING:' must never be stripped even when the - # same name is already imported at module level — removing them would leave - # an empty (and syntactically invalid) if-block. - source = textwrap.dedent( - """\ - from typing import TYPE_CHECKING - from mymod import Foo - - if TYPE_CHECKING: - from mymod import Foo - """ - ) - result = _prune_inline_redundant_imports(source) - assert result == source - - -def test_generate_file_splits_removes_inline_redundant_imports(): - # When a split new file has both a top-level import and an inline re-import - # of the same name, the inline one should be removed. - source = textwrap.dedent( - """\ - from mymod import Helper - - def test_uses_helper(): - from mymod import Helper - assert Helper() - """ - ) - entity = _make_entity("test_uses_helper", 3, 5) - c = _classified(entities=[entity]) - plan = _plan( - [GroupPlacement(group=["test_uses_helper"], target_file="test_split.py")] - ) - result = generate_file_splits(c, plan, source, "big.py") - new_src = result.new_files["test_split.py"] - # The inline re-import should be removed; the module-level one covers it. - assert new_src.count("from mymod import Helper") == 1 - - -# --------------------------------------------------------------------------- -# _find_cross_file_imports — absolute import mode -# --------------------------------------------------------------------------- - - -def test_find_cross_file_imports_abs_pkg_package_prefix(): - # abs_pkg="tests" → "from tests.block_1 import _MODEL" - entity_source_map = {"fn_a": "def fn_a():\n return _MODEL\n"} - name_to_target_file = {"_MODEL": "block_1.py"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["fn_a"], entity_source_map, name_to_target_file, "test_fn.py", abs_pkg="tests" - ) - assert from_imports == ["from tests.block_1 import _MODEL"] - assert module_imports == [] - assert rewrites == {} - - -def test_find_cross_file_imports_abs_pkg_root_level(): - # abs_pkg="" → "from block_1 import _MODEL" (no package prefix) - entity_source_map = {"fn_a": "def fn_a():\n return _MODEL\n"} - name_to_target_file = {"_MODEL": "block_1.py"} - from_imports, module_imports, rewrites = _find_cross_file_imports( - ["fn_a"], entity_source_map, name_to_target_file, "test_fn.py", abs_pkg="" - ) - assert from_imports == ["from block_1 import _MODEL"] - assert module_imports == [] - assert rewrites == {} - - -# --------------------------------------------------------------------------- -# _find_cross_file_type_checking_imports -# --------------------------------------------------------------------------- - - -def test_find_cross_file_type_checking_imports_basic(): - # _LLMAccumulator appears only in a quoted annotation in fn_a. - # It lives in block_1.py — a TYPE_CHECKING import should be generated. - entity_source_map = { - "fn_a": 'def fn_a(x: Optional["_LLMAccumulator"]) -> None:\n pass\n' - } - name_to_target_file = {"_LLMAccumulator": "block_1.py"} - result = _find_cross_file_type_checking_imports( - ["fn_a"], entity_source_map, name_to_target_file, "placements.py" - ) - assert result == ["from .block_1 import _LLMAccumulator"] - - -def test_find_cross_file_type_checking_imports_same_file_excluded(): - # _LLMAccumulator goes to the same target file — no import needed. - entity_source_map = { - "fn_a": 'def fn_a(x: Optional["_LLMAccumulator"]) -> None:\n pass\n' - } - name_to_target_file = {"_LLMAccumulator": "placements.py"} - result = _find_cross_file_type_checking_imports( - ["fn_a"], entity_source_map, name_to_target_file, "placements.py" - ) - assert result == [] - - -def test_find_cross_file_type_checking_imports_runtime_excluded(): - # _LLMAccumulator is used at runtime (not just annotation) — excluded. - entity_source_map = {"fn_a": "def fn_a():\n return _LLMAccumulator()\n"} - name_to_target_file = {"_LLMAccumulator": "block_1.py"} - result = _find_cross_file_type_checking_imports( - ["fn_a"], entity_source_map, name_to_target_file, "placements.py" - ) - assert result == [] - - -def test_find_cross_file_type_checking_imports_no_annotations(): - # No quoted annotations at all → empty result. - entity_source_map = {"fn_a": "def fn_a():\n pass\n"} - name_to_target_file = {"_LLMAccumulator": "block_1.py"} - result = _find_cross_file_type_checking_imports( - ["fn_a"], entity_source_map, name_to_target_file, "placements.py" - ) - assert result == [] - - -def test_find_cross_file_type_checking_imports_not_in_map(): - # Referenced quoted name not in name_to_target_file → no import. - entity_source_map = {"fn_a": 'def fn_a(x: "UnknownType") -> None:\n pass\n'} - result = _find_cross_file_type_checking_imports( - ["fn_a"], entity_source_map, {}, "placements.py" - ) - assert result == [] - - -def test_find_cross_file_type_checking_imports_top_level_var_excluded(): - # A name in top_level_var_names is skipped (handled separately). - entity_source_map = { - "fn_a": 'def fn_a(x: Optional["SAFE_MODE"]) -> None:\n pass\n' - } - name_to_target_file = {"SAFE_MODE": "constants.py"} - result = _find_cross_file_type_checking_imports( - ["fn_a"], - entity_source_map, - name_to_target_file, - "placements.py", - top_level_var_names={"SAFE_MODE"}, - ) - assert result == [] - - -def test_find_cross_file_type_checking_imports_abs_pkg(): - # With abs_pkg set, use absolute import style. - entity_source_map = { - "fn_a": 'def fn_a(x: Optional["_LLMAccumulator"]) -> None:\n pass\n' - } - name_to_target_file = {"_LLMAccumulator": "block_1.py"} - result = _find_cross_file_type_checking_imports( - ["fn_a"], - entity_source_map, - name_to_target_file, - "test_fn.py", - abs_pkg="tests", - ) - assert result == ["from tests.block_1 import _LLMAccumulator"] - - -def test_find_cross_file_type_checking_imports_entity_not_in_map(): - # Entity not in entity_source_map → treated as empty, no imports. - result = _find_cross_file_type_checking_imports( - ["ghost"], {}, {"_X": "other.py"}, "placements.py" - ) - assert result == [] - - -# --------------------------------------------------------------------------- -# _abs_package_for_dir -# --------------------------------------------------------------------------- - - -def test_abs_package_for_dir_subdir(tmp_path): - (tmp_path / "pyproject.toml").touch() - tests_dir = tmp_path / "tests" - tests_dir.mkdir() - test_file = tests_dir / "test_engine.py" - test_file.touch() - assert _abs_package_for_dir(str(test_file)) == "tests" - - -def test_abs_package_for_dir_root_level(tmp_path): - (tmp_path / "pyproject.toml").touch() - test_file = tmp_path / "test_engine.py" - test_file.touch() - assert _abs_package_for_dir(str(test_file)) == "" - - -def test_abs_package_for_dir_no_project_root(monkeypatch): - monkeypatch.setattr( - "crispen.file_limiter.code_gen._find_project_root", lambda _p: None - ) - assert _abs_package_for_dir("/some/random/path/test_engine.py") is None - - -def test_abs_package_for_dir_non_ancestor_root(tmp_path, monkeypatch): - # Defensive branch: project root is not an ancestor of the file's directory. - other_dir = tmp_path / "other" - other_dir.mkdir() - monkeypatch.setattr( - "crispen.file_limiter.code_gen._find_project_root", lambda _p: other_dir - ) - test_file = tmp_path / "tests" / "test_engine.py" - test_file.parent.mkdir() - test_file.touch() - assert _abs_package_for_dir(str(test_file)) is None - - -# --------------------------------------------------------------------------- -# generate_file_splits — test file uses absolute imports -# --------------------------------------------------------------------------- - - -def test_generate_test_file_reexports_use_absolute_imports(tmp_path): - # When the original is a test file, re-exports in the updated original - # must use absolute imports so pytest can load the file. - (tmp_path / "pyproject.toml").touch() - tests_dir = tmp_path / "tests" - tests_dir.mkdir() - test_file = tests_dir / "test_engine.py" - test_file.write_text("") - - source = "import os\n\ndef foo():\n os.getcwd()\n" - entity = _make_entity("foo", 3, 4) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["foo"], target_file="test_helpers.py")]) - - result = generate_file_splits(c, plan, source, str(test_file)) - - assert "from tests.test_helpers import foo" in result.original_source - assert "from .test_helpers import foo" not in result.original_source - - -def test_generate_test_file_cross_imports_use_absolute_imports(tmp_path): - # Cross-file imports in generated test split files must also be absolute. - (tmp_path / "pyproject.toml").touch() - tests_dir = tmp_path / "tests" - tests_dir.mkdir() - test_file = tests_dir / "test_engine.py" - test_file.write_text("") - - source = "_CONST = 42\n\ndef test_fn():\n return _CONST\n" - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, ["_CONST"]) - e_fn = _make_entity("test_fn", 3, 4) - c = _classified(entities=[e_block, e_fn]) - plan = _plan( - [ - GroupPlacement(group=["_block_1"], target_file="test_constants.py"), - GroupPlacement(group=["test_fn"], target_file="test_fns.py"), - ] - ) - - result = generate_file_splits(c, plan, source, str(test_file)) - - fn_src = result.new_files["test_fns.py"] - # _CONST is a TOP_LEVEL variable that is never reassigned → plain absolute - # from-import (idiomatic Python; module alias only needed if reassigned). - assert "from tests.test_constants import _CONST" in fn_src - assert "import tests.test_constants as test_constants" not in fn_src - assert "test_constants._CONST" not in fn_src - - -# --------------------------------------------------------------------------- -# _add_re_exports — relative_from (subdir __init__.py) -# --------------------------------------------------------------------------- - - -def test_add_re_exports_relative_from_uses_relative_prefix(): - # When relative_from is set, imports are computed via _relative_import_prefix - # rather than _target_module_name, so "service/__init__.py" → ".utils" - # (not ".service.utils"). - source = "# stayed\n" - entity = _make_entity("Foo", 1, 1) - placements = [GroupPlacement(group=["Foo"], target_file="service/utils.py")] - entity_map = {"Foo": entity} - entity_source_map = {"Foo": "class Foo: pass"} - - result = _add_re_exports( - source, - placements, - entity_map, - entity_source_map, - relative_from="service/__init__.py", - ) - - assert "from .utils import Foo" in result - # Must NOT use the fully-qualified form that would be wrong from __init__.py. - assert "from .service.utils" not in result - - -# --------------------------------------------------------------------------- -# generate_file_splits — subdir_name parameter -# --------------------------------------------------------------------------- - - -def test_generate_file_splits_subdir_name_uses_init_as_original_basename(): - # When subdir_name="service", the dependency graph treats "service/__init__.py" - # as the original file node. Because main (public) is re-exported from - # __init__, _extract_shared_helpers pulls helper into service/main.py to - # break the __init__ → main → __init__ cycle. The split must not abort. - source = "def helper():\n return 1\n\ndef main():\n return helper()\n" - e_helper = _make_entity("helper", 1, 2) - e_main = _make_entity("main", 4, 5) - c = _classified(entities=[e_helper, e_main]) - # Only main is migrated; helper stays in "original" (→ service/__init__.py). - plan = _plan([GroupPlacement(group=["main"], target_file="service/main.py")]) - - result = generate_file_splits(c, plan, source, "service.py", subdir_name="service") - - assert not result.abort - # helper is extracted into service/main.py to break the re-export cycle. - main_src = result.new_files["service/main.py"] - assert "def helper" in main_src - assert "def main" in main_src - # Re-exports use the short relative prefix ".main", not ".service.main". - assert "from .main import" in result.original_source - assert "from .service.main" not in result.original_source - - -def test_generate_file_splits_subdir_name_re_exports_use_relative_prefix(): - # With subdir_name set (non-test), re-exports in the "original" source - # (which becomes __init__.py) use ".utils" not ".service.utils". - # target_file already has the "service/" prefix (added by runner.py). - source = "def foo():\n pass\n" - e_foo = _make_entity("foo", 1, 2) - c = _classified(entities=[e_foo]) - plan = _plan([GroupPlacement(group=["foo"], target_file="service/utils.py")]) - - result = generate_file_splits(c, plan, source, "service.py", subdir_name="service") - - assert not result.abort - assert "from .utils import foo" in result.original_source - assert "from .service.utils" not in result.original_source - - -def test_generate_file_splits_subdir_name_cross_file_uses_relative(): - # In subdir mode, cross-file imports between new files use relative imports - # even when the original file is a test (abs_pkg would normally apply). - # NOTE: runner.py prefixes target_file with subdir_name before this call, - # so target_files already include "svc/" here. - source = "def helper():\n return 1\n\ndef test_fn():\n return helper()\n" - e_helper = _make_entity("helper", 1, 2) - e_test = _make_entity("test_fn", 4, 5) - c = _classified(entities=[e_helper, e_test]) - plan = _plan( - [ - GroupPlacement(group=["helper"], target_file="svc/helpers.py"), - GroupPlacement(group=["test_fn"], target_file="svc/test_fns.py"), - ] - ) - - # Use a path that looks like a test file so abs_pkg would normally be set. - result = generate_file_splits( - c, plan, source, "tests/test_svc.py", subdir_name="svc" - ) - - assert not result.abort - # Cross-file import from test_fns.py to helpers.py should be relative. - test_src = result.new_files["svc/test_fns.py"] - assert "from .helpers import helper" in test_src - - -def test_generate_file_splits_test_subdir_nonmigrated_imports_from_original(): - # Non-migrated TOP_LEVEL variables (e.g. module-level constants) stay in - # the original test file. A new subfile that references a constant that is - # never reassigned should use a plain ``from`` import (idiomatic Python); - # module-alias access is only needed when the constant is mutated at runtime. - source = "_CONFIG = 'val'\n\ndef test_fn():\n return _CONFIG\n" - # Use TOP_LEVEL kind so _extract_shared_helpers does not pull _CONFIG into - # the new file (it only extracts FUNCTION/CLASS entities). - e_config = Entity(EntityKind.TOP_LEVEL, "_CONFIG", 1, 1, ["_CONFIG"]) - e_test = _make_entity("test_fn", 3, 4) - c = _classified(entities=[e_config, e_test]) - plan = _plan([GroupPlacement(group=["test_fn"], target_file="svc/test_fns.py")]) - - result = generate_file_splits( - c, plan, source, "tests/test_svc.py", subdir_name="svc" - ) - - assert not result.abort - test_src = result.new_files["svc/test_fns.py"] - # _CONFIG is never reassigned → plain from-import (no module alias). - assert "from ..test_svc import _CONFIG" in test_src - assert "from .. import test_svc" not in test_src - assert "test_svc._CONFIG" not in test_src - - -def test_generate_file_splits_has_main_uses_filename_as_original_basename(): - # When has_main=True, original_basename is the flat filename ("service.py"), - # not "service_lib/__init__.py". Re-exports in the original file reference - # the subdir modules directly (e.g. "from service_lib.utils import foo"). - source = "def foo():\n pass\n\nif __name__ == '__main__':\n foo()\n" - e_foo = _make_entity("foo", 1, 2) - c = _classified(entities=[e_foo]) - plan = _plan([GroupPlacement(group=["foo"], target_file="service_lib/utils.py")]) - - result = generate_file_splits( - c, plan, source, "service.py", subdir_name="service_lib", has_main=True - ) - - assert not result.abort - # Re-export in original file uses the subdir module path. - assert "service_lib" in result.original_source - # No __init__.py is created by code_gen (the runner handles that decision). - assert "service_lib/__init__.py" not in result.new_files - - -# --------------------------------------------------------------------------- -# _bump_relative_imports -# --------------------------------------------------------------------------- - - -def test_bump_relative_imports_single_dot(): - assert _bump_relative_imports("from .foo import bar") == "from ..foo import bar" - - -def test_bump_relative_imports_two_dots(): - assert _bump_relative_imports("from .. import baz") == "from ... import baz" - - -def test_bump_relative_imports_leaves_absolute(): - src = "import os\nfrom typing import List" - assert _bump_relative_imports(src) == src - - -def test_bump_relative_imports_multiline(): - src = "from .a import x\nimport sys\nfrom ..b import y\n" - result = _bump_relative_imports(src) - assert "from ..a import x" in result - assert "from ...b import y" in result - assert "import sys" in result - - -def test_bump_relative_imports_n_two(): - assert _bump_relative_imports("from .. import foo", n=2) == "from .... import foo" - - -def test_bump_relative_imports_n_zero(): - src = "from .foo import bar" - assert _bump_relative_imports(src, n=0) == src - - -# generate_file_splits — subdir_name bumps relative imports - - -def test_generate_file_splits_subdir_bumps_needed_imports(): - # In subdir-split mode, relative imports from the original file that appear - # in new sub-files must be incremented by one level so they still resolve - # correctly from inside the subdirectory package. - source = "from .sibling import CONST\n\ndef foo():\n return CONST\n" - e_foo = _make_entity("foo", 3, 4) - c = _classified(entities=[e_foo]) - plan = _plan([GroupPlacement(group=["foo"], target_file="service/utils.py")]) - - result = generate_file_splits(c, plan, source, "service.py", subdir_name="service") - - assert not result.abort - utils_src = result.new_files["service/utils.py"] - assert "from ..sibling import CONST" in utils_src - assert "from .sibling import CONST" not in utils_src - - -def test_generate_file_splits_subdir_bumps_init_imports(): - # In subdir-split mode, relative imports in the non-migrated original source - # (which becomes subdir/__init__.py) must also be bumped by one level so - # they still point at the correct modules from inside the package. - source2 = ( - "from .. import llm_client\n" - "from .base import Base\n\n" - "def stayed():\n return llm_client, Base\n\n" - "def migrated():\n pass\n" - ) - e_stayed2 = _make_entity("stayed", 4, 5) - e_migrated2 = _make_entity("migrated", 7, 8) - c = _classified(entities=[e_stayed2, e_migrated2]) - plan = _plan([GroupPlacement(group=["migrated"], target_file="pkg/helpers.py")]) - - result = generate_file_splits(c, plan, source2, "pkg.py", subdir_name="pkg") - - assert not result.abort - init_src = result.original_source - assert "from ... import llm_client" in init_src - assert "from ..base import Base" in init_src - assert "from .. import llm_client" not in init_src - assert "from .base import Base" not in init_src - - -def test_generate_file_splits_subdir_bumps_two_levels_deep(): - # When the LLM places a new file two directories deep (e.g. - # "pkg/pkg/core.py"), relative imports must be bumped by 2 dots, not 1. - # This matches the real-world scenario where subdir_name="pkg" but the - # advisor proposes "pkg/pkg/core.py" as a target. - source = "from .. import llm_client\n\ndef func():\n return llm_client\n" - e_func = _make_entity("func", 3, 4) - c = _classified(entities=[e_func]) - plan = _plan([GroupPlacement(group=["func"], target_file="pkg/pkg/core.py")]) - - result = generate_file_splits(c, plan, source, "pkg.py", subdir_name="pkg") - - assert not result.abort - core_src = result.new_files["pkg/pkg/core.py"] - # 2 levels deep → original ".." becomes "...." (4 dots) - assert "from .... import llm_client" in core_src - assert "from .. import llm_client" not in core_src - assert "from ... import llm_client" not in core_src - - -def test_generate_file_splits_subdir_injects_tc_import_for_nonmigrated_entity(): - # When a _block_N TOP_LEVEL entity that holds the `if TYPE_CHECKING:` block - # is migrated to a sub-file, any non-migrated entity that references the - # guarded name in a quoted annotation must receive the TYPE_CHECKING import - # in the updated original (__init__.py). - # - # The original file has three entities: - # _block_1 — the TYPE_CHECKING block (migrated to sub.py) - # helper — migrated to sub.py - # entry — stays in __init__.py, references "MyConfig" in annotation - source = ( - "from typing import TYPE_CHECKING\n" - "if TYPE_CHECKING:\n" - " from .config import MyConfig\n" - "\n" - "def helper():\n" - " pass\n" - "\n" - "def entry(cfg: 'MyConfig') -> None:\n" - " helper()\n" - ) - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, []) - e_helper = _make_entity("helper", 5, 6) - e_entry = _make_entity("entry", 8, 9) - c = _classified(entities=[e_block, e_helper, e_entry]) - plan = _plan( - [GroupPlacement(group=["_block_1", "helper"], target_file="pkg/sub.py")] - ) - - result = generate_file_splits(c, plan, source, "pkg.py", subdir_name="pkg") - - assert not result.abort - init_src = result.original_source - # The TYPE_CHECKING import must be injected and bumped for the new depth. - assert "if TYPE_CHECKING:" in init_src - assert "from ..config import MyConfig" in init_src - - -# --------------------------------------------------------------------------- -# _strip_top_level_import_lines -# --------------------------------------------------------------------------- - - -def test_strip_top_level_import_lines_removes_imports(): - src = "import os\nfrom typing import List\n\n_CONST = 1\n" - result = _strip_top_level_import_lines(src) - assert "import os" not in result - assert "from typing import List" not in result - assert "_CONST = 1" in result - - -def test_strip_top_level_import_lines_no_imports(): - src = "_CONST = 1\n" - assert _strip_top_level_import_lines(src) == src - - -def test_strip_top_level_import_lines_syntax_error(): - src = "def (\n" - assert _strip_top_level_import_lines(src) == src - - -def test_strip_top_level_import_lines_strips_type_checking_block(): - # `if TYPE_CHECKING:` blocks must be stripped so that their imports are - # not emitted verbatim in sub-files (wrong path, wrong file). - src = "if TYPE_CHECKING:\n" " from .config import MyConfig\n" "\n" "_CONST = 1\n" - result = _strip_top_level_import_lines(src) - assert "TYPE_CHECKING" not in result - assert "MyConfig" not in result - assert "_CONST = 1" in result - - -# --------------------------------------------------------------------------- -# _extract_module_docstring -# --------------------------------------------------------------------------- - - -def test_extract_module_docstring_present(): - src = '"""My module."""\n\nimport os\n' - assert _extract_module_docstring(src) == '"""My module."""' - - -def test_extract_module_docstring_absent(): - src = "import os\n\ndef foo():\n pass\n" - assert _extract_module_docstring(src) is None - - -def test_extract_module_docstring_syntax_error(): - assert _extract_module_docstring("def (\n") is None - - -def test_extract_module_docstring_non_string_expr(): - # First statement is an expression but not a string constant. - src = "1 + 1\n\ndef foo():\n pass\n" - assert _extract_module_docstring(src) is None - - -# --------------------------------------------------------------------------- -# _strip_module_docstring -# --------------------------------------------------------------------------- - - -def test_strip_module_docstring_removes_docstring(): - src = '"""My module."""\n\n_CONST = 1\n' - result = _strip_module_docstring(src) - assert '"""My module."""' not in result - assert "_CONST = 1" in result - - -def test_strip_module_docstring_no_docstring(): - src = "_CONST = 1\n" - assert _strip_module_docstring(src) == src - - -def test_strip_module_docstring_syntax_error(): - src = "def (\n" - assert _strip_module_docstring(src) == src - - -# --------------------------------------------------------------------------- -# _source_is_only_docstring -# --------------------------------------------------------------------------- - - -def test_source_is_only_docstring_true(): - assert _source_is_only_docstring('"""Just a docstring."""\n') is True - - -def test_source_is_only_docstring_with_other_content(): - assert _source_is_only_docstring('"""Doc."""\n\nimport os\n') is False - - -def test_source_is_only_docstring_no_docstring(): - assert _source_is_only_docstring("import os\n") is False - - -def test_source_is_only_docstring_syntax_error(): - assert _source_is_only_docstring("def (\n") is False - - -# --------------------------------------------------------------------------- -# generate_file_splits — TOP_LEVEL entity import deduplication -# --------------------------------------------------------------------------- - - -def test_generate_top_level_entity_imports_not_duplicated(): - # When a TOP_LEVEL entity source contains regular imports (e.g. `import os`) - # those must NOT appear twice in the generated file: once from - # _find_needed_imports and again from the entity source itself. - source = "import os\n\n_CONST = os.sep\n\ndef foo():\n return os.getcwd()\n" - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["os", "_CONST"]) - e_foo = _make_entity("foo", 5, 6) - c = _classified(entities=[e_block, e_foo]) - plan = _plan( - [ - GroupPlacement(group=["_block_1", "foo"], target_file="utils.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - new_src = result.new_files["utils.py"] - assert new_src.count("import os") == 1 - - -# --------------------------------------------------------------------------- -# generate_file_splits — module docstring placement in subdir-split mode -# --------------------------------------------------------------------------- - - -def test_generate_subdir_module_docstring_goes_to_init(): - # In subdir-split mode the module docstring belongs in __init__.py, not - # in the split-off child module. Migrate the preamble entity (_block_1) - # along with foo so the docstring is removed from the original source, - # triggering the restore-to-__init__ logic. - source = textwrap.dedent( - """\ - \"\"\"Top-level module doc.\"\"\" - - import os - - def foo(): - return os.sep - - def bar(): - return foo() - """ - ) - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["os"]) - e_foo = _make_entity("foo", 5, 6) - e_bar = _make_entity("bar", 8, 9) - c = _classified(entities=[e_block, e_foo, e_bar]) - plan = _plan( - [GroupPlacement(group=["_block_1", "foo"], target_file="pkg/helpers.py")] - ) - - result = generate_file_splits(c, plan, source, "pkg.py", subdir_name="pkg") - - assert not result.abort - init_src = result.original_source - helpers_src = result.new_files["pkg/helpers.py"] - # Docstring belongs in __init__.py. - assert '"""Top-level module doc."""' in init_src - # Docstring must NOT appear in the child module. - assert '"""Top-level module doc."""' not in helpers_src - - -def test_generate_subdir_docstring_already_in_init_not_duplicated(): - # If the TOP_LEVEL entity stays in the original (not migrated), the - # docstring remains in the updated source and must not be prepended again. - source = textwrap.dedent( - """\ - \"\"\"Top-level module doc.\"\"\" - - _CONST = 1 - - def stayed(): - return _CONST - - def migrated(): - pass - """ - ) - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["_CONST"]) - e_stayed = _make_entity("stayed", 5, 6) - e_migrated = _make_entity("migrated", 8, 9) - c = _classified(entities=[e_block, e_stayed, e_migrated]) - plan = _plan([GroupPlacement(group=["migrated"], target_file="pkg/helpers.py")]) - - result = generate_file_splits(c, plan, source, "pkg.py", subdir_name="pkg") - - assert not result.abort - init_src = result.original_source - assert init_src.count('"""Top-level module doc."""') == 1 - - -def test_generate_subdir_module_docstring_goes_to_test_init(): - # For test-file subdir splits the module docstring goes into - # subdir/__init__.py, not into the re-export stub file. - source = textwrap.dedent( - """\ - \"\"\"Tests for the runner module.\"\"\" - - import os - - def test_foo(): - return os.sep - - def test_bar(): - return test_foo() - """ - ) - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["os"]) - e_foo = _make_entity("test_foo", 5, 6) - e_bar = _make_entity("test_bar", 8, 9) - c = _classified(entities=[e_block, e_foo, e_bar]) - plan = _plan( - [GroupPlacement(group=["_block_1", "test_foo"], target_file="svc/test_foo.py")] - ) - - result = generate_file_splits( - c, plan, source, "tests/test_svc.py", subdir_name="svc" - ) - - assert not result.abort - init_src = result.new_files["svc/__init__.py"] - child_src = result.new_files["svc/test_foo.py"] - updated_src = result.original_source - # Docstring belongs in __init__.py. - assert '"""Tests for the runner module."""' in init_src - # Docstring must NOT appear in the child test file or the stub file. - assert '"""Tests for the runner module."""' not in child_src - assert '"""Tests for the runner module."""' not in updated_src - - -def test_generate_subdir_test_docstring_only_remaining_clears_original(): - # Regression: when a test-file subdir split migrates all entities and the - # only thing left in the original is the module docstring (a TOP_LEVEL - # entity that is not migrated by _remove_entity_lines), the docstring must - # be routed to __init__.py and the original file must be cleared for - # deletion by the engine. - source = textwrap.dedent( - """\ - \"\"\"Tests for the widget module. - Covers edge cases. - \"\"\" - - def test_alpha(): - pass - - def test_beta(): - pass - """ - ) - # The module docstring is a TOP_LEVEL entity spanning lines 1-3. - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, []) - e_alpha = _make_entity("test_alpha", 5, 6) - e_beta = _make_entity("test_beta", 8, 9) - c = _classified(entities=[e_block, e_alpha, e_beta]) - # Only the test functions are migrated; the TOP_LEVEL entity stays. - plan = _plan( - [ - GroupPlacement(group=["test_alpha"], target_file="widget/test_alpha.py"), - GroupPlacement(group=["test_beta"], target_file="widget/test_beta.py"), - ] - ) - - result = generate_file_splits( - c, plan, source, "tests/test_widget.py", subdir_name="widget" - ) - - assert not result.abort - # Docstring must end up in __init__.py. - init_src = result.new_files["widget/__init__.py"] - assert '"""Tests for the widget module.' in init_src - # Original source must be empty so the engine deletes it. - assert result.original_source == "" - - -def test_generate_subdir_docstring_not_stripped_from_non_subdir_split(): - # Outside subdir-split mode, a TOP_LEVEL entity's docstring is preserved - # in the new file (only imports are stripped, not docstrings). - source = '"""Module doc."""\n\nimport os\n\ndef foo():\n return os.sep\n' - e_block = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 3, ["os"]) - e_foo = _make_entity("foo", 5, 6) - c = _classified(entities=[e_block, e_foo]) - plan = _plan([GroupPlacement(group=["_block_1", "foo"], target_file="utils.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - new_src = result.new_files["utils.py"] - assert '"""Module doc."""' in new_src - - -# --------------------------------------------------------------------------- -# _is_test_name -# --------------------------------------------------------------------------- - - -def test_is_test_name_test_class(): - assert _is_test_name("TestFoo") is True - - -def test_is_test_name_test_function(): - assert _is_test_name("test_bar") is True - - -def test_is_test_name_non_test(): - assert _is_test_name("helper") is False - assert _is_test_name("Foo") is False - assert _is_test_name("_test_private") is False - - -# --------------------------------------------------------------------------- -# _is_pytest_fixture -# --------------------------------------------------------------------------- - - -def test_is_pytest_fixture_syntax_error(): - assert _is_pytest_fixture("def (") is False - - -def test_is_pytest_fixture_empty_body(): - # Empty source → empty tree body → not a fixture. - assert _is_pytest_fixture("") is False - - -def test_is_pytest_fixture_class_node(): - # Class definition is not a FunctionDef → returns False. - assert _is_pytest_fixture("class Foo:\n pass\n") is False - - -def test_is_pytest_fixture_no_decorator(): - assert _is_pytest_fixture("def client():\n pass\n") is False - - -def test_is_pytest_fixture_bare_name(): - # @fixture (plain name, no call) - src = "@fixture\ndef client():\n pass\n" - assert _is_pytest_fixture(src) is True - - -def test_is_pytest_fixture_bare_name_called(): - # @fixture() (called with no args) - src = "@fixture()\ndef client():\n pass\n" - assert _is_pytest_fixture(src) is True - - -def test_is_pytest_fixture_attribute(): - # @pytest.fixture (attribute access, no call) - src = "@pytest.fixture\ndef client():\n pass\n" - assert _is_pytest_fixture(src) is True - - -def test_is_pytest_fixture_attribute_called(): - # @pytest.fixture(scope="session") - src = '@pytest.fixture(scope="session")\ndef client():\n pass\n' - assert _is_pytest_fixture(src) is True - - -def test_is_pytest_fixture_non_matching_decorator(): - # @other_decorator — Name but id != "fixture"; not an Attribute. - src = "@other_decorator\ndef client():\n pass\n" - assert _is_pytest_fixture(src) is False - - -# --------------------------------------------------------------------------- -# _split_cross_imports_by_test -# --------------------------------------------------------------------------- - - -def test_split_cross_imports_by_test_pure_non_test(): - non_test, test_named = _split_cross_imports_by_test(["from .foo import helper"]) - assert non_test == ["from .foo import helper"] - assert test_named == [] - - -def test_split_cross_imports_by_test_pure_test(): - non_test, test_named = _split_cross_imports_by_test( - ["from .foo import TestFoo, test_bar"] - ) - assert non_test == [] - assert test_named == ["from .foo import TestFoo, test_bar"] - - -def test_split_cross_imports_by_test_mixed(): - non_test, test_named = _split_cross_imports_by_test( - ["from .foo import TestFoo, helper, test_bar"] - ) - assert non_test == ["from .foo import helper"] - assert test_named == ["from .foo import TestFoo, test_bar"] - - -def test_split_cross_imports_by_test_plain_import_passthrough(): - # Plain "import x" lines (no "from") pass through to non_test unchanged. - non_test, test_named = _split_cross_imports_by_test(["import os"]) - assert non_test == ["import os"] - assert test_named == [] - - -# --------------------------------------------------------------------------- -# _inject_inline_imports -# --------------------------------------------------------------------------- - - -def test_inject_inline_imports_into_function(): - src = "def foo():\n return 1\n" - result = _inject_inline_imports(src, ["from .bar import Baz"]) - assert result == "def foo():\n from .bar import Baz\n return 1\n" - - -def test_inject_inline_imports_skips_docstring(): - src = 'def foo():\n """Doc."""\n return 1\n' - result = _inject_inline_imports(src, ["from .bar import Baz"]) - assert ( - result == 'def foo():\n """Doc."""\n from .bar import Baz\n return 1\n' - ) - - -def test_inject_inline_imports_into_class(): - src = "class Foo:\n x = 1\n" - result = _inject_inline_imports(src, ["from .bar import Baz"]) - assert result == "class Foo:\n from .bar import Baz\n x = 1\n" - - -def test_inject_inline_imports_toplevel_noop(): - # TOP_LEVEL entity (bare if-statement): no body scope, returns unchanged. - src = "if True:\n pass\n" - result = _inject_inline_imports(src, ["from .bar import Baz"]) - assert result == src - - -def test_inject_inline_imports_empty_list_noop(): - src = "def foo():\n pass\n" - assert _inject_inline_imports(src, []) == src - - -def test_inject_inline_imports_syntax_error_noop(): - src = "def (invalid" - assert _inject_inline_imports(src, ["from .x import Y"]) == src - - -def test_inject_inline_imports_empty_source_noop(): - # Empty source parses to empty tree.body — returns unchanged. - assert _inject_inline_imports("", ["from .x import Y"]) == "" - - -def test_inject_inline_imports_only_docstring_injects_after(): - # Function with only a docstring — inserts after docstring (at body[0] line) - # since len(body) == 1. - src = 'def foo():\n """Only doc."""\n' - result = _inject_inline_imports(src, ["from .bar import Baz"]) - assert result == 'def foo():\n from .bar import Baz\n """Only doc."""\n' - - -# --------------------------------------------------------------------------- -# _find_main_block_entity -# --------------------------------------------------------------------------- - - -def test_find_main_block_entity_present(): - from crispen.file_limiter.entity_parser import parse_entities - - source = textwrap.dedent( - """\ - def run(): - pass - - if __name__ == "__main__": - run() - """ - ) - entities = parse_entities(source) - esmap = {e.name: source.splitlines(keepends=True) for e in entities} - # Rebuild entity_source_map properly - lines = source.splitlines(keepends=True) - esmap = { - e.name: "".join(lines[e.start_line - 1 : e.end_line]).rstrip() for e in entities - } - result = _find_main_block_entity(entities, esmap) - assert result is not None - assert result.startswith("_block_") - - -def test_find_main_block_entity_absent(): - from crispen.file_limiter.entity_parser import parse_entities - - source = "def foo():\n pass\n" - entities = parse_entities(source) - lines = source.splitlines(keepends=True) - esmap = { - e.name: "".join(lines[e.start_line - 1 : e.end_line]).rstrip() for e in entities - } - assert _find_main_block_entity(entities, esmap) is None - - -def test_find_main_block_entity_syntax_error_skipped(): - from crispen.file_limiter.entity_parser import Entity, EntityKind - - # Entity whose source is invalid Python: should be skipped gracefully. - entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 1, []) - result = _find_main_block_entity([entity], {"_block_1": "def (invalid"}) - assert result is None - - -# --------------------------------------------------------------------------- -# _find_main_direct_callees -# --------------------------------------------------------------------------- - - -def test_find_main_direct_callees_basic(): - src = 'if __name__ == "__main__":\n run_tests()\n' - callees = _find_main_direct_callees(src, {"run_tests", "other"}) - assert callees == {"run_tests"} - - -def test_find_main_direct_callees_not_in_entity_names(): - src = 'if __name__ == "__main__":\n unknown()\n' - callees = _find_main_direct_callees(src, {"run_tests"}) - assert callees == set() - - -def test_find_main_direct_callees_syntax_error(): - assert _find_main_direct_callees("def (invalid", {"foo"}) == set() - - -def test_find_main_direct_callees_no_main_block(): - src = "run_tests()\n" - assert _find_main_direct_callees(src, {"run_tests"}) == set() - - -# --------------------------------------------------------------------------- -# _inject_inline_test_imports_original -# --------------------------------------------------------------------------- - - -def test_inject_inline_test_imports_original_basic(): - source = textwrap.dedent( - """\ - def runner(): - TestFoo() - """ - ) - migrated = {"TestFoo": "sub/test_foo.py"} - result = _inject_inline_test_imports_original( - source, migrated, abs_pkg="pkg.tests", original_basename="test_orig.py" - ) - assert "from pkg.tests.sub.test_foo import TestFoo" in result - # Import appears inside the function body, not before the def line. - lines = result.splitlines() - def_idx = next(i for i, l in enumerate(lines) if l.startswith("def runner")) - import_idx = next(i for i, l in enumerate(lines) if "import TestFoo" in l) - assert import_idx > def_idx - - -def test_inject_inline_test_imports_original_skips_docstring(): - source = textwrap.dedent( - """\ - def runner(): - \"\"\"Run tests.\"\"\" - TestFoo() - """ - ) - migrated = {"TestFoo": "sub/test_foo.py"} - result = _inject_inline_test_imports_original( - source, migrated, abs_pkg="tests", original_basename="test_orig.py" - ) - lines = result.splitlines() - doc_idx = next(i for i, l in enumerate(lines) if '"""Run tests."""' in l) - import_idx = next(i for i, l in enumerate(lines) if "import TestFoo" in l) - assert import_idx > doc_idx - - -def test_inject_inline_test_imports_original_no_reference(): - source = "def runner():\n pass\n" - migrated = {"TestFoo": "sub/test_foo.py"} - result = _inject_inline_test_imports_original( - source, migrated, abs_pkg="tests", original_basename="test_orig.py" - ) - assert result == source - - -def test_inject_inline_test_imports_original_empty_map(): - source = "def runner():\n TestFoo()\n" - result = _inject_inline_test_imports_original( - source, {}, abs_pkg="tests", original_basename="test_orig.py" - ) - assert result == source - - -def test_inject_inline_test_imports_original_syntax_error(): - result = _inject_inline_test_imports_original( - "def (invalid", - {"TestFoo": "sub/test_foo.py"}, - abs_pkg="tests", - original_basename="test_orig.py", - ) - assert result == "def (invalid" - - -def test_inject_inline_test_imports_original_relative_import(): - source = "def runner():\n TestFoo()\n" - migrated = {"TestFoo": "sub/test_foo.py"} - result = _inject_inline_test_imports_original( - source, migrated, abs_pkg=None, original_basename="test_orig.py" - ) - assert "from .sub.test_foo import TestFoo" in result - - -def test_inject_inline_test_imports_original_unreferenced_symbol_skipped(): - # Function references `helper` (not test-named) and `other_func`, neither - # of which is in migrated_test_symbols — the false branch of `if tfile:`. - source = "def runner():\n helper()\n other_func()\n" - migrated = {"TestFoo": "sub/test_foo.py"} - result = _inject_inline_test_imports_original( - source, migrated, abs_pkg="tests", original_basename="test_orig.py" - ) - assert result == source - - -# --------------------------------------------------------------------------- -# generate_file_splits — shebang handling -# --------------------------------------------------------------------------- - - -def test_generate_shebang_stripped_from_new_file(): - # Shebang on line 1 should NOT appear in generated new files. - source = "#!/usr/bin/env python3\n\ndef foo():\n pass\n\ndef bar():\n foo()\n" - e_foo = Entity(EntityKind.FUNCTION, "foo", 3, 4, ["foo"]) - e_bar = Entity(EntityKind.FUNCTION, "bar", 6, 7, ["bar"]) - c = _classified(entities=[e_foo, e_bar]) - plan = _plan([GroupPlacement(group=["bar"], target_file="helpers.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - assert "#!/usr/bin/env python3" not in result.new_files["helpers.py"] - - -def test_generate_shebang_preserved_in_original_when_entity_migrated(): - # When the entity owning line 1 (with shebang comment) is migrated, - # the shebang must be restored at the top of the original file. - source = "#!/usr/bin/env python3\ndef foo():\n pass\n\ndef bar():\n pass\n" - e_foo = Entity(EntityKind.FUNCTION, "foo", 1, 3, ["foo"]) - e_bar = Entity(EntityKind.FUNCTION, "bar", 5, 6, ["bar"]) - c = _classified(entities=[e_foo, e_bar]) - plan = _plan([GroupPlacement(group=["foo"], target_file="helpers.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - assert result.original_source.startswith("#!/usr/bin/env python3\n") - assert "#!/usr/bin/env python3" not in result.new_files["helpers.py"] - - -def test_generate_shebang_preserved_when_not_migrated(): - # When the shebang entity stays in the original, shebang remains at top. - source = "#!/usr/bin/env python3\ndef foo():\n pass\n\ndef bar():\n pass\n" - e_foo = Entity(EntityKind.FUNCTION, "foo", 1, 3, ["foo"]) - e_bar = Entity(EntityKind.FUNCTION, "bar", 5, 6, ["bar"]) - c = _classified(entities=[e_foo, e_bar]) - plan = _plan([GroupPlacement(group=["bar"], target_file="helpers.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - assert result.original_source.startswith("#!/usr/bin/env python3\n") - - -# --------------------------------------------------------------------------- -# generate_file_splits — __main__ sticky behaviour -# --------------------------------------------------------------------------- - - -def test_generate_main_block_stays_in_original(): - source = textwrap.dedent( - """\ - def run(): - pass - - def other(): - pass - - if __name__ == "__main__": - run() - """ - ) - e_run = Entity(EntityKind.FUNCTION, "run", 1, 2, ["run"]) - e_other = Entity(EntityKind.FUNCTION, "other", 4, 5, ["other"]) - e_main = Entity(EntityKind.TOP_LEVEL, "_block_7", 7, 8, []) - c = _classified(entities=[e_run, e_other, e_main]) - # Plan tries to migrate run + __main__ block and other. - plan = _plan( - [ - GroupPlacement(group=["run", "_block_7"], target_file="helpers.py"), - GroupPlacement(group=["other"], target_file="helpers.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - # __main__ block stays in original. - assert 'if __name__ == "__main__"' in result.original_source - assert 'if __name__ == "__main__"' not in result.new_files.get("helpers.py", "") - - -def test_generate_main_callee_stays_in_original(): - source = textwrap.dedent( - """\ - def run(): - pass - - def other(): - pass - - if __name__ == "__main__": - run() - """ - ) - e_run = Entity(EntityKind.FUNCTION, "run", 1, 2, ["run"]) - e_other = Entity(EntityKind.FUNCTION, "other", 4, 5, ["other"]) - e_main = Entity(EntityKind.TOP_LEVEL, "_block_7", 7, 8, []) - c = _classified(entities=[e_run, e_other, e_main]) - # Plan tries to migrate run (the direct callee of __main__). - plan = _plan( - [ - GroupPlacement(group=["run"], target_file="helpers.py"), - GroupPlacement(group=["other"], target_file="helpers.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - # run() is a direct __main__ callee — must stay in original. - assert "def run():" in result.original_source - # other() is not a callee — may be migrated. - assert "helpers.py" in result.new_files - - -# --------------------------------------------------------------------------- -# generate_file_splits — test-named symbol inline imports -# --------------------------------------------------------------------------- - - -def test_generate_test_named_cross_import_inlined(): - # TestHelper migrates to helpers.py; runner stays in original and - # references TestHelper — the import must be injected inside runner's body. - source = textwrap.dedent( - """\ - class TestHelper: - def test_x(self): - pass - - def runner(): - TestHelper() - """ - ) - e_cls = Entity(EntityKind.CLASS, "TestHelper", 1, 3, ["TestHelper"]) - e_run = Entity(EntityKind.FUNCTION, "runner", 5, 6, ["runner"]) - c = _classified(entities=[e_cls, e_run]) - plan = _plan([GroupPlacement(group=["TestHelper"], target_file="helpers.py")]) - - result = generate_file_splits(c, plan, source, "big.py") - - orig = result.original_source - # No module-level re-export of TestHelper. - lines = orig.splitlines() - top_level_import_lines = [ - ln for ln in lines if ln.startswith("from") and "TestHelper" in ln - ] - assert top_level_import_lines == [] - # Import appears inside runner's body. - assert " from .helpers import TestHelper" in orig - - -def test_generate_test_named_inline_not_applied_to_toplevel_entity(): - # A TOP_LEVEL entity referencing a test-named symbol falls back to - # module-level import since it has no body scope to inject into. - source = textwrap.dedent( - """\ - class TestHelper: - def test_x(self): - pass - - _inst = TestHelper() - """ - ) - e_cls = Entity(EntityKind.CLASS, "TestHelper", 1, 3, ["TestHelper"]) - e_block = Entity(EntityKind.TOP_LEVEL, "_block_5", 5, 5, ["_inst"]) - c = _classified(entities=[e_cls, e_block]) - plan = _plan( - [GroupPlacement(group=["TestHelper", "_block_5"], target_file="helpers.py")] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - # TestHelper and _block_5 were migrated together — no cross-file issue here. - # Test just ensures no crash and the file is produced. - assert "helpers.py" in result.new_files - - -def test_generate_test_named_inlined_in_function_in_new_file(): - # TestA goes to file_a.py; func_b (which calls TestA) goes to file_b.py. - # The cross-file import of TestA into file_b.py should be injected inline - # inside func_b's body rather than at the top of file_b.py. - source = textwrap.dedent( - """\ - class TestA: - def test_x(self): - pass - - def func_b(): - TestA() - """ - ) - e_a = Entity(EntityKind.CLASS, "TestA", 1, 3, ["TestA"]) - e_b = Entity(EntityKind.FUNCTION, "func_b", 5, 6, ["func_b"]) - c = _classified(entities=[e_a, e_b]) - plan = _plan( - [ - GroupPlacement(group=["TestA"], target_file="file_a.py"), - GroupPlacement(group=["func_b"], target_file="file_b.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - file_b = result.new_files["file_b.py"] - lines = file_b.splitlines() - # No module-level import of TestA. - assert not any(ln.startswith("from") and "TestA" in ln for ln in lines) - # Inline import inside func_b. - assert " from .file_a import TestA" in file_b - - -def test_generate_toplevel_entity_in_new_file_test_import_falls_back_to_module_level(): - # A TOP_LEVEL entity in a new file that references a test-named symbol - # from another new file: no function body to inject into, falls back to - # module-level import. Two TOP_LEVEL entities referencing the same - # test name exercise the dedup path on the second. - source = textwrap.dedent( - """\ - class TestA: - def test_x(self): - pass - - _inst1 = TestA() - - def _sep(): - pass - - _inst2 = TestA() - """ - ) - e_a = Entity(EntityKind.CLASS, "TestA", 1, 3, ["TestA"]) - e_b1 = Entity(EntityKind.TOP_LEVEL, "_block_5", 5, 5, ["_inst1"]) - e_sep = Entity(EntityKind.FUNCTION, "_sep", 7, 8, ["_sep"]) - e_b2 = Entity(EntityKind.TOP_LEVEL, "_block_10", 10, 10, ["_inst2"]) - c = _classified(entities=[e_a, e_b1, e_sep, e_b2]) - plan = _plan( - [ - GroupPlacement(group=["TestA"], target_file="file_a.py"), - GroupPlacement( - group=["_block_5", "_sep", "_block_10"], target_file="file_b.py" - ), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - file_b = result.new_files["file_b.py"] - # Module-level import is acceptable for TOP_LEVEL entities (no body scope). - assert "TestA" in file_b - # Dedup: the same import appears only once despite two TOP_LEVEL entities - # both referencing TestA. - assert file_b.count("import TestA") == 1 - - -def test_generate_cross_import_dedup_across_entities(): - # helper goes to helpers.py; foo and bar both go to workers.py and both - # reference helper — the cross-file import should appear once (dedup). - source = textwrap.dedent( - """\ - def helper(): - pass - - def foo(): - helper() - - def bar(): - helper() - """ - ) - e_h = Entity(EntityKind.FUNCTION, "helper", 1, 2, ["helper"]) - e_foo = Entity(EntityKind.FUNCTION, "foo", 4, 5, ["foo"]) - e_bar = Entity(EntityKind.FUNCTION, "bar", 7, 8, ["bar"]) - c = _classified(entities=[e_h, e_foo, e_bar]) - plan = _plan( - [ - GroupPlacement(group=["helper"], target_file="helpers.py"), - GroupPlacement(group=["foo", "bar"], target_file="workers.py"), - ] - ) - - result = generate_file_splits(c, plan, source, "big.py") - - workers = result.new_files["workers.py"] - # "from .helpers import helper" should appear exactly once. - assert workers.count("import helper") == 1 - - -# --------------------------------------------------------------------------- -# generate_file_splits — pytest conftest routing -# --------------------------------------------------------------------------- - - -def test_generate_pytest_conftest_disabled_no_conftest(): - # Default (pytest_conftest=False): fixture goes to assigned file, re-exported. - src = "@pytest.fixture\ndef client():\n pass\n" - entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["client"], target_file="fixtures.py")]) - - result = generate_file_splits(c, plan, src, "test_big.py") - - assert "fixtures.py" in result.new_files - assert "conftest.py" not in result.new_files - assert "client" in result.new_files["fixtures.py"] - - -def test_generate_pytest_conftest_subdir_routes_to_subdir_conftest(): - # With pytest_conftest=True AND subdir_name set, fixtures go to - # /conftest.py (not the parent conftest.py). This prevents - # multiple test files in the same directory from conflicting when they - # each have a fixture of the same name. - src = "@pytest.fixture\ndef client():\n pass\n" - entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["client"], target_file="expr/fixtures.py")]) - - result = generate_file_splits( - c, plan, src, "test_big.py", subdir_name="expr", pytest_conftest=True - ) - - assert "expr/conftest.py" in result.new_files - assert "def client():" in result.new_files["expr/conftest.py"] - assert "conftest.py" not in result.new_files # parent conftest untouched - assert "import client" not in result.original_source - - -def test_generate_pytest_conftest_subdir_fixture_referenced_in_remaining_goes_to_parent(): # noqa: E501 - # When a fixture is migrated from a subdir split but its name still appears - # in entities that remain in the original file, route it to the parent - # conftest.py (not the subdir conftest) so those tests can find it. - src = textwrap.dedent( - """\ - @pytest.fixture - def client(): - pass - - def test_big(client): - pass - """ - ) - e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - e_test = Entity(EntityKind.FUNCTION, "test_big", 5, 6, ["test_big"]) - # Only the fixture is migrated; the test stays in the original. - c = _classified(entities=[e_client, e_test]) - plan = _plan([GroupPlacement(group=["client"], target_file="expr/fixtures.py")]) - - result = generate_file_splits( - c, plan, src, "test_big.py", subdir_name="expr", pytest_conftest=True - ) - - # Fixture goes to parent conftest.py, not the subdir one. - assert "conftest.py" in result.new_files - assert "def client():" in result.new_files["conftest.py"] - assert "expr/conftest.py" not in result.new_files - # No import of client back into the original. - assert "import client" not in result.original_source - - -def test_generate_pytest_conftest_subdir_fixture_overrides_parent_conftest(tmp_path): - # When the fixture is referenced in remaining source AND the parent conftest - # already has a fixture with the same name (the module was overriding it), - # the fixture is *copied* (not moved) to the subdir conftest so migrated - # tests get the override; the entity also stays in the original file so - # the original test discovers it from its own module. - parent_conftest = tmp_path / "conftest.py" - parent_conftest.write_text( - "@pytest.fixture\ndef client():\n return 'base'\n", encoding="utf-8" - ) - src = textwrap.dedent( - """\ - @pytest.fixture - def client(): - return 'override' - - def test_big(client): - pass - """ - ) - e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - e_test = Entity(EntityKind.FUNCTION, "test_big", 5, 6, ["test_big"]) - c = _classified(entities=[e_client, e_test]) - plan = _plan([GroupPlacement(group=["client"], target_file="expr/fixtures.py")]) - original_path = str(tmp_path / "test_big.py") - - result = generate_file_splits( - c, plan, src, original_path, subdir_name="expr", pytest_conftest=True - ) - - # Fixture goes to subdir conftest for migrated tests. - assert "expr/conftest.py" in result.new_files - assert "def client():" in result.new_files["expr/conftest.py"] - assert "return 'override'" in result.new_files["expr/conftest.py"] - # Parent conftest is NOT modified (would drop the override via merge). - assert "conftest.py" not in result.new_files - # Fixture stays in original file so the original test finds the override. - assert "def client():" in result.original_source - assert "return 'override'" in result.original_source - # No re-export import injected. - assert "import client" not in result.original_source - - -def test_generate_pytest_conftest_subdir_parent_conftest_imports_only(tmp_path): - # When parent conftest exists but contains only imports (no function defs), - # no conflict is detected and the fixture routes to parent conftest normally. - parent_conftest = tmp_path / "conftest.py" - parent_conftest.write_text("import pytest\n", encoding="utf-8") - src = textwrap.dedent( - """\ - @pytest.fixture - def client(): - pass - - def test_big(client): - pass - """ - ) - e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - e_test = Entity(EntityKind.FUNCTION, "test_big", 5, 6, ["test_big"]) - c = _classified(entities=[e_client, e_test]) - plan = _plan([GroupPlacement(group=["client"], target_file="expr/fixtures.py")]) - original_path = str(tmp_path / "test_big.py") - - result = generate_file_splits( - c, plan, src, original_path, subdir_name="expr", pytest_conftest=True - ) - - # No conflict in parent conftest → fixture routes to parent conftest. - assert "conftest.py" in result.new_files - assert "def client():" in result.new_files["conftest.py"] - assert "expr/conftest.py" not in result.new_files - - -def test_generate_pytest_conftest_subdir_parent_conftest_syntax_error(tmp_path): - # When parent conftest has a syntax error, the OSError/SyntaxError handler - # silently ignores it (no names loaded), so no conflict is detected and the - # fixture routes to parent conftest normally. - parent_conftest = tmp_path / "conftest.py" - parent_conftest.write_text("def (broken syntax", encoding="utf-8") - src = textwrap.dedent( - """\ - @pytest.fixture - def client(): - pass - - def test_big(client): - pass - """ - ) - e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - e_test = Entity(EntityKind.FUNCTION, "test_big", 5, 6, ["test_big"]) - c = _classified(entities=[e_client, e_test]) - plan = _plan([GroupPlacement(group=["client"], target_file="expr/fixtures.py")]) - original_path = str(tmp_path / "test_big.py") - - result = generate_file_splits( - c, plan, src, original_path, subdir_name="expr", pytest_conftest=True - ) - - # Unreadable parent conftest → no conflict detected → parent conftest. - assert "conftest.py" in result.new_files - assert "def client():" in result.new_files["conftest.py"] - assert "expr/conftest.py" not in result.new_files - - -def test_generate_pytest_conftest_fixture_goes_to_conftest(): - # With pytest_conftest=True, fixture entity lands in conftest.py, not the - # LLM-assigned file, and no re-export import appears in the original. - src = "@pytest.fixture\ndef client():\n pass\n" - entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["client"], target_file="fixtures.py")]) - - result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) - - assert "conftest.py" in result.new_files - assert "def client():" in result.new_files["conftest.py"] - # No import of client back into the original (no F401/F811). - assert "import client" not in result.original_source - # The LLM-assigned file is dropped (all entities redirected). - assert "fixtures.py" not in result.new_files - - -def test_generate_pytest_conftest_mixed_group_splits(): - # Fixture goes to conftest.py; non-fixture stays in the assigned file. - src = textwrap.dedent( - """\ - @pytest.fixture - def client(): - pass - - def helper(): - pass - """ - ) - e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - e_helper = Entity(EntityKind.FUNCTION, "helper", 5, 6, ["helper"]) - c = _classified(entities=[e_client, e_helper]) - plan = _plan([GroupPlacement(group=["client", "helper"], target_file="support.py")]) - - result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) - - assert "conftest.py" in result.new_files - assert "def client():" in result.new_files["conftest.py"] - assert "support.py" in result.new_files - assert "def helper():" in result.new_files["support.py"] - assert "import client" not in result.original_source - - -def test_generate_pytest_conftest_no_fixtures_no_conftest(): - # pytest_conftest=True but no fixture entities → no conftest.py created. - src = "def helper():\n pass\n" - entity = Entity(EntityKind.FUNCTION, "helper", 1, 2, ["helper"]) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["helper"], target_file="support.py")]) - - result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) - - assert "conftest.py" not in result.new_files - assert "support.py" in result.new_files - - -def test_generate_pytest_conftest_prepends_existing(tmp_path): - # When conftest.py already exists on disk, its content is prepended. - existing = tmp_path / "conftest.py" - existing.write_text( - "# existing fixture\ndef prior():\n pass\n", encoding="utf-8" - ) - - src = "@pytest.fixture\ndef client():\n pass\n" - entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["client"], target_file="fixtures.py")]) - original_path = str(tmp_path / "test_big.py") - - result = generate_file_splits(c, plan, src, original_path, pytest_conftest=True) - - conftest_src = result.new_files["conftest.py"] - assert "# existing fixture" in conftest_src - assert "def prior():" in conftest_src - assert "def client():" in conftest_src - # Existing content should come first. - assert conftest_src.index("prior") < conftest_src.index("client") - - -def test_generate_pytest_conftest_name_conflict_keeps_in_target(tmp_path): - # When conftest.py already defines a function with the same name as the - # fixture being routed, the fixture stays in its LLM-assigned target file - # instead of being dropped by _merge_conftest_sources. This preserves the - # entity in the split output so that _verify_preservation passes. - existing = tmp_path / "conftest.py" - existing.write_text( - "@pytest.fixture\nasync def client():\n return 'old'\n", encoding="utf-8" - ) - - src = "@pytest.fixture\nasync def client():\n return 'new'\n" - entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["client"], target_file="fixtures.py")]) - original_path = str(tmp_path / "test_big.py") - - result = generate_file_splits(c, plan, src, original_path, pytest_conftest=True) - - # Fixture must appear in the output — in the LLM-assigned file, not conftest. - assert "fixtures.py" in result.new_files - assert "def client():" in result.new_files["fixtures.py"] - # conftest.py should not be created/modified (no new fixtures were routed there). - assert "conftest.py" not in result.new_files - - -def test_generate_pytest_conftest_name_conflict_mixed_group(tmp_path): - # When a placement group contains both a conftest-conflict fixture AND a - # regular function, the fixture is excluded from re-exports but the regular - # function is still re-exported. This covers the branch that rebuilds the - # GroupPlacement with only the non-conflict names. - existing = tmp_path / "conftest.py" - existing.write_text( - "@pytest.fixture\ndef client():\n return 'old'\n", encoding="utf-8" - ) - - src = ( - "@pytest.fixture\ndef client():\n return 'new'\n\n" - "def helper():\n pass\n" - ) - e_client = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - e_helper = Entity(EntityKind.FUNCTION, "helper", 5, 6, ["helper"]) - c = _classified(entities=[e_client, e_helper]) - plan = _plan([GroupPlacement(group=["client", "helper"], target_file="helpers.py")]) - original_path = str(tmp_path / "test_big.py") - - result = generate_file_splits(c, plan, src, original_path, pytest_conftest=True) - - # Both entities migrate to helpers.py. - assert "helpers.py" in result.new_files - assert "def client():" in result.new_files["helpers.py"] - assert "def helper():" in result.new_files["helpers.py"] - # helper is re-exported (public non-fixture); client is not (conftest conflict). - assert "helper" in result.original_source - assert "client" not in result.original_source - - -def test_generate_pytest_conftest_unreadable_conftest_falls_through(tmp_path): - # When conftest.py exists but has a syntax error, the OSError/SyntaxError - # handler silently ignores it and routes the fixture to conftest normally. - existing = tmp_path / "conftest.py" - existing.write_text("def (broken syntax", encoding="utf-8") - - src = "@pytest.fixture\ndef client():\n pass\n" - entity = Entity(EntityKind.FUNCTION, "client", 1, 3, ["client"]) - c = _classified(entities=[entity]) - plan = _plan([GroupPlacement(group=["client"], target_file="fixtures.py")]) - original_path = str(tmp_path / "test_big.py") - - result = generate_file_splits(c, plan, src, original_path, pytest_conftest=True) - - # With unreadable conftest, routing proceeds normally → fixture goes to conftest. - assert "conftest.py" in result.new_files - assert "def client():" in result.new_files["conftest.py"] - - -# --------------------------------------------------------------------------- -# _file_has_only_fixtures -# --------------------------------------------------------------------------- - - -def test_file_has_only_fixtures_syntax_error(): - assert _file_has_only_fixtures("def (") is False - - -def test_file_has_only_fixtures_empty(): - assert _file_has_only_fixtures("") is False - - -def test_file_has_only_fixtures_no_fixture(): - # Regular function only — not a fixture. - assert _file_has_only_fixtures("def helper():\n pass\n") is False - - -def test_file_has_only_fixtures_with_test_function(): - # Has both a fixture and a test function → not fixture-only. - src = textwrap.dedent( - """\ - @pytest.fixture - def client(): - pass - - def test_foo(client): - pass - """ - ) - assert _file_has_only_fixtures(src) is False - - -def test_file_has_only_fixtures_with_test_class(): - # Has both a fixture and a Test class → not fixture-only. - src = textwrap.dedent( - """\ - @pytest.fixture - def client(): - pass - - class TestFoo: - pass - """ - ) - assert _file_has_only_fixtures(src) is False - - -def test_file_has_only_fixtures_with_non_fixture_function(): - # Has a fixture and a plain helper function → not fixture-only. - src = textwrap.dedent( - """\ - @pytest.fixture - def client(): - pass - - def helper(): - pass - """ - ) - assert _file_has_only_fixtures(src) is False - - -def test_file_has_only_fixtures_with_class(): - # Has a fixture and a regular class → not fixture-only. - src = textwrap.dedent( - """\ - @pytest.fixture - def client(): - pass - - class Config: - pass - """ - ) - assert _file_has_only_fixtures(src) is False - - -def test_file_has_only_fixtures_single_fixture(): - # Just a fixture and an import → fixture-only. - src = textwrap.dedent( - """\ - import pytest - - @pytest.fixture - def client(): - pass - """ - ) - assert _file_has_only_fixtures(src) is True - - -def test_file_has_only_fixtures_multiple_fixtures(): - # Multiple fixtures with no tests → fixture-only. - src = textwrap.dedent( - """\ - import pytest - - @pytest.fixture - def client(): - pass - - @pytest.fixture - def db(): - pass - """ - ) - assert _file_has_only_fixtures(src) is True - - -def test_file_has_only_fixtures_async_fixture(): - # Async fixture → fixture-only. - src = textwrap.dedent( - """\ - import pytest - - @pytest.fixture - async def client(): - pass - """ - ) - assert _file_has_only_fixtures(src) is True - - -def test_file_has_only_fixtures_with_docstring(): - # Module docstring + fixture → fixture-only (docstring is allowed). - src = textwrap.dedent( - """\ - \"\"\"Module docstring.\"\"\" - - import pytest - - @pytest.fixture - def client(): - pass - """ - ) - assert _file_has_only_fixtures(src) is True - - -# --------------------------------------------------------------------------- -# generate_file_splits: fixture-only stranded test file cleanup -# --------------------------------------------------------------------------- - - -def test_generate_stays_fixture_emptied_when_tests_migrated(): - # When a fixture "stays" in the original test file but all tests migrate - # out, the original becomes fixture-only → route fixture to conftest.py - # and empty the original so the engine deletes it. - src = textwrap.dedent( - """\ - import pytest - - @pytest.fixture - def client(): - pass - - def test_foo(client): - pass - """ - ) - e_fixture = Entity(EntityKind.FUNCTION, "client", 3, 5, ["client"]) - e_test = Entity(EntityKind.FUNCTION, "test_foo", 7, 8, ["test_foo"]) - c = _classified(entities=[e_fixture, e_test]) - # Only test_foo is migrated; client "stays" in original. - plan = _plan( - [GroupPlacement(group=["test_foo"], target_file="expression/test_foo.py")] - ) - - result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) - - # Original should be empty (engine will delete it). - assert result.original_source == "" - # Fixture should be routed to conftest.py. - assert "conftest.py" in result.new_files - assert "def client():" in result.new_files["conftest.py"] - - -def test_generate_stays_fixture_merged_with_existing_conftest(tmp_path): - # If conftest.py already exists on disk (e.g. same fixture already there), - # the merge deduplicates so the fixture is not repeated. - existing = tmp_path / "conftest.py" - existing.write_text( - "import pytest\n\n\n@pytest.fixture\ndef client():\n pass\n", - encoding="utf-8", - ) - - src = textwrap.dedent( - """\ - import pytest - - @pytest.fixture - def client(): - pass - - def test_foo(client): - pass - """ - ) - e_fixture = Entity(EntityKind.FUNCTION, "client", 3, 5, ["client"]) - e_test = Entity(EntityKind.FUNCTION, "test_foo", 7, 8, ["test_foo"]) - c = _classified(entities=[e_fixture, e_test]) - plan = _plan( - [GroupPlacement(group=["test_foo"], target_file="expression/test_foo.py")] - ) - original_path = str(tmp_path / "test_expression.py") - - result = generate_file_splits(c, plan, src, original_path, pytest_conftest=True) - - assert result.original_source == "" - # Fixture should appear exactly once in conftest.py (deduplicated). - assert result.new_files["conftest.py"].count("def client():") == 1 - - -def test_generate_stays_fixture_not_emptied_when_tests_remain(): - # If test functions still remain in the original, the fixture-only cleanup - # does NOT trigger — the file should keep both fixture and test. - src = textwrap.dedent( - """\ - import pytest - - @pytest.fixture - def client(): - pass - - def test_foo(client): - pass - - def test_bar(client): - pass - """ - ) - e_fixture = Entity(EntityKind.FUNCTION, "client", 3, 5, ["client"]) - e_test_foo = Entity(EntityKind.FUNCTION, "test_foo", 7, 8, ["test_foo"]) - e_test_bar = Entity(EntityKind.FUNCTION, "test_bar", 10, 11, ["test_bar"]) - c = _classified(entities=[e_fixture, e_test_foo, e_test_bar]) - # Only test_foo migrates; test_bar stays → original still has a test. - plan = _plan( - [GroupPlacement(group=["test_foo"], target_file="expression/test_foo.py")] - ) - - result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) - - # Original should still contain the remaining test and fixture. - assert "def test_bar" in result.original_source - assert result.original_source != "" - - -def test_generate_stays_fixture_not_emptied_when_conftest_disabled(): - # When pytest_conftest=False, stranded fixtures are left in the original. - src = textwrap.dedent( - """\ - import pytest - - @pytest.fixture - def client(): - pass - - def test_foo(client): - pass - """ - ) - e_fixture = Entity(EntityKind.FUNCTION, "client", 3, 5, ["client"]) - e_test = Entity(EntityKind.FUNCTION, "test_foo", 7, 8, ["test_foo"]) - c = _classified(entities=[e_fixture, e_test]) - plan = _plan( - [GroupPlacement(group=["test_foo"], target_file="expression/test_foo.py")] - ) - - result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=False) - - # Original keeps the fixture (conftest routing disabled). - assert "def client():" in result.original_source - assert "conftest.py" not in result.new_files - - -def test_generate_stays_fixture_merged_with_already_written_conftest(): - # If conftest.py was already written by this same split run (e.g. another - # entity was already routed there), merge into it rather than reading disk. - src = textwrap.dedent( - """\ - import pytest - - @pytest.fixture - def client(): - pass - - @pytest.fixture - def db(): - pass - - def test_foo(client): - pass - """ - ) - e_client = Entity(EntityKind.FUNCTION, "client", 3, 5, ["client"]) - e_db = Entity(EntityKind.FUNCTION, "db", 7, 9, ["db"]) - e_test = Entity(EntityKind.FUNCTION, "test_foo", 11, 12, ["test_foo"]) - c = _classified(entities=[e_client, e_db, e_test]) - # db migrates (and goes to conftest.py via pytest routing); test_foo migrates; - # client stays but is then stranded. - plan = _plan( - [ - GroupPlacement(group=["db"], target_file="fixtures.py"), - GroupPlacement(group=["test_foo"], target_file="expression/test_foo.py"), - ] - ) - - result = generate_file_splits(c, plan, src, "test_big.py", pytest_conftest=True) - - assert result.original_source == "" - conftest_src = result.new_files["conftest.py"] - # Both migrated db and stranded client fixtures should be in conftest. - assert "def db():" in conftest_src - assert "def client():" in conftest_src - - -# --------------------------------------------------------------------------- -# _merge_conftest_sources -# --------------------------------------------------------------------------- - - -def test_merge_conftest_sources_deduplicates_imports(): - # Imports that already exist are not repeated. - existing = "import pytest\n\n\n@pytest.fixture\ndef prior():\n pass\n" - new = "import pytest\n\n\n@pytest.fixture\ndef client():\n pass\n" - result = _merge_conftest_sources(existing, new) - assert result.count("import pytest") == 1 - - -def test_merge_conftest_sources_deduplicates_functions(): - # A function already in existing is not appended again. - existing = "@pytest.fixture\ndef client():\n return 1\n" - new = "@pytest.fixture\ndef client():\n return 2\n" - result = _merge_conftest_sources(existing, new) - assert result.count("def client():") == 1 - assert "return 1" in result - assert "return 2" not in result - - -def test_merge_conftest_sources_appends_new_fixture(): - # A new fixture not in existing is appended. - existing = "@pytest.fixture\ndef prior():\n pass\n" - new = "@pytest.fixture\ndef client():\n pass\n" - result = _merge_conftest_sources(existing, new) - assert "def prior():" in result - assert "def client():" in result - assert result.index("prior") < result.index("client") - - -def test_merge_conftest_sources_no_changes_returns_existing(): - # When nothing new to add, return existing unchanged. - existing = "import pytest\n\n\n@pytest.fixture\ndef client():\n pass\n" - new = "import pytest\n\n\n@pytest.fixture\ndef client():\n pass\n" - result = _merge_conftest_sources(existing, new) - assert result == existing - - -def test_merge_conftest_sources_inserts_new_imports_before_functions(): - # New imports are inserted after existing imports but before functions — no E402. - existing = "import pytest\n\n\n@pytest.fixture\ndef prior():\n pass\n" - new = "import asyncio\n\n\n@pytest.fixture\ndef client():\n pass\n" - result = _merge_conftest_sources(existing, new) - assert "import asyncio" in result - assert "def client():" in result - # Imports must come before the first function definition. - assert result.index("import asyncio") < result.index("def prior():") - - -def test_merge_conftest_sources_syntax_error_fallback(): - # Falls back to simple concatenation when existing cannot be parsed. - existing = "def (broken" - new = "import pytest\n" - result = _merge_conftest_sources(existing, new) - assert "def (broken" in result - assert "import pytest" in result - - -def test_merge_conftest_sources_preserves_comments(): - # Comments in the existing conftest are preserved. - existing = "# shared fixtures\nimport pytest\n\n\ndef prior():\n pass\n" - new = "@pytest.fixture\ndef client():\n pass\n" - result = _merge_conftest_sources(existing, new) - assert "# shared fixtures" in result - assert "def client():" in result - - -def test_merge_conftest_sources_from_import_dedup(): - # from-style imports are also deduplicated via the _import_key F: path. - existing = "from conftest import setup\n\n\ndef prior():\n pass\n" - new = "from conftest import setup\n\n\ndef client():\n pass\n" - result = _merge_conftest_sources(existing, new) - assert result.count("from conftest import setup") == 1 - assert "def client():" in result - - -def test_merge_conftest_sources_only_new_imports_no_defs(): - # When only new imports are added but no new functions, ends with newline. - existing = "import pytest\n\n\ndef prior():\n pass\n" - new = "import asyncio\n" - result = _merge_conftest_sources(existing, new) - assert "import asyncio" in result - assert result.endswith("\n") - # No duplicate function definition appended. - assert result.count("def prior():") == 1 - - -def test_merge_conftest_sources_non_import_non_def_in_new(): - # Bare statements (assignments, expressions) in new_content are silently ignored. - existing = "def prior():\n pass\n" - new = "X = 42\n" - result = _merge_conftest_sources(existing, new) - # Nothing to import or define → returns existing unchanged. - assert result == existing - - -# --------------------------------------------------------------------------- -# _strip_orphaned_section_headers -# --------------------------------------------------------------------------- - - -def test_strip_orphaned_3line_header_at_eof(): - """3-line block with no code after it is removed.""" - div = "# ---\n" - source = "def foo():\n pass\n\n\n" + div + "# Old Section\n" + div - result = _strip_orphaned_section_headers(source) - assert "# Old Section" not in result - assert "def foo():" in result - - -def test_strip_orphaned_single_line_header_at_eof(): - """Single-line header with no code after it is removed.""" - source = "def foo():\n pass\n\n# --- Removed ---\n" - result = _strip_orphaned_section_headers(source) - assert "# --- Removed ---" not in result - assert "def foo():" in result - - -def test_strip_not_orphaned_3line_header(): - """3-line block followed by substantive code is kept.""" - div = "# ---\n" - source = div + "# Helpers\n" + div + "\n\ndef helper():\n pass\n" - result = _strip_orphaned_section_headers(source) - assert "# Helpers" in result - assert "def helper():" in result - - -def test_strip_not_orphaned_single_line_header(): - """Single-line header followed by substantive code is kept.""" - source = "# --- Tools ---\n\ndef tool():\n pass\n" - result = _strip_orphaned_section_headers(source) - assert "# --- Tools ---" in result - - -def test_strip_orphaned_header_followed_only_by_another_header(): - """Header followed only by another header (and then nothing) — both orphaned.""" - source = "def foo():\n" " pass\n" "\n" "# --- First ---\n" "# --- Second ---\n" - result = _strip_orphaned_section_headers(source) - assert "# --- First ---" not in result - assert "# --- Second ---" not in result - assert "def foo():" in result - - -def test_strip_partial_orphan(): - """Only the header with no code after it is removed; the other stays.""" - source = ( - "# --- Active ---\n" "\n" "def foo():\n" " pass\n" "\n" "# --- Empty ---\n" - ) - result = _strip_orphaned_section_headers(source) - assert "# --- Active ---" in result - assert "# --- Empty ---" not in result - - -def test_strip_no_headers_returns_unchanged(): - """Source with no section headers is returned unchanged.""" - source = "def foo():\n pass\n" - assert _strip_orphaned_section_headers(source) == source - - -def test_strip_all_headers_have_content(): - """When every header has content below it, source is returned unchanged.""" - source = ( - "# --- A ---\n" - "\n" - "def a():\n" - " pass\n" - "\n" - "# --- B ---\n" - "\n" - "def b():\n" - " pass\n" - ) - result = _strip_orphaned_section_headers(source) - assert "# --- A ---" in result - assert "# --- B ---" in result - - -def test_strip_equals_single_line_header_orphaned(): - """=== style orphaned header is also removed.""" - source = "def foo():\n pass\n\n# === OLD SECTION ===\n" - result = _strip_orphaned_section_headers(source) - assert "# === OLD SECTION ===" not in result - - -# --------------------------------------------------------------------------- -# _normalize_blank_lines -# --------------------------------------------------------------------------- - - -def test_normalize_blank_lines_strips_leading_blanks(): - """Leading blank lines are removed (prevents E303 at top of file).""" - source = "\n\n\ndef foo():\n pass\n" - result = _normalize_blank_lines(source) - assert result.startswith("def foo():") - - -def test_normalize_blank_lines_collapses_excess_top_level(): - """4+ consecutive newlines between top-level defs collapse to 3.""" - source = "def foo():\n pass\n\n\n\n\ndef bar():\n pass\n" - result = _normalize_blank_lines(source) - assert "\n\n\n\n" not in result - assert "def foo():" in result - assert "def bar():" in result - - -def test_normalize_blank_lines_collapses_body_blanks(): - """2+ blank lines inside an indented body collapse to 1 (prevents E303 in body).""" - source = "def foo():\n x = 1\n\n\n y = 2\n" - result = _normalize_blank_lines(source) - assert "\n\n\n y" not in result - assert "\n\n y" in result - - -def test_normalize_blank_lines_empty_source(): - """Whitespace-only source returns empty string.""" - assert _normalize_blank_lines("\n\n\n") == "" - - -def test_normalize_blank_lines_trailing_newline(): - """Result always ends with exactly one newline.""" - source = "x = 1\n\n\n" - result = _normalize_blank_lines(source) - assert result.endswith("\n") - assert not result.endswith("\n\n") - - -def test_normalize_blank_lines_preserves_multiline_string_body_blanks(): - """Blank lines inside a multi-line string literal are never collapsed. - - Regression: _EXCESS_BLANK_BODY_RE matched \\n{3,}(?=[ \\t]) inside - triple-quoted strings, collapsing 2 blank lines before an indented line - to 1 (e.g. stored source-code fixtures in tests). - """ - # The triple-quoted string contains 2 blank lines before an indented `def`. - # That produces the sequence \\n\\n\\n def inside the raw source, - # which _EXCESS_BLANK_BODY_RE would collapse to \\n\\n def. - source = textwrap.dedent( - """\ - import textwrap - def foo(): - src = textwrap.dedent( - \"\"\"\\ - @dataclass - class _SplitTask: - pass - - - def _find_free_vars(): - x = 1 - \"\"\" - ) - """ - ) - result = _normalize_blank_lines(source) - # Two blank lines before the indented `def` inside the string must survive. - # After outer textwrap.dedent the string content has 8-space indentation. - assert "\n\n\n def _find_free_vars" in result - - -def test_normalize_blank_lines_still_collapses_excess_outside_strings(): - """Blank-line collapsing still fires for code outside string literals.""" - source = "def foo():\n x = 1\n\n\n y = 2\n" - result = _normalize_blank_lines(source) - assert "\n\n\n y" not in result - assert "\n\n y" in result - - -# --------------------------------------------------------------------------- -# _multiline_string_ranges # --------------------------------------------------------------------------- - - -def test_multiline_string_ranges_triple_quoted(): - """Detects a triple-quoted string spanning multiple lines.""" - source = 'x = """\nhello\n"""\n' - ranges = _multiline_string_ranges(source) - assert len(ranges) == 1 - start, end = ranges[0] - assert source[start:end] == '"""\nhello\n"""' - - -def test_multiline_string_ranges_single_line_string_ignored(): - """Single-line strings (no literal newline) are not returned.""" - source = 'x = "hello\\n"\n' - ranges = _multiline_string_ranges(source) - assert ranges == [] - - -def test_multiline_string_ranges_no_strings(): - """Returns empty list when there are no string literals.""" - source = "x = 1 + 2\n" - ranges = _multiline_string_ranges(source) - assert ranges == [] - - -def test_multiline_string_ranges_invalid_source(): - """Falls back to empty list on tokenization error.""" - # Unterminated string triggers TokenError. - source = 'x = """\nhello\n' - ranges = _multiline_string_ranges(source) - assert ranges == [] - - -# --------------------------------------------------------------------------- -# _sub_skip_strings -# --------------------------------------------------------------------------- - - -def test_sub_skip_strings_does_not_touch_string_content(): - """Pattern match inside a multi-line string is not substituted.""" - import re - - pattern = re.compile(r"\n{3,}(?=[ \t])") - source = 'def f():\n s = """\n a\n\n\n b\n """\n' - result = _sub_skip_strings(pattern, "\n\n", source) - # The sequence inside the string must survive unchanged. - assert "\n\n\n b" in result - - -def test_sub_skip_strings_applies_outside_strings(): - """Pattern match outside string literals is substituted normally.""" - import re - - pattern = re.compile(r"\n{3,}(?=[ \t])") - source = "def f():\n x = 1\n\n\n y = 2\n" - result = _sub_skip_strings(pattern, "\n\n", source) - assert "\n\n\n y" not in result - assert "\n\n y" in result - - -def test_sub_skip_strings_no_strings_falls_through(): - """When there are no multi-line strings the plain .sub() path is taken.""" - import re - - pattern = re.compile(r"x") - source = "x = 1\n" - result = _sub_skip_strings(pattern, "y", source) - assert result == "y = 1\n" - - -# --------------------------------------------------------------------------- -# _strip_orphaned_indented_comments +# _bump_relative_imports # --------------------------------------------------------------------------- -def test_strip_orphaned_indented_comments_removes_orphan(): - """Indented comment at module level (outside any AST node) is removed.""" - source = "\n\n # This comment was left behind after function removal\n" - result = _strip_orphaned_indented_comments(source) - assert "# This comment was left behind" not in result - - -def test_strip_orphaned_indented_comments_keeps_inside_function(): - """Indented comment inside a function body is preserved.""" - source = "def foo():\n # normal comment\n pass\n" - result = _strip_orphaned_indented_comments(source) - assert "# normal comment" in result - - -def test_strip_orphaned_indented_comments_keeps_module_level_comment(): - """Non-indented module-level comment is preserved.""" - source = "# module comment\ndef foo():\n pass\n" - result = _strip_orphaned_indented_comments(source) - assert "# module comment" in result - - -def test_strip_orphaned_indented_comments_syntax_error(): - """SyntaxError in source returns source unchanged.""" - source = " # orphaned\ndef f(: pass\n" - result = _strip_orphaned_indented_comments(source) - assert result == source +# generate_file_splits — subdir_name bumps relative imports diff --git a/tests/test_duplicate_extractor.py b/tests/test_duplicate_extractor.py index d625161..d03bfe9 100644 --- a/tests/test_duplicate_extractor.py +++ b/tests/test_duplicate_extractor.py @@ -1,6374 +1,25 @@ -"""Tests for duplicate_extractor: 100% branch coverage.""" - -import textwrap -from unittest.mock import MagicMock, patch - -import libcst as cst -import pytest -from libcst.metadata import MetadataWrapper - -from crispen.errors import CrispenAPIError -from crispen.refactors.duplicate_extractor import ( - _ApiTimeout, - _build_helper_insertion, - _has_funcdef, - _collect_attribute_names, - _collect_called_attr_names, - _collect_ast_store_names, - _replace_unused_in_target, - _scope_end_line, - _extract_defined_names, - _FunctionCollector, - _FunctionInfo, - _SeqInfo, - _SequenceCollector, - _apply_edits, - _build_function_body_fps, - _collect_called_names, - _filter_maximal_groups, - _find_duplicate_groups, - _has_internal_overlap, - _find_insertion_point, - _skip_class_docstring, - _generate_no_arg_call, - _has_call_to, - _has_def, - _find_escaping_vars, - _has_mutable_literal_is_check, - _has_param_overwritten_before_read, - _llm_generate_call, - _llm_veto_func_match, - _names_assigned_in, - _node_weight, - _normalize_replacement_indentation, - _normalize_source, - _overlaps_diff, - _missing_free_vars, - _is_pure_literal, - _names_in_edit_texts, - _pyflakes_new_undefined_names, - _pyflakes_strip_unused_simple_assigns, - _run_with_timeout, - _sequence_weight, - _seq_ends_with_return, - _seq_source_contains_yield, - _replacement_contains_return, - _replacement_steals_post_block_line, - _lift_and_dedup_imports, - _helper_imports_local_name, - _strip_helper_docstring, - _strip_unused_call_assignments, - _verify_extraction, - _would_create_proxy_wrappers, - DuplicateExtractor, -) - -# --------------------------------------------------------------------------- -# _node_weight -# --------------------------------------------------------------------------- - - -def _parse_stmt(src: str) -> cst.BaseStatement: - return cst.parse_module(src).body[0] - - -def test_node_weight_simple_one(): - assert _node_weight(_parse_stmt("a = 1\n")) == 1 - - -def test_node_weight_simple_two_semicolons(): - # Two small stmts on one line separated by semicolon - stmt = _parse_stmt("a = 1; b = 2\n") - assert _node_weight(stmt) == 2 - - -def test_node_weight_indented_block(): - block = _parse_stmt("if True:\n a = 1\n b = 2\n").body - assert _node_weight(block) == 2 - - -def test_node_weight_else(): - if_node = _parse_stmt("if True:\n a = 1\nelse:\n b = 2\n") - else_node = if_node.orelse - assert _node_weight(else_node) == 1 - - -def test_node_weight_finally(): - try_node = _parse_stmt("try:\n a = 1\nfinally:\n b = 2\n") - finally_node = try_node.finalbody - assert _node_weight(finally_node) == 1 - - -def test_node_weight_functiondef(): - stmt = _parse_stmt("def foo():\n pass\n") - assert _node_weight(stmt) == 1 - - -def test_node_weight_classdef(): - stmt = _parse_stmt("class Foo:\n pass\n") - assert _node_weight(stmt) == 1 - - -def test_node_weight_non_statement(): - name_node = cst.Name("foo") - assert _node_weight(name_node) == 0 - - -def test_node_weight_if_no_else(): - # weight = 1 (if) + 2 (body) - stmt = _parse_stmt("if x:\n a = 1\n b = 2\n") - assert _node_weight(stmt) == 3 - - -def test_node_weight_if_with_else(): - # weight = 1 (if) + 1 (body) + 1 (else body) - stmt = _parse_stmt("if x:\n a = 1\nelse:\n b = 2\n") - assert _node_weight(stmt) == 3 - - -def test_node_weight_for(): - # weight = 1 (for) + 1 (body) - stmt = _parse_stmt("for i in x:\n a = 1\n") - assert _node_weight(stmt) == 2 - - -def test_node_weight_for_with_else(): - # weight = 1 (for) + 1 (body) + 1 (else body) - stmt = _parse_stmt("for i in x:\n a = 1\nelse:\n b = 2\n") - assert _node_weight(stmt) == 3 - - -def test_node_weight_while(): - stmt = _parse_stmt("while x:\n a = 1\n") - assert _node_weight(stmt) == 2 - - -def test_node_weight_try_with_handler(): - # weight = 1 (try) + 1 (body) + 1 (handler body) - stmt = _parse_stmt("try:\n a = 1\nexcept:\n b = 2\n") - assert _node_weight(stmt) == 3 - - -def test_node_weight_try_with_handler_and_finally(): - # weight = 1 + 1 + 1 + 1 (finally body) - stmt = _parse_stmt("try:\n a = 1\nexcept:\n b = 2\nfinally:\n c = 3\n") - assert _node_weight(stmt) == 4 - - -def test_node_weight_try_with_orelse(): - # weight = 1 + 1 (body) + 1 (handler) + 1 (else body) - stmt = _parse_stmt("try:\n a = 1\nexcept:\n b = 2\nelse:\n c = 3\n") - assert _node_weight(stmt) == 4 - - -def test_node_weight_with(): - stmt = _parse_stmt("with open('f') as fh:\n a = 1\n") - assert _node_weight(stmt) == 2 - - -def test_sequence_weight_empty(): - assert _sequence_weight([]) == 0 - - -def test_sequence_weight_mixed(): - stmts = [ - _parse_stmt("a = 1\n"), - _parse_stmt("if x:\n b = 2\n"), - ] - assert _sequence_weight(stmts) == 1 + 2 - - -# --------------------------------------------------------------------------- -# _has_def -# --------------------------------------------------------------------------- - - -def test_has_def_no_def(): - stmts = [_parse_stmt("a = 1\n"), _parse_stmt("b = 2\n")] - assert _has_def(stmts) is False - - -def test_has_def_with_functiondef(): - stmts = [_parse_stmt("a = 1\n"), _parse_stmt("def foo():\n pass\n")] - assert _has_def(stmts) is True - - -def test_has_def_with_classdef(): - stmts = [_parse_stmt("class Foo:\n pass\n")] - assert _has_def(stmts) is True - - -# --------------------------------------------------------------------------- -# _normalize_source -# --------------------------------------------------------------------------- - - -def test_normalize_source_normalizes_vars(): - src = "result = compute(data)\noutput = transform(result)\n" - norm = _normalize_source(src) - # All names (both assigned and free) are replaced with positional placeholders - assert "result" not in norm - assert "output" not in norm - assert "compute" not in norm - assert "data" not in norm - - -def test_normalize_source_same_fingerprint(): - src_a = "x = compute(data)\ny = transform(x)\n" - src_b = "val = compute(data)\nres = transform(val)\n" - assert _normalize_source(src_a) == _normalize_source(src_b) - - -def test_normalize_source_different_ops(): - # Structurally different code (different number of statements) should differ - src_a = "x = a + b\n" - src_b = "x = a + b\ny = x * 2\n" - assert _normalize_source(src_a) != _normalize_source(src_b) - - -def test_normalize_source_invalid_syntax(): - src = "def f(: pass" - # Falls back to original source - assert _normalize_source(src) == src - - -def test_normalize_source_load_context_replaced(): - # Var assigned then used: both should be normalized the same - src_a = "x = 1\ny = x + 1\n" - src_b = "a = 1\nb = a + 1\n" - assert _normalize_source(src_a) == _normalize_source(src_b) - - -def test_normalize_source_load_not_in_map(): - # Free variables (Load context, never stored) are also normalized, - # so two blocks with different free variable names get the same fingerprint. - src_a = "y = a + 1\n" - src_b = "z = b + 1\n" - assert _normalize_source(src_a) == _normalize_source(src_b) - - -def test_normalize_source_repeated_store(): - # Same name assigned twice: _placeholder called with cached key (False branch) - src = "x = 1\nx = 2\n" - norm = _normalize_source(src) - # Both assignments normalize to the same placeholder - assert norm.count("_v0") == 2 - - -def test_normalize_source_del_context(): - # Del context falls through to return node unchanged - src = "del x\n" - norm = _normalize_source(src) - assert "x" in norm - - -def test_normalize_source_free_variables_match(): - # Blocks differing only in free variable names should get the same fingerprint. - # This is the core case: `p = a * 2; if p > 100: p += 1` vs the same with q/b. - src_a = "p = a * 2\nif p > 100:\n p += 1\n" - src_b = "q = b * 2\nif q > 100:\n q += 1\n" - assert _normalize_source(src_a) == _normalize_source(src_b) - - -def test_normalize_source_indented_blocks_match(): - # Source collected from inside a function is indented; dedent must happen - # before ast.parse so that structurally identical blocks still match. - src_a = " p = a * 2\n if p > 100:\n p += 1\n" - src_b = " q = b * 2\n if q > 100:\n q += 1\n" - assert _normalize_source(src_a) == _normalize_source(src_b) - - -# --------------------------------------------------------------------------- -# _overlaps_diff -# --------------------------------------------------------------------------- - - -def _make_seq(start: int, end: int) -> _SeqInfo: - return _SeqInfo( - stmts=[], - start_line=start, - end_line=end, - scope="", - source="", - fingerprint="", - ) - - -def test_overlaps_diff_yes(): - seq = _make_seq(5, 10) - assert _overlaps_diff(seq, [(8, 12)]) is True - - -def test_overlaps_diff_no(): - seq = _make_seq(5, 10) - assert _overlaps_diff(seq, [(11, 20)]) is False - - -def test_overlaps_diff_exact_boundary(): - seq = _make_seq(5, 10) - assert _overlaps_diff(seq, [(10, 15)]) is True - - -# --------------------------------------------------------------------------- -# _find_duplicate_groups -# --------------------------------------------------------------------------- - - -def test_find_duplicate_groups_empty(): - assert _find_duplicate_groups([], [(1, 5)]) == [] - - -def test_find_duplicate_groups_singleton(): - seq = _make_seq(1, 3) - seq.fingerprint = "fp1" - seqs = [seq] - # Only one seq with this fingerprint — not a duplicate - assert _find_duplicate_groups(seqs, [(1, 3)]) == [] - - -def test_find_duplicate_groups_no_diff_overlap(): - s1 = _SeqInfo([], 1, 3, "", "", "fp1") - s2 = _SeqInfo([], 10, 12, "", "", "fp1") - # Neither overlaps diff range (20, 30) - assert _find_duplicate_groups([s1, s2], [(20, 30)]) == [] - - -def test_find_duplicate_groups_valid(): - s1 = _SeqInfo([], 1, 3, "", "", "fp1") - s2 = _SeqInfo([], 10, 12, "", "", "fp1") - groups = _find_duplicate_groups([s1, s2], [(1, 3)]) - assert len(groups) == 1 - assert set(id(s) for s in groups[0]) == {id(s1), id(s2)} - - -# --------------------------------------------------------------------------- -# _has_internal_overlap -# --------------------------------------------------------------------------- - - -def test_has_internal_overlap_no_overlap(): - s1 = _SeqInfo([], 1, 3, "", "", "fp1") - s2 = _SeqInfo([], 10, 12, "", "", "fp1") - assert not _has_internal_overlap([s1, s2]) - - -def test_has_internal_overlap_adjacent_no_overlap(): - # end_line of s1 == start_line - 1 of s2: not overlapping - s1 = _SeqInfo([], 1, 5, "", "", "fp1") - s2 = _SeqInfo([], 6, 10, "", "", "fp1") - assert not _has_internal_overlap([s1, s2]) - - -def test_has_internal_overlap_touching(): - # end_line of s1 == start_line of s2: overlap (shared boundary line) - s1 = _SeqInfo([], 1, 5, "", "", "fp1") - s2 = _SeqInfo([], 5, 9, "", "", "fp1") - assert _has_internal_overlap([s1, s2]) - - -def test_has_internal_overlap_proper_overlap(): - s1 = _SeqInfo([], 27, 30, "", "", "fp1") - s2 = _SeqInfo([], 29, 32, "", "", "fp1") - assert _has_internal_overlap([s1, s2]) - - -def test_has_internal_overlap_unsorted_order(): - # Sequences given in reverse order — function must sort before checking. - s1 = _SeqInfo([], 29, 32, "", "", "fp1") - s2 = _SeqInfo([], 27, 30, "", "", "fp1") - assert _has_internal_overlap([s1, s2]) - - -def test_find_duplicate_groups_skips_internally_overlapping(): - # Simulate the op_range pattern: two pairs [A,B] and [B,C] that share a - # statement. The group has internal overlap and must be filtered out. - s1 = _SeqInfo([], 27, 30, "", "", "fp1") - s2 = _SeqInfo([], 29, 32, "", "", "fp1") - # Diff covers both sequences. - groups = _find_duplicate_groups([s1, s2], [(27, 32)]) - assert groups == [] - - -def test_find_duplicate_groups_caps_at_max_groups(): - sequences = [] - for i in range(6): - fp = f"fp{i}" - # Place each group in a disjoint band of 20 lines so _filter_maximal_groups - # keeps all 6 (none overlap), and the max_groups=3 cap is what limits output. - sequences.append(_SeqInfo([], i * 20 + 1, i * 20 + 3, "", "", fp)) - sequences.append(_SeqInfo([], i * 20 + 10, i * 20 + 12, "", "", fp)) - # Diff range covers all sequences so the diff-overlap filter passes for all. - groups = _find_duplicate_groups(sequences, [(1, 130)], max_groups=3) - assert len(groups) == 3 - - -# --------------------------------------------------------------------------- -# _filter_maximal_groups -# --------------------------------------------------------------------------- - - -def test_filter_maximal_groups_empty(): - assert _filter_maximal_groups([]) == [] - - -def test_filter_maximal_groups_single_group(): - s1 = _SeqInfo([], 1, 10, "", "", "fp1") - s2 = _SeqInfo([], 20, 29, "", "", "fp1") - group = [s1, s2] - result = _filter_maximal_groups([group]) - assert result == [group] - - -def test_filter_maximal_groups_removes_subsumed(): - # Large group spans lines 1-10; small group spans 1-5 (subset). - # Only the large group should be kept. - s_large_a = _SeqInfo([], 1, 10, "", "", "fp_large") - s_large_b = _SeqInfo([], 20, 29, "", "", "fp_large") - large_group = [s_large_a, s_large_b] - - s_small_a = _SeqInfo([], 1, 5, "", "", "fp_small") - s_small_b = _SeqInfo([], 20, 24, "", "", "fp_small") - small_group = [s_small_a, s_small_b] - - result = _filter_maximal_groups([small_group, large_group]) - assert len(result) == 1 - assert result[0] is large_group - - -def test_filter_maximal_groups_keeps_non_overlapping(): - # Two groups with completely disjoint line ranges — both should be kept. - s1a = _SeqInfo([], 1, 5, "", "", "fp1") - s1b = _SeqInfo([], 30, 34, "", "", "fp1") - group1 = [s1a, s1b] - - s2a = _SeqInfo([], 10, 14, "", "", "fp2") - s2b = _SeqInfo([], 40, 44, "", "", "fp2") - group2 = [s2a, s2b] - - result = _filter_maximal_groups([group1, group2]) - assert len(result) == 2 - - -# --------------------------------------------------------------------------- -# _verify_extraction -# --------------------------------------------------------------------------- - - -def test_verify_extraction_valid(): - helper = "def helper(x):\n return x + 1\n" - replacements = ["result = helper(a)\n"] - assert _verify_extraction(helper, replacements) is True - - -def test_verify_extraction_invalid_helper(): - helper = "def helper(x:\n pass\n" # unclosed paren → syntax error after dedent - replacements = ["result = helper(a)\n"] - assert _verify_extraction(helper, replacements) is False - - -def test_verify_extraction_invalid_replacement(): - helper = "def helper(x):\n return x\n" - # Dedented replacement still has a syntax error - replacements = ["result = helper(a\n"] # unclosed paren - assert _verify_extraction(helper, replacements) is False - - -def test_verify_extraction_no_helper_source(): - # Exercises the helper_source is None branch (skips helper compile check). - assert _verify_extraction(None, ["result = f()\n"]) is True - - -def test_verify_extraction_fails_on_param_overwrite(): - # Helper where the parameter is immediately overwritten before being read. - helper = "def setup(mock_obj):\n mock_obj = object()\n return mock_obj\n" - assert _verify_extraction(helper, ["x = setup(y)\n"]) is False - - -def test_verify_extraction_allows_return_in_replacement(): - # Replacements inside function bodies legally contain 'return'; the dummy- - # function wrapper must allow this without triggering a false rejection. - helper = "def helper(x):\n return x\n" - replacements = [" return helper(a)\n"] - assert _verify_extraction(helper, replacements) is True - - -def test_verify_extraction_allows_multiline_return_replacement(): - # Multi-line replacement ending with a return statement. - helper = "def helper(source):\n return helper(source)\n" - replacements = [ - " tree = helper(source)\n if tree is None:\n return set()\n" - ] - assert _verify_extraction(helper, replacements) is True - - -def test_verify_extraction_allows_continue_in_replacement(): - # 'continue' is valid inside a loop body; the dummy wrapper now includes a - # for loop so this is not rejected as a SyntaxError. - helper = "def helper():\n pass\n" - replacements = [" if done:\n continue\n"] - assert _verify_extraction(helper, replacements) is True - - -def test_verify_extraction_allows_break_in_replacement(): - # Same as above but for 'break'. - helper = "def helper():\n pass\n" - replacements = [" if done:\n break\n"] - assert _verify_extraction(helper, replacements) is True - - -def test_verify_extraction_allows_await_in_replacement(): - # Replacements inside async functions legally contain 'await'; the async - # dummy-function wrapper must allow this without triggering a false rejection. - helper = "async def helper(x):\n return await x\n" - replacements = [" result = await helper(coro)\n"] - assert _verify_extraction(helper, replacements) is True - - -def test_verify_extraction_allows_async_helper(): - # async def helpers are valid Python and must compile successfully. - helper = "async def helper(client, x):\n return await client.get(x)\n" - replacements = [" val = await helper(client, url)\n"] - assert _verify_extraction(helper, replacements) is True - - -def test_verify_extraction_rejects_invalid_await_replacement(): - # Replacement with `await` that also has a real syntax error must still fail. - helper = "async def helper(x):\n return await x\n" - replacements = [" result = await helper(coro\n"] # unclosed paren - assert _verify_extraction(helper, replacements) is False - - -# --------------------------------------------------------------------------- -# _has_mutable_literal_is_check -# --------------------------------------------------------------------------- - - -def test_has_mutable_literal_is_check_set_constructor(): - assert _has_mutable_literal_is_check("if x is set(): pass") is True - - -def test_has_mutable_literal_is_check_list_constructor(): - assert _has_mutable_literal_is_check("if x is list(): pass") is True - - -def test_has_mutable_literal_is_check_dict_constructor(): - assert _has_mutable_literal_is_check("if x is dict(): pass") is True - - -def test_has_mutable_literal_is_check_list_literal(): - assert _has_mutable_literal_is_check("if x is []: pass") is True - - -def test_has_mutable_literal_is_check_dict_literal(): - assert _has_mutable_literal_is_check("if x is {}: pass") is True - - -def test_has_mutable_literal_is_check_isnot(): - assert _has_mutable_literal_is_check("if x is not set(): pass") is True - - -def test_has_mutable_literal_is_check_none_is_fine(): - assert _has_mutable_literal_is_check("if x is None: pass") is False - - -def test_has_mutable_literal_is_check_isinstance_is_fine(): - assert _has_mutable_literal_is_check("if isinstance(x, set): pass") is False - - -def test_has_mutable_literal_is_check_equality_is_fine(): - # == comparison with set() is valid; only identity (`is`) is wrong - assert _has_mutable_literal_is_check("if x == set(): pass") is False - - -def test_has_mutable_literal_is_check_syntax_error(): - assert _has_mutable_literal_is_check("def f(x:") is False - - -def test_verify_extraction_rejects_mutable_is_in_helper(): - helper = "def h(x):\n if x is set(): return True\n return False\n" - assert _verify_extraction(helper, ["h(a)\n"]) is False - - -def test_verify_extraction_rejects_mutable_is_in_replacement(): - helper = "def h(x):\n return x\n" - assert _verify_extraction(helper, ["if r is set(): pass\n"]) is False - - -def test_verify_extraction_rejects_indented_mutable_is_in_replacement(): - # Indented replacements (function-body code) are wrapped before checking, - # so `is set()` is caught even when ast.parse would fail on raw indented text. - helper = "def h(x):\n return x\n" - assert _verify_extraction(helper, [" if r is set(): pass\n"]) is False - - -# --------------------------------------------------------------------------- -# _collect_attribute_names -# --------------------------------------------------------------------------- - - -def test_collect_attribute_names_basic(): - assert _collect_attribute_names("x.foo()\ny.bar") == {"foo", "bar"} - - -def test_collect_attribute_names_nested(): - assert "baz" in _collect_attribute_names("a.b.baz()") - - -def test_collect_attribute_names_syntax_error(): - assert _collect_attribute_names("def f(x:") == set() - - -def test_collect_attribute_names_no_attrs(): - assert _collect_attribute_names("x = 1 + 2") == set() - - -# --------------------------------------------------------------------------- -# _collect_called_attr_names -# --------------------------------------------------------------------------- - - -def test_collect_called_attr_names_method_call(): - # obj.foo() → "foo" is a called attribute - assert _collect_called_attr_names("obj.foo()") == {"foo"} - - -def test_collect_called_attr_names_ignores_plain_access(): - # obj.bar (not called) → not included - assert "bar" not in _collect_called_attr_names("x = obj.bar") - - -def test_collect_called_attr_names_ignores_type_annotation(): - # ast.AST used as a type annotation is NOT a method call → not flagged - assert "AST" not in _collect_called_attr_names( - "def f(x) -> Optional[ast.AST]: pass" - ) - - -def test_collect_called_attr_names_syntax_error(): - assert _collect_called_attr_names("def f(x:") == set() - - -def test_collect_called_attr_names_no_calls(): - assert _collect_called_attr_names("x = 1 + 2") == set() - - -# --------------------------------------------------------------------------- -# _has_call_to -# --------------------------------------------------------------------------- - - -def test_has_call_to_direct_call(): - assert _has_call_to("foo", "foo()\n") is True - - -def test_has_call_to_attribute_call(): - assert _has_call_to("foo", "obj.foo()\n") is True - - -def test_has_call_to_missing(): - assert _has_call_to("foo", "bar()\n") is False - - -def test_has_call_to_syntax_error(): - assert _has_call_to("foo", "def f(x:") is False - - -# --------------------------------------------------------------------------- -# _has_funcdef -# --------------------------------------------------------------------------- - - -def test_has_funcdef_present(): - assert _has_funcdef("_helper", "def _helper(x):\n pass\n") is True - - -def test_has_funcdef_async(): - assert _has_funcdef("_helper", "async def _helper(x):\n pass\n") is True - - -def test_has_funcdef_missing(): - assert _has_funcdef("_helper", "def other(x):\n pass\n") is False - - -def test_has_funcdef_syntax_error(): - assert _has_funcdef("_helper", "def f(x:") is False - - -# --------------------------------------------------------------------------- -# _normalize_replacement_indentation -# --------------------------------------------------------------------------- - - -def _make_seq_with_source(source: str) -> _SeqInfo: - return _SeqInfo( - stmts=[], start_line=1, end_line=1, scope="f", source=source, fingerprint="" - ) - - -def test_normalize_indentation_already_correct(): - # Replacement already matches the block's indentation — unchanged. - seq = _make_seq_with_source(" x = compute()\n y = finalize(x)\n") - replacement = " result = helper()\n" - assert ( - _normalize_replacement_indentation(seq, replacement) - == " result = helper()\n" - ) - - -def test_normalize_indentation_col0_to_indented(): - # Replacement at column 0 is re-indented to match the original block. - seq = _make_seq_with_source(" x = compute()\n y = finalize(x)\n") - replacement = "result = helper()\n" - assert ( - _normalize_replacement_indentation(seq, replacement) - == " result = helper()\n" - ) - - -def test_normalize_indentation_multiline(): - # Multi-line replacement at column 0 gets uniformly re-indented. - seq = _make_seq_with_source(" x = a()\n y = b(x)\n") - replacement = "x = helper()\nif x is None:\n x = default()\n" - expected = ( - " x = helper()\n if x is None:\n x = default()\n" - ) - assert _normalize_replacement_indentation(seq, replacement) == expected - - -def test_normalize_indentation_module_level_block(): - # Module-level block (no indent) — replacement is just dedented. - seq = _make_seq_with_source("x = compute()\ny = finalize(x)\n") - replacement = "result = helper()\n" - assert _normalize_replacement_indentation(seq, replacement) == "result = helper()\n" - - -def test_normalize_indentation_empty_source(): - # Empty source — no indentation can be inferred; replacement returned as-is. - seq = _make_seq_with_source("") - replacement = "result = helper()\n" - assert _normalize_replacement_indentation(seq, replacement) == replacement - - -# --------------------------------------------------------------------------- -# _has_param_overwritten_before_read -# --------------------------------------------------------------------------- - - -def test_has_param_overwritten_before_read_false_when_param_is_read(): - # Parameter is read before (or without) being reassigned — should return False. - helper = "def fn(x):\n return x + 1\n" - assert _has_param_overwritten_before_read(helper) is False - - -def test_has_param_overwritten_before_read_true_when_immediately_overwritten(): - # Parameter is assigned on the first statement without being read — True. - helper = "def setup(client):\n client = object()\n return client\n" - assert _has_param_overwritten_before_read(helper) is True - - -def test_has_param_overwritten_before_read_false_for_conditional_default(): - # The ``if x is None: x = default`` pattern reads before writing — False. - helper = "def fn(x=None):\n if x is None:\n x = []\n return x\n" - assert _has_param_overwritten_before_read(helper) is False - - -def test_has_param_overwritten_before_read_vararg_and_kwarg(): - # Covers the vararg/kwarg branches — neither is overwritten here. - helper = "def fn(*args, **kwargs):\n return args, kwargs\n" - assert _has_param_overwritten_before_read(helper) is False - - -# --------------------------------------------------------------------------- -# _pyflakes_new_undefined_names -# --------------------------------------------------------------------------- - - -def test_pyflakes_new_undefined_names_returns_empty_when_no_new_issues(): - # Names undefined in both original and candidate → no NEW issues. - original = "def foo():\n return bar()\n" - candidate = "def _h():\n pass\n\ndef foo():\n return bar()\n" - assert _pyflakes_new_undefined_names(original, candidate) == set() - - -def test_pyflakes_new_undefined_names_detects_introduced_name(): - # candidate introduces a reference to an unassigned name not in original. - original = "def foo():\n x = 1\n return x\n" - # candidate removes the assignment, leaving x undefined at the call site - candidate = "def _h():\n x = 1\n\ndef foo():\n _h(x)\n return x\n" - assert "x" in _pyflakes_new_undefined_names(original, candidate) - - -# --------------------------------------------------------------------------- -# _is_pure_literal -# --------------------------------------------------------------------------- - - -def test_is_pure_literal_constant(): - import ast - - assert _is_pure_literal(ast.parse("0", mode="eval").body) - assert _is_pure_literal(ast.parse('"s"', mode="eval").body) - assert _is_pure_literal(ast.parse("None", mode="eval").body) - assert _is_pure_literal(ast.parse("True", mode="eval").body) - - -def test_is_pure_literal_containers(): - import ast - - assert _is_pure_literal(ast.parse("[]", mode="eval").body) - assert _is_pure_literal(ast.parse("(1, 2)", mode="eval").body) - assert _is_pure_literal(ast.parse("{1: 2}", mode="eval").body) - assert _is_pure_literal(ast.parse("{1, 2}", mode="eval").body) - - -def test_is_pure_literal_call_is_false(): - import ast - - assert not _is_pure_literal(ast.parse("func()", mode="eval").body) - - -def test_is_pure_literal_name_is_false(): - import ast - - assert not _is_pure_literal(ast.parse("x", mode="eval").body) - - -def test_is_pure_literal_nested_call_is_false(): - import ast - - assert not _is_pure_literal(ast.parse("[func()]", mode="eval").body) - - -# --------------------------------------------------------------------------- -# _pyflakes_strip_unused_simple_assigns -# --------------------------------------------------------------------------- - - -def test_pyflakes_strip_unused_simple_assigns_removes_literal_init(): - # last_import_line = 0 becomes unused after extraction. - source = textwrap.dedent( - """\ - def foo(source): - last_import_line = 0 - lines = source.splitlines() - return lines - """ - ) - result = _pyflakes_strip_unused_simple_assigns(source, {"last_import_line"}) - assert "last_import_line" not in result - assert "lines = source.splitlines()" in result - - -def test_pyflakes_strip_unused_simple_assigns_keeps_call_rhs(): - # x = func() must NOT be stripped — it has side effects. - source = textwrap.dedent( - """\ - def foo(): - x = side_effect() - return 1 - """ - ) - result = _pyflakes_strip_unused_simple_assigns(source, {"x"}) - assert "x = side_effect()" in result - - -def test_pyflakes_strip_unused_simple_assigns_no_change_when_used(): - source = textwrap.dedent( - """\ - def foo(source): - last_import_line = 0 - for line in source.splitlines(): - last_import_line += 1 - return last_import_line - """ - ) - result = _pyflakes_strip_unused_simple_assigns(source, {"last_import_line"}) - assert result == source - - -def test_pyflakes_strip_unused_simple_assigns_fallback_on_empty_block(): - # If stripping would leave a block with no statements (syntax error), - # the original source is returned unchanged. - source = textwrap.dedent( - """\ - def foo(): - x = 0 - """ - ) - # After stripping x = 0 the function body is empty — SyntaxError. - result = _pyflakes_strip_unused_simple_assigns(source, {"x"}) - assert result == source - - -def test_pyflakes_strip_unused_simple_assigns_module_level_unchanged(): - # Module-level assignments are not flagged as UnusedVariable by pyflakes. - source = "x = 0\n" - result = _pyflakes_strip_unused_simple_assigns(source, {"x"}) - assert result == source - - -def test_pyflakes_strip_unused_simple_assigns_skips_unrelated_names(): - # A variable unused after extraction but NOT in allowed_names is preserved. - source = textwrap.dedent( - """\ - def foo(source): - unrelated = 0 - lines = source.splitlines() - return lines - """ - ) - # "unrelated" is not in the allowed set → must not be removed. - result = _pyflakes_strip_unused_simple_assigns(source, {"last_import_line"}) - assert "unrelated = 0" in result - - -def test_pyflakes_strip_unused_simple_assigns_empty_allowed(): - # Empty allowed_names means nothing can be stripped. - source = textwrap.dedent( - """\ - def foo(source): - x = 0 - lines = source.splitlines() - return lines - """ - ) - result = _pyflakes_strip_unused_simple_assigns(source, set()) - assert result == source - - -# --------------------------------------------------------------------------- -# _names_in_edit_texts -# --------------------------------------------------------------------------- - - -def test_names_in_edit_texts_collects_from_all_edits(): - groups = [ - ( - "_helper", - [ - (1, 3, "def _helper(last_import_line):\n return last_import_line\n"), - (5, 6, "result = _helper(x)\n"), - ], - "msg", - ) - ] - names = _names_in_edit_texts(groups) - assert "last_import_line" in names - assert "_helper" in names - assert "result" in names - assert "x" in names - - -def test_names_in_edit_texts_skips_syntax_errors(): - groups = [("_h", [(1, 2, "def (\n")], "msg")] - # Should not raise — returns whatever names were parseable. - names = _names_in_edit_texts(groups) - assert isinstance(names, set) - - -# --------------------------------------------------------------------------- -# _missing_free_vars -# --------------------------------------------------------------------------- - - -def test_missing_free_vars_catches_missing_name(): - # The exact bug pattern: `new_source` is a local variable read in the - # original block, but the LLM turned it into `transformer.new_source` - # (an attribute access). Neither the call site nor the helper body contain - # a bare `new_source` Name node. - source = ( - "def run(transformer, file_msgs, filepath):\n" - " new_source = get_source()\n" - " current_source = new_source\n" - ) - block_src = " current_source = new_source\n" - call_src = " current_source = _h(transformer, filepath, file_msgs)\n" - helper_src = ( - "def _h(transformer, filepath, file_msgs):\n" - " return transformer.new_source\n" - ) - assert "new_source" in _missing_free_vars(block_src, [call_src], helper_src, source) - - -def test_missing_free_vars_no_missing_when_passed_as_arg(): - # Free var is passed as an argument to the helper → not missing. - source = ( - "def run():\n new_source = get_source()\n current_source = new_source\n" - ) - block_src = " current_source = new_source\n" - call_src = " current_source = _h(new_source)\n" - helper_src = "def _h(new_source):\n return new_source\n" - assert _missing_free_vars(block_src, [call_src], helper_src, source) == set() - - -def test_missing_free_vars_ignores_block_locals(): - # `x` is assigned AND read within the block — it is a local, not a free - # variable. It should not be flagged even if it's absent from the helper. - source = "def run():\n x = 1\n result = x + 1\n" - block_src = " x = 1\n result = x + 1\n" - call_src = " result = _h()\n" - helper_src = "def _h():\n x = 1\n return x + 1\n" - assert _missing_free_vars(block_src, [call_src], helper_src, source) == set() - - -def test_missing_free_vars_ignores_module_level_names(): - # `compute`, `transform`, `finalize` are module-level function names that - # are never assigned anywhere — the helper can reference them directly. - source = ( - "def foo():\n" - " x = compute(data)\n" - " y = transform(x)\n" - " z = finalize(y)\n" - ) - block_src = " x = compute(data)\n y = transform(x)\n z = finalize(y)\n" - call_src = " _helper(data)\n" - helper_src = "def _helper(data):\n pass\n" - assert _missing_free_vars(block_src, [call_src], helper_src, source) == set() - - -def test_missing_free_vars_syntax_error_in_block_returns_empty(): - assert ( - _missing_free_vars("not valid python!!!", ["x = 1\n"], "def f(): pass\n", "") - == set() - ) - - -def test_missing_free_vars_syntax_error_in_replacement_returns_empty(): - source = "def run():\n a = 1\n" - assert ( - _missing_free_vars("x = a\n", ["not valid!!!\n"], "def f(): pass\n", source) - == set() - ) - - -def test_missing_free_vars_syntax_error_in_source_returns_empty(): - assert ( - _missing_free_vars("x = a\n", ["y = a\n"], "def f(a): pass\n", "not valid!!!") - == set() - ) - - -def test_missing_free_vars_empty_block_returns_empty(): - # A block with no reads has no free vars → nothing can be missing. - source = "def run():\n x = 1\n" - block_src = " x = 1\n" - call_src = " _h()\n" - helper_src = "def _h():\n x = 1\n" - assert _missing_free_vars(block_src, [call_src], helper_src, source) == set() - - -def test_missing_free_vars_function_parameter_is_caught(): - # A function parameter that's free in the block must appear in the - # replacement — parameters are local to the function and cannot be - # accessed by a helper without being passed as an argument. - source = "def run(verbose):\n msg = verbose\n" - block_src = " msg = verbose\n" - call_src = " msg = _h()\n" - helper_src = "def _h():\n pass\n" - assert "verbose" in _missing_free_vars(block_src, [call_src], helper_src, source) - - -# --------------------------------------------------------------------------- -# _names_assigned_in -# --------------------------------------------------------------------------- - - -def test_names_assigned_in_simple(): - assert _names_assigned_in("x = 1\n") == {"x"} - - -def test_names_assigned_in_tuple_unpack(): - assert _names_assigned_in("x, y = f()\n") == {"x", "y"} - - -def test_names_assigned_in_augassign(): - assert _names_assigned_in("x += 1\n") == {"x"} - - -def test_names_assigned_in_no_assign(): - assert _names_assigned_in("f()\n") == set() - - -def test_names_assigned_in_syntax_error(): - assert _names_assigned_in("def (\n") == set() - - -# --------------------------------------------------------------------------- -# _extract_defined_names -# --------------------------------------------------------------------------- - - -def test_extract_defined_names_basic(): - source = textwrap.dedent( - """\ - def foo(): - pass - - async def bar(): - pass - - class Baz: - pass - """ - ) - assert _extract_defined_names(source) == {"foo", "bar", "Baz"} - - -def test_extract_defined_names_syntax_error(): - assert _extract_defined_names("def (\n") == set() - - -# --------------------------------------------------------------------------- -# _find_escaping_vars -# --------------------------------------------------------------------------- - - -def _make_esc_seq(start: int, end: int) -> _SeqInfo: - """Create a _SeqInfo for escaping-vars tests.""" - return _SeqInfo( - stmts=[], - start_line=start, - end_line=end, - scope="foo", - source="", - fingerprint="", - ) - - -def test_find_escaping_vars_no_assignments(): - # Block has no assignments → skip (branch A), returns empty set. - source_lines = [ - "def foo():\n", - " compute()\n", - " transform()\n", - " use_result()\n", - ] - seq = _make_esc_seq(2, 3) - assert _find_escaping_vars([seq], source_lines) == set() - - -def test_find_escaping_vars_nothing_after_block(): - # Block is the last thing in scope → after_lines empty (branch D), returns set(). - source_lines = [ - "def foo():\n", - " x = compute()\n", - " y = transform(x)\n", - " z = finalize(y)\n", - ] - seq = _make_esc_seq(2, 4) - assert _find_escaping_vars([seq], source_lines) == set() - - -def test_find_escaping_vars_escapes(): - # Block assigns z; z is used after the block → {"z"}. - # Also covers: blank line (branch B) and lower-indent stop (branch C). - source_lines = [ - "def foo():\n", - " x = compute()\n", - " y = transform(x)\n", - " z = finalize(y)\n", # block ends line 4 - "\n", # blank → branch B - " assert z == 42\n", # same indent, uses z - "\n", - "def bar():\n", # indent 0 < 4 → branch C (stop) - " pass\n", - ] - seq = _make_esc_seq(2, 4) - assert _find_escaping_vars([seq], source_lines) == {"z"} - - -def test_find_escaping_vars_no_escape(): - # Block assigns x/y/z; none referenced after the block → set(). - source_lines = [ - "def foo():\n", - " x = compute()\n", - " y = transform(x)\n", - " z = finalize(y)\n", - " print('done')\n", # uses 'print', not x/y/z - ] - seq = _make_esc_seq(2, 4) - assert _find_escaping_vars([seq], source_lines) == set() - - -def test_find_escaping_vars_syntax_error_after(): - # After source is invalid Python → SyntaxError branch: continue, returns set(). - source_lines = [ - "def foo():\n", - " x = compute()\n", - " y = transform(x)\n", - " z = finalize(y)\n", - " def bar(x\n", # unclosed paren at same indent - ] - seq = _make_esc_seq(2, 4) - assert _find_escaping_vars([seq], source_lines) == set() - - -def test_find_escaping_vars_module_level_stops_at_def(): - # Module-level block (indent 0): a non-def/class line is included, - # then a def line stops the scan (break via re.match). - source_lines = [ - "x = compute()\n", - "y = transform(x)\n", - "z = finalize(y)\n", # block ends line 3 - "CONSTANT = 42\n", # module-level non-def → appended (False branch of re.match) - "def foo(z):\n", # module-level def → stop - " return z\n", - ] - seq = _make_esc_seq(1, 3) - # CONSTANT is in after_lines; not in assigned → set(). - # z inside def foo(z) is not scanned (stopped before that def). - assert _find_escaping_vars([seq], source_lines) == set() - - -# --------------------------------------------------------------------------- -# _apply_edits -# --------------------------------------------------------------------------- - - -def test_apply_edits_no_edits(): - source = "a = 1\nb = 2\n" - assert _apply_edits(source, []) == source - - -def test_apply_edits_replacement(): - source = "a = 1\nb = 2\nc = 3\n" - # Replace line index 1 (b = 2) with new content - result = _apply_edits(source, [(1, 2, "x = 99\n")]) - assert result == "a = 1\nx = 99\nc = 3\n" - - -def test_apply_edits_insertion(): - source = "a = 1\nb = 2\n" - # Insert before line index 1 (b = 2) - result = _apply_edits(source, [(1, 1, "INSERTED\n")]) - assert result == "a = 1\nINSERTED\nb = 2\n" - - -def test_apply_edits_overlapping_skipped(): - source = "a = 1\nb = 2\nc = 3\n" - edits = [ - (0, 2, "FIRST\n"), - (1, 3, "SECOND\n"), # overlaps with first - ] - result = _apply_edits(source, edits) - # Higher-start edit (SECOND) wins; FIRST overlaps and is skipped - assert "SECOND" in result - assert "FIRST" not in result - - -def test_apply_edits_no_trailing_newline_source(): - source = "a = 1" # no trailing newline - result = _apply_edits(source, [(0, 1, "b = 2\n")]) - assert result == "b = 2\n" - - -def test_apply_edits_no_trailing_newline_text(): - source = "a = 1\nb = 2\n" - # Replacement text without trailing newline - result = _apply_edits(source, [(0, 1, "x = 99")]) - assert result == "x = 99\nb = 2\n" - - -# --------------------------------------------------------------------------- -# _find_insertion_point -# --------------------------------------------------------------------------- - - -def test_find_insertion_point_module_with_imports(): - source = "import os\nfrom sys import argv\n\ndef foo():\n pass\n" - # Should insert after the last import (index 1), so return 2 - assert _find_insertion_point(source, "") == 2 - - -def test_find_insertion_point_module_no_imports(): - source = "a = 1\n" - # No imports: last_import stays -1, returns 0 - assert _find_insertion_point(source, "") == 0 - - -def test_find_insertion_point_function_found(): - source = "import os\n\ndef target():\n pass\n" - # def target is at line index 2 - assert _find_insertion_point(source, "target") == 2 - - -def test_find_insertion_point_function_not_found(): - source = "a = 1\n" - # Falls back to 0 - assert _find_insertion_point(source, "missing_func") == 0 - - -def test_find_insertion_point_class_method_inserts_before_class(): - # def bar is indented inside class Foo; helper must go before the class, - # not inside it (which would end the class and turn _analyze into a nested func). - source = "import os\n\nclass Foo:\n\n def bar(self):\n pass\n" - # source_lines: ["import os", "", "class Foo:", "", - # " def bar(self):", " pass"] - # "def bar" found at i=4 (indent=4). Walk back: - # j=3 → blank → skip; j=2 → "class Foo:" indent=0 < 4 → return 2 - assert _find_insertion_point(source, "bar") == 2 - - -def test_find_insertion_point_nested_function_no_class(): - # def inner is indented inside def outer (no enclosing class). - # method_indent > 0, loop finds a non-class def at lower indent → break. - # Falls through to decorator walk, which returns i (the line of def inner). - source = "def outer():\n def inner():\n pass\n" - # "def inner" found at i=1 (indent=4). Walk back: - # j=0 → "def outer():" indent=0 < 4, not a class → break. - # Falls through to return 1. - assert _find_insertion_point(source, "inner") == 1 - - -def test_find_insertion_point_nested_func_ignores_unrelated_class(): - # Regression: a nested function inside a module-level function must not - # be confused with a class method just because an unrelated class appears - # earlier in the file. Before the fix the backward walk would skip past - # the outer function (non-class, lower indent) and incorrectly match the - # unrelated class, causing the helper to be inserted between the class's - # decorator and its class statement. - import textwrap as _textwrap - - source = _textwrap.dedent( - """\ - @dataclass - class _SplitTask: - pass - - - def _find_free_vars(): - x = 1 - def _collect_loads(): - pass - """ - ) - # source_lines (0-based): - # 0: "@dataclass\n" - # 1: "class _SplitTask:\n" - # 2: " pass\n" - # 3: "\n" - # 4: "\n" - # 5: "def _find_free_vars():\n" - # 6: " x = 1\n" - # 7: " def _collect_loads():\n" - # 8: " pass\n" - # "def _collect_loads" found at i=7 (indent=4). Walk back: - # j=6: " x = 1" indent=4, not < 4 → continue - # j=5: "def _find_free_vars():" indent=0 < 4, NOT class → break - # Falls through to decorator walk: j=6 (" x = 1"), not a decorator - # → break → return j+1 = 7. - # The old (unfixed) code would have continued past j=5 and returned 1, - # placing the helper between @dataclass and class _SplitTask:. - result = _find_insertion_point(source, "_collect_loads") - assert result != 1, "must not insert inside @dataclass/_SplitTask boundary" - assert result == 7 - - -def test_find_insertion_point_indented_func_at_file_start(): - # Edge case: the target def has method_indent > 0 but is at line 0 so the - # backward-search loop range is empty. Falls through to decorator walk - # which also exits immediately (j=-1), returning 0. - source = " def inner():\n pass\n" - # "def inner" found at i=0 (indent=4). range(-1, -1, -1) is empty → loop - # body never runs → fall through to decorator walk → j = -1 → return 0. - assert _find_insertion_point(source, "inner") == 0 - - -def test_find_insertion_point_async_def(): - # Regression: helpers extracted from async functions were inserted at line 0 - # (before imports) because the pattern only matched 'def', not 'async def'. - source = ( - "import pytest\n" # 0 - "\n" # 1 - "async def target(client):\n" # 2 - " pass\n" # 3 - ) - assert _find_insertion_point(source, "target") == 2 - - -def test_find_insertion_point_async_def_with_decorator(): - # async def with a preceding decorator: helper should land before the decorator. - source = ( - "import pytest\n" # 0 - "\n" # 1 - "@pytest.mark.asyncio\n" # 2 - "async def target(client):\n" # 3 - " pass\n" # 4 - ) - assert _find_insertion_point(source, "target") == 2 - - -def test_find_insertion_point_skips_over_decorators(): - # Helper must be inserted before the decorator block, not between the - # decorators and the def they decorate. - source = ( - "import os\n" # 0 - "\n" # 1 - "@decorator\n" # 2 - "def target():\n" # 3 - " pass\n" # 4 - ) - # Without the fix this would return 3 (the def line); with the fix it - # should return 2 (the @decorator line). - assert _find_insertion_point(source, "target") == 2 - - -def test_find_insertion_point_skips_over_multiline_decorator(): - # Multi-line decorator: @patch(\n "..."\n) above the def. - source = ( - "import os\n" # 0 - "\n" # 1 - "@patch(\n" # 2 - ' "some.module"\n' # 3 - ")\n" # 4 - "def target():\n" # 5 - " pass\n" # 6 - ) - # Should return 2 (before the @patch line), not 5 (the def line). - assert _find_insertion_point(source, "target") == 2 - - -# --------------------------------------------------------------------------- -# _skip_class_docstring -# --------------------------------------------------------------------------- - - -def test_skip_class_docstring_no_docstring(): - source = "class Foo:\n def method(self):\n pass\n" - lines = source.splitlines() - # after_class_line=1 (line " def method..."), no docstring → unchanged - assert _skip_class_docstring(lines, 1) == 1 - - -def test_skip_class_docstring_triple_double_quote_single_line(): - source = 'class Foo:\n """A docstring."""\n def method(self):\n pass\n' - lines = source.splitlines() - # after_class_line=1 is the docstring line; should return 2 - assert _skip_class_docstring(lines, 1) == 2 - - -def test_skip_class_docstring_triple_single_quote_single_line(): - source = "class Foo:\n '''A docstring.'''\n def method(self):\n pass\n" - lines = source.splitlines() - assert _skip_class_docstring(lines, 1) == 2 - - -def test_skip_class_docstring_triple_quote_multiline(): - source = ( - "class Foo:\n" - ' """First line.\n' - " Second line.\n" - ' """\n' - " def method(self):\n" - " pass\n" - ) - lines = source.splitlines() - # Closing """ is on line 3 (0-based); should return 4 - assert _skip_class_docstring(lines, 1) == 4 - - -def test_skip_class_docstring_with_leading_blank_line(): - source = 'class Foo:\n\n """Docstring."""\n def method(self):\n pass\n' - lines = source.splitlines() - # Line 1 is blank, line 2 is the docstring; should return 3 - assert _skip_class_docstring(lines, 1) == 3 - - -def test_skip_class_docstring_empty_class(): - source = "class Foo:\n pass\n" - lines = source.splitlines() - assert _skip_class_docstring(lines, 1) == 1 - - -def test_skip_class_docstring_only_blank_lines(): - # after_class_line points past end of file after skipping blanks - lines = ["class Foo:", " "] - assert _skip_class_docstring(lines, 1) == 1 - - -def test_skip_class_docstring_malformed_multiline_no_close(): - # Triple-quoted docstring that never closes (malformed) — returns end-of-lines - lines = ["class Foo:", ' """This never closes', " still going"] - result = _skip_class_docstring(lines, 1) - assert result == 3 # past end of lines, best-effort - - -def test_skip_class_docstring_single_quote(): - # Single-quoted one-liner docstring - lines = ["class Foo:", ' "A brief note."', " def method(self): pass"] - assert _skip_class_docstring(lines, 1) == 2 - - -# --------------------------------------------------------------------------- -# _build_helper_insertion -# --------------------------------------------------------------------------- - - -def test_build_helper_insertion_blank_before_insert_pos(): - # Blank line at index 1 is before insert_pos=2 (before_blanks=1, after_blanks=0). - # insert_at=2 (pure insertion), leading=max(0,2-1)=1 so text starts with "\n". - source = "import os\n\ndef foo():\n pass\n" - lines = source.splitlines(keepends=True) - helper = "def _helper():\n pass\n" - start, end, text = _build_helper_insertion(lines, 2, helper, "module_level") - assert start == 2 - assert end == 2 # pure insertion - assert text.startswith("\n") - assert not text.startswith("\n\n") # only 1 leading blank needed - assert text.endswith("\n\n") - assert "def _helper():" in text - - -def test_build_helper_insertion_no_surrounding_blanks(): - # No blanks to absorb → pure insertion with 2 blank lines each side. - source = "import os\ndef foo():\n pass\n" - lines = source.splitlines(keepends=True) - helper = "def _helper():\n pass\n" - start, end, text = _build_helper_insertion(lines, 1, helper, "module_level") - assert start == 1 - assert end == 1 # pure insertion - assert text.startswith("\n\n") - assert text.endswith("\n\n") - - -def test_build_helper_insertion_staticmethod_uses_one_blank(): - # Staticmethod placement: 1 blank line before and after. - source = "class Foo:\n def bar(self):\n pass\n" - lines = source.splitlines(keepends=True) - helper = " @staticmethod\n def _h():\n pass\n" - start, end, text = _build_helper_insertion(lines, 1, helper, "staticmethod:Foo") - assert start == 1 - assert end == 1 # no blanks to absorb - assert text.startswith("\n") - assert not text.startswith("\n\n") - assert text.endswith("\n\n") # clean + 1 trailing blank = \n + \n - - -def test_build_helper_insertion_blank_at_insert_pos(): - # insert_pos=1 is the blank line itself (after_blanks=1, before_blanks=0). - # insert_at=1+1=2 (pure insertion after the blank), leading=max(0,2-1)=1. - source = "import os\n\ndef foo():\n pass\n" - lines = source.splitlines(keepends=True) - helper = "def _helper():\n pass\n" - start, end, text = _build_helper_insertion(lines, 1, helper, "module_level") - assert start == 2 - assert end == 2 # pure insertion - assert text.startswith("\n") - assert not text.startswith("\n\n") # only 1 leading blank needed - assert text.endswith("\n\n") - - -def test_build_helper_insertion_strips_extra_newlines_from_helper(): - # If the LLM returns a helper with leading/trailing blank lines, they are stripped. - source = "import os\ndef foo():\n pass\n" - lines = source.splitlines(keepends=True) - helper = "\n\ndef _helper():\n pass\n\n\n" - start, end, text = _build_helper_insertion(lines, 1, helper, "module_level") - assert text.startswith("\n\n") - assert text.endswith("\n\n") - assert "\n\n\n\ndef _helper" not in text # no extra leading blanks inside text - - -def test_build_helper_insertion_two_at_same_scope(): - # Two helpers inserted before the same def via _apply_edits: both must appear. - source = "import os\n\n\ndef foo():\n pass\n" - lines = source.splitlines(keepends=True) - edits = [ - _build_helper_insertion(lines, 3, "def _h1():\n pass\n", "module_level"), - _build_helper_insertion(lines, 3, "def _h2():\n pass\n", "module_level"), - ] - result = _apply_edits(source, edits) - assert "def _h1():" in result - assert "def _h2():" in result - assert "def foo():" in result - - -def test_successful_extraction_has_two_blank_lines(monkeypatch): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - # Each function has 4 statements. The first statement is STRUCTURALLY different - # between them (if-block vs assignment), so the normalizer produces different - # fingerprints for the full 4-stmt body. Only the trailing 3-stmt block - # (compute/transform/finalize) is duplicated, so the proxy-wrapper guard - # does not trigger (3 stmts < body_stmt_count 4). - source = textwrap.dedent( - """\ - import os - - def foo(): - if debug: - validate(data) - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(): - result = validate(data) - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor([(12, 14)], source=source) - - assert de._new_source is not None - # Exactly 2 blank lines before and after the inserted helper. - assert "\n\n\ndef _helper" in de._new_source - assert "\n\n\n\ndef _helper" not in de._new_source - assert "def _helper(data):\n pass\n\n\ndef foo" in de._new_source - - -def test_helper_placed_before_class_not_inside(monkeypatch): - """Helper extracted from class methods must be placed BEFORE the class. - - When duplicate blocks live inside class methods, inserting a module-level - helper before the method (inside the class body) ends the class definition - prematurely and turns the remaining methods into nested functions. The fix - in _find_insertion_point walks backwards to the enclosing class. - """ - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - import os - - class MyClass: - def method_a(self, x): - if self.debug: - pass - a = compute(x) - b = transform(a) - c = finalize(b) - return c - - def method_b(self, x): - result = None - a = compute(x) - b = transform(a) - c = finalize(b) - return c - """ - ) - helper = "def _do_work(x):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_do_work", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " return _do_work(x)\n", - " return _do_work(x)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor([(1, 100)], source=source) - - assert de._new_source is not None - compile(de._new_source, "", "exec") - # Helper must appear BEFORE the class definition, not inside it. - helper_pos = de._new_source.find("def _do_work") - class_pos = de._new_source.find("class MyClass") - assert ( - helper_pos < class_pos - ), "helper was placed after/inside class instead of before it" - # The class structure must be intact: MyClass still has both methods. - import ast as _ast - - tree = _ast.parse(de._new_source) - classes = [n for n in _ast.walk(tree) if isinstance(n, _ast.ClassDef)] - assert len(classes) == 1 - assert classes[0].name == "MyClass" - methods = [n.name for n in classes[0].body if isinstance(n, _ast.FunctionDef)] - assert "method_a" in methods - assert "method_b" in methods - - -# --------------------------------------------------------------------------- -# _collect_called_names -# --------------------------------------------------------------------------- - - -def test_collect_called_names_direct(): - names = _collect_called_names("foo()\n") - assert "foo" in names - - -def test_collect_called_names_method(): - names = _collect_called_names("obj.bar()\n") - assert "bar" in names - - -def test_collect_called_names_empty(): - names = _collect_called_names("x = 1\n") - assert names == set() - - -def test_collect_called_names_syntax_error(): - names = _collect_called_names("def f(: pass") - assert names == set() - - -def test_collect_called_names_other_callable(): - # func is a subscript (neither Name nor Attribute): funcs[0]() - # Covers the elif-False branch in _collect_called_names. - names = _collect_called_names("funcs[0]()\n") - assert "funcs" not in names # subscript call adds nothing - - -# --------------------------------------------------------------------------- -# _build_function_body_fps -# --------------------------------------------------------------------------- - - -def _make_func_info(name: str, body_source: str = " pass\n") -> _FunctionInfo: - return _FunctionInfo( - name=name, - source=f"def {name}():\n{body_source}", - scope="", - body_source=body_source, - body_stmt_count=1, - params=[], - ) - - -def test_build_fps_includes_called(): - body = " x = 1\n y = 2\n z = 3\n" - func = _make_func_info("foo", body) - fps = _build_function_body_fps([func], {"foo"}) - fp = _normalize_source(body) - assert fp in fps - assert fps[fp].name == "foo" - - -def test_build_fps_excludes_uncalled(): - func = _make_func_info("bar") - fps = _build_function_body_fps([func], {"foo"}) - assert fps == {} - - -def test_build_fps_empty_functions(): - fps = _build_function_body_fps([], {"foo"}) - assert fps == {} - - -# --------------------------------------------------------------------------- -# _SequenceCollector (integration via DuplicateExtractor internals) -# --------------------------------------------------------------------------- - - -def _collect_sequences(source: str, max_seq_len: int = 8): - tree = cst.parse_module(source) - lines = source.splitlines(keepends=True) - collector = _SequenceCollector(lines, max_seq_len=max_seq_len) - MetadataWrapper(tree).visit(collector) - return collector.sequences - - -def test_collector_finds_sequences(): - source = textwrap.dedent( - """\ - def foo(): - a = 1 - b = 2 - c = 3 - """ - ) - seqs = _collect_sequences(source) - assert len(seqs) > 0 - - -def test_collector_skips_light_sequences(): - # Only 2 statements — below weight threshold of 3 - source = textwrap.dedent( - """\ - def foo(): - a = 1 - b = 2 - """ - ) - seqs = _collect_sequences(source) - assert all(seq.start_line != seq.end_line or len(seq.stmts) >= 2 for seq in seqs) - # All 2-stmt windows skipped because weight < 3 - assert len([s for s in seqs if len(s.stmts) == 2]) == 0 - - -def test_collector_skips_defs(): - source = textwrap.dedent( - """\ - def foo(): - pass - def bar(): - pass - def baz(): - pass - """ - ) - seqs = _collect_sequences(source) - # Module-level sequences of defs should be skipped - for seq in seqs: - assert not _has_def(seq.stmts) - - -def test_collector_scope_tracking(): - source = textwrap.dedent( - """\ - def my_func(): - a = 1 - b = 2 - c = 3 - """ - ) - seqs = _collect_sequences(source) - func_seqs = [s for s in seqs if s.scope == "my_func"] - assert len(func_seqs) > 0 - - -def test_sequence_collector_custom_max_seq_len(): - # max_seq_len=2 means windows are at most 2 statements. - # With 4 statements each of weight 1, all 2-stmt windows have weight 2 < - # MIN_WEIGHT=3. So no sequences pass the weight filter → sequences == []. - source = textwrap.dedent( - """\ - def foo(): - a = 1 - b = 2 - c = 3 - d = 4 - """ - ) - seqs = _collect_sequences(source, max_seq_len=2) - # No 3-stmt (or larger) windows generated; all ≤2-stmt windows fail weight check. - assert all(len(s.stmts) <= 2 for s in seqs) - assert seqs == [] - - -# --------------------------------------------------------------------------- -# _FunctionCollector unit tests -# --------------------------------------------------------------------------- - - -def _collect_functions(source: str): - tree = cst.parse_module(source) - lines = source.splitlines(keepends=True) - collector = _FunctionCollector(lines) - MetadataWrapper(tree).visit(collector) - return collector.functions - - -def test_function_collector_module_level(): - source = "def foo():\n pass\n" - funcs = _collect_functions(source) - assert len(funcs) == 1 - assert funcs[0].name == "foo" - assert funcs[0].scope == "" - assert funcs[0].body_stmt_count == 1 - assert funcs[0].params == [] - - -def test_function_collector_class_level(): - source = "class C:\n def method(self):\n pass\n" - funcs = _collect_functions(source) - assert len(funcs) == 1 - assert funcs[0].name == "method" - assert funcs[0].scope == "C" - assert funcs[0].body_stmt_count == 1 - assert funcs[0].params == ["self"] - - -def test_function_collector_skips_nested(): - source = "def outer():\n def inner():\n pass\n" - funcs = _collect_functions(source) - assert len(funcs) == 1 - assert funcs[0].name == "outer" - assert funcs[0].body_stmt_count == 1 - assert funcs[0].params == [] - - -def test_function_collector_collects_body_source(): - source = "def foo():\n x = 1\n y = 2\n" - funcs = _collect_functions(source) - assert len(funcs) == 1 - assert "x = 1" in funcs[0].body_source - - -def test_function_collector_collects_stmt_count(): - source = "def foo():\n pass\n" - funcs = _collect_functions(source) - assert funcs[0].body_stmt_count == 1 - - -def test_function_collector_collects_params(): - source = "def f(x, y):\n pass\n" - funcs = _collect_functions(source) - assert funcs[0].params == ["x", "y"] - - -def test_function_collector_no_params(): - source = "def f():\n pass\n" - funcs = _collect_functions(source) - assert funcs[0].params == [] - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — no source -# --------------------------------------------------------------------------- - - -def test_no_source_no_analysis(): - de = DuplicateExtractor([(1, 5)]) - assert de._new_source is None - assert de.get_rewritten_source() is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — no duplicates -# --------------------------------------------------------------------------- - - -def test_no_duplicates_no_llm_calls(monkeypatch): - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - source = textwrap.dedent( - """\ - def foo(): - x = a + b - y = x * 2 - - def bar(): - if condition: - result = value - else: - result = other - """ - ) - # Structurally different blocks → no duplicate group → no API calls needed - de = DuplicateExtractor([(6, 9)], source=source) - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — missing API key -# --------------------------------------------------------------------------- - -_DUP_SOURCE = textwrap.dedent( - """\ - def foo(): - if debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ -) -_DUP_RANGES = [(10, 12)] # overlaps bar's duplicate block (x/y/z lines) - -# Source where foo's duplicate block assigns z, and foo uses z after the block. -# _has_escaping_vars should detect this and skip the extraction. -_ESC_SOURCE = textwrap.dedent( - """\ - def foo(): - x = compute(data) - y = transform(x) - z = finalize(y) - assert z == expected - - def bar(): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ -) -_ESC_RANGES = [(9, 11)] # overlaps bar's duplicate block (x/y/z lines) - - -def test_missing_api_key_raises(monkeypatch): - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - with pytest.raises(CrispenAPIError, match="ANTHROPIC_API_KEY"): - DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — API error in veto -# --------------------------------------------------------------------------- - - -def _make_veto_response(is_valid: bool, reason: str = "test") -> MagicMock: - block = MagicMock() - block.type = "tool_use" - block.name = "evaluate_duplicate" - block.input = {"is_valid_duplicate": is_valid, "reason": reason} - resp = MagicMock() - resp.content = [block] - return resp - - -def _make_extract_response(data: dict) -> MagicMock: - block = MagicMock() - block.type = "tool_use" - block.name = "extract_helper" - block.input = data - resp = MagicMock() - resp.content = [block] - return resp - - -def _make_verify_response(is_correct: bool, issues: list) -> MagicMock: - block = MagicMock() - block.type = "tool_use" - block.name = "verify_extraction" - block.input = {"is_correct": is_correct, "issues": issues} - resp = MagicMock() - resp.content = [block] - return resp - - -def test_api_error_in_veto_raises(monkeypatch): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = Exception("rate limit") - - with pytest.raises(CrispenAPIError): - DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) - - -def test_api_error_in_extract_raises(monkeypatch): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - # First call (veto) succeeds, second call (extract) fails - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - Exception("rate limit"), - ] - - with pytest.raises(CrispenAPIError): - DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — parse error in source -# --------------------------------------------------------------------------- - - -def test_parse_error_in_analyze(monkeypatch): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic.Anthropic"): - # Invalid Python: _analyze should return silently - de = DuplicateExtractor([(1, 1)], source="def f(: pass") - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — veto rejects -# --------------------------------------------------------------------------- - - -def test_veto_rejects_no_changes(monkeypatch): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.return_value = _make_veto_response(False) - - de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) - - assert de._new_source is None - assert de.changes_made == [] - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — wrong number of call site replacements -# --------------------------------------------------------------------------- - - -def test_wrong_replacement_count_skipped(monkeypatch): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "helper", - "placement": "module_level", - "helper_source": "def helper():\n pass\n", - "call_site_replacements": ["helper()\n"], # should be 2 - } - ), - ] - - de = DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - extraction_retries=0, - llm_verify_retries=0, - ) - - assert de._new_source is None - - -def test_wrong_replacement_count_skipped_verbose_false(monkeypatch): - # verbose=False covers the False branch of the new if-self.verbose guard. - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "helper", - "placement": "module_level", - "helper_source": "def helper():\n pass\n", - "call_site_replacements": ["helper()\n"], # should be 2 - } - ), - ] - - de = DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - verbose=False, - extraction_retries=0, - llm_verify_retries=0, - ) - - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — escaping variables passed to extraction prompt -# --------------------------------------------------------------------------- - - -def test_escaping_vars_passed_to_extract(monkeypatch): - # foo's block assigns z; foo uses z after the block. - # _find_escaping_vars returns {"z"}, which is passed to _llm_extract. - # The extraction prompt must contain the note instructing the LLM to return z. - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - helper_src = ( - "def _helper(data):\n" - " x = compute(data)\n" - " y = transform(x)\n" - " z = finalize(y)\n" - " return z\n" - ) - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper_src, - "call_site_replacements": [ - " z = _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor(_ESC_RANGES, source=_ESC_SOURCE) - - # The extraction prompt must include the escaping-variable note. - extract_call = mock_client.messages.create.call_args_list[1] - extract_prompt = extract_call.kwargs["messages"][0]["content"] - assert "immediately follows the block" in extract_prompt - assert de._new_source is not None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — assembled output not valid Python -# --------------------------------------------------------------------------- - - -def _make_invalid_assembled_extractor(monkeypatch, verbose=True): - """Helper: DuplicateExtractor where _apply_edits returns invalid Python.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic") as mock_anthropic, - patch( - "crispen.refactors.duplicate_extractor._apply_edits", - return_value="def f(:\n pass\n", # invalid Python - ), - ): - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(x):\n pass\n", - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - ] - return DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - verbose=verbose, - extraction_retries=0, - llm_verify_retries=0, - ) - - -def test_invalid_assembled_source_skipped(monkeypatch): - # Individual components pass _verify_extraction but the per-group assembled - # edit is invalid Python — the group is skipped without poisoning others. - de = _make_invalid_assembled_extractor(monkeypatch) - assert de._new_source is None - assert de.changes_made == [] - - -def test_invalid_assembled_source_skipped_verbose_false(monkeypatch): - # verbose=False: per-group compile-failure log suppressed (covers False branch). - de = _make_invalid_assembled_extractor(monkeypatch, verbose=False) - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — pyflakes new-undefined-names check -# --------------------------------------------------------------------------- - - -def _make_pyflakes_check_extractor(monkeypatch, verbose=True): - """Helper: extraction that passes compile() but pyflakes finds a new undefined - name.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic") as mock_anthropic, - patch( - "crispen.refactors.duplicate_extractor._pyflakes_new_undefined_names", - return_value={"mock_client"}, - ), - ): - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(x):\n pass\n", - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - ] - return DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - verbose=verbose, - extraction_retries=0, - llm_verify_retries=0, - ) - - -def test_pyflakes_check_skips_group_verbose(monkeypatch, capsys): - # Pyflakes finds a new undefined name → group is skipped (verbose path). - de = _make_pyflakes_check_extractor(monkeypatch, verbose=True) - assert de._new_source is None - assert ( - "undefined name(s) introduced by edit: mock_client" in capsys.readouterr().err - ) - - -def test_pyflakes_check_skips_group_verbose_false(monkeypatch): - # verbose=False: pyflakes failure is silent. - de = _make_pyflakes_check_extractor(monkeypatch, verbose=False) - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — _missing_free_vars check -# --------------------------------------------------------------------------- - - -def _make_missing_free_vars_extractor(monkeypatch, verbose=True): - """Helper: extraction that passes all earlier guards but _missing_free_vars - detects a free variable absent from the replacement.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic") as mock_anthropic, - patch( - "crispen.refactors.duplicate_extractor._missing_free_vars", - return_value={"new_source"}, - ), - ): - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(x):\n pass\n", - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - ] - return DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - verbose=verbose, - extraction_retries=0, - llm_verify_retries=0, - ) - - -def test_missing_free_vars_check_skips_group_verbose(monkeypatch, capsys): - # _missing_free_vars returns a non-empty set → group is rejected (verbose). - de = _make_missing_free_vars_extractor(monkeypatch, verbose=True) - assert de._new_source is None - assert ( - "free variable(s) from original block missing in replacement: new_source" - in capsys.readouterr().err - ) - - -def test_missing_free_vars_check_skips_group_verbose_false(monkeypatch): - # verbose=False: _missing_free_vars failure is silent. - de = _make_missing_free_vars_extractor(monkeypatch, verbose=False) - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — verification fails -# --------------------------------------------------------------------------- - - -def test_verify_fails_skipped(monkeypatch): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "helper", - "placement": "module_level", - "helper_source": "def helper(x:\n pass\n", # unclosed paren - "call_site_replacements": [ - "helper(data)\n", - "helper(data)\n", - ], - } - ), - ] - - de = DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - extraction_retries=0, - llm_verify_retries=0, - ) - - assert de._new_source is None - - -def test_verify_fails_skipped_verbose_false(monkeypatch): - # verbose=False covers the False branch of the new if-self.verbose guard. - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "helper", - "placement": "module_level", - "helper_source": "def helper(x:\n pass\n", # unclosed paren - "call_site_replacements": [ - "helper(data)\n", - "helper(data)\n", - ], - } - ), - ] - - de = DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - verbose=False, - extraction_retries=0, - llm_verify_retries=0, - ) - - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — replacement steals post-block line -# --------------------------------------------------------------------------- - -_POST_STEAL_SOURCE = textwrap.dedent( - """\ - def foo(): - x = compute(data) - y = transform(x) - z = finalize(y) - return z - - def bar(): - x = compute(data) - y = transform(x) - z = finalize(y) - logger.info("done") - """ -) -_POST_STEAL_RANGES = [(8, 10)] # overlaps bar's 3-statement block - - -def test_replacement_steals_post_block_skipped(monkeypatch): - """Replacement whose last line matches the post-block line is rejected.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "_do_work", - "placement": "module_level", - "helper_source": ( - "def _do_work(data):\n" - " x = compute(data)\n" - " y = transform(x)\n" - " z = finalize(y)\n" - ), - "call_site_replacements": [ - " _do_work(data)\n return z\n", # steals "return z" - " _do_work(data)\n", - ], - } - ), - ] - de = DuplicateExtractor( - _POST_STEAL_RANGES, - source=_POST_STEAL_SOURCE, - extraction_retries=0, - llm_verify_retries=0, - ) - - assert de._new_source is None - - -def test_replacement_steals_post_block_skipped_verbose_false(monkeypatch): - """verbose=False covers the False branch of the verbose guard.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "_do_work", - "placement": "module_level", - "helper_source": ( - "def _do_work(data):\n" - " x = compute(data)\n" - " y = transform(x)\n" - " z = finalize(y)\n" - ), - "call_site_replacements": [ - " _do_work(data)\n return z\n", # steals "return z" - " _do_work(data)\n", - ], - } - ), - ] - de = DuplicateExtractor( - _POST_STEAL_RANGES, - source=_POST_STEAL_SOURCE, - verbose=False, - extraction_retries=0, - llm_verify_retries=0, - ) - - assert de._new_source is None - - -def _make_new_attr_extractor(monkeypatch, verbose=True): - """Helper: LLM returns a helper that calls a method not in the original source.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "helper", - "placement": "module_level", - # helper calls .invented_method() — not present in _DUP_SOURCE - "helper_source": ( - "def helper(data):\n" " data.invented_method()\n" - ), - "call_site_replacements": [ - "helper(data)\n", - "helper(data)\n", - ], - } - ), - ] - return DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - verbose=verbose, - extraction_retries=0, - llm_verify_retries=0, - ) - - -def test_new_attribute_check_skips_group_verbose(monkeypatch, capsys): - de = _make_new_attr_extractor(monkeypatch, verbose=True) - assert de._new_source is None - assert "new attribute access" in capsys.readouterr().err - - -def test_new_attribute_check_skips_group_verbose_false(monkeypatch): - de = _make_new_attr_extractor(monkeypatch, verbose=False) - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — per-group call check -# --------------------------------------------------------------------------- - - -def _make_no_call_extractor(monkeypatch, verbose=True): - """Helper: LLM returns call replacements that don't call the helper function.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(data):\n pass\n", - # Call replacements don't reference _helper at all. - "call_site_replacements": [ - " pass\n", - " pass\n", - ], - } - ), - ] - return DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - verbose=verbose, - extraction_retries=0, - llm_verify_retries=0, - ) - - -def test_no_call_check_skips_group_verbose(monkeypatch, capsys): - de = _make_no_call_extractor(monkeypatch, verbose=True) - assert de._new_source is None - assert "not called in candidate output" in capsys.readouterr().err - - -def test_no_call_check_skips_group_verbose_false(monkeypatch): - de = _make_no_call_extractor(monkeypatch, verbose=False) - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — final combined call check -# --------------------------------------------------------------------------- - - -def _make_uncalled_in_combined_extractor(monkeypatch, verbose=True): - """Simulate: per-group call check passes, but combined output lacks the call. - - Achieved by patching _has_call_to: returns True for the per-group check - (first call) and False for the final combined check (second call). - """ - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic") as mock_anthropic, - patch( - "crispen.refactors.duplicate_extractor._has_call_to", - side_effect=[True, False], - ), - ): - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(data):\n pass\n", - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - return DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, verbose=verbose) - - -def test_uncalled_in_combined_drops_group_verbose(monkeypatch, capsys): - de = _make_uncalled_in_combined_extractor(monkeypatch, verbose=True) - assert de._new_source is None - assert "DROPPED" in capsys.readouterr().err - - -def test_uncalled_in_combined_drops_group_verbose_false(monkeypatch): - de = _make_uncalled_in_combined_extractor(monkeypatch, verbose=False) - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — helper defined in per-group candidate but missing from -# combined output (insertion blocked by overlapping blank-line replacement) -# --------------------------------------------------------------------------- - - -def _make_undefined_in_combined_extractor(monkeypatch, verbose=True): - """Simulate: per-group checks all pass but helper definition is absent from - the combined output (insertion edit blocked by overlap detector). - - Achieved by patching _has_funcdef: returns True for the per-group pyflakes - check (not called there directly, but we patch the combined check) and - False for the final combined check. - """ - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic") as mock_anthropic, - patch( - "crispen.refactors.duplicate_extractor._has_funcdef", - side_effect=[False], - ), - ): - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(data):\n pass\n", - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - return DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, verbose=verbose) - - -def test_undefined_helper_in_combined_drops_group_verbose(monkeypatch, capsys): - de = _make_undefined_in_combined_extractor(monkeypatch, verbose=True) - assert de._new_source is None - assert "not defined in combined output" in capsys.readouterr().err - - -def test_undefined_helper_in_combined_drops_group_verbose_false(monkeypatch): - de = _make_undefined_in_combined_extractor(monkeypatch, verbose=False) - assert de._new_source is None - - -def test_undefined_helper_in_combined_two_groups_one_dropped(monkeypatch): - """Two groups: first group's helper missing from combined, second kept. - - _has_funcdef returns [False, True]: first group dropped, second kept. - This exercises the all_edits.extend(g_edits) loop after the drop (line 2304). - """ - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic") as mock_anthropic, - patch( - "crispen.refactors.duplicate_extractor._has_funcdef", - side_effect=[False, True], - ), - ): - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper1", - "placement": "module_level", - "helper_source": "def _helper1():\n pass\n", - "call_site_replacements": [ - " _helper1()\n", - " _helper1()\n", - ], - } - ), - _make_verify_response(True, []), - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper2", - "placement": "module_level", - "helper_source": "def _helper2():\n pass\n", - "call_site_replacements": [ - " _helper2()\n", - " _helper2()\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor( - _TWO_PAIR_RANGES, source=_TWO_PAIR_SOURCE, verbose=False - ) - # First group dropped (undefined), second group kept → new source written - assert de._new_source is not None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — two groups, one dropped in combined check (line 1533) -# --------------------------------------------------------------------------- - -# Source with two structurally distinct duplicate pairs so _find_duplicate_groups -# returns two separate groups. The groups differ in argument count so that -# _ASTNormalizer produces different fingerprints for each group: -# group 1 (foo/bar): 3-stmt bodies using 2-argument calls → fingerprint A -# group 2 (baz/qux): 3-stmt bodies using 3-argument calls → fingerprint B -_TWO_PAIR_SOURCE = textwrap.dedent( - """\ - import os - - def foo(): - if debug: - pass - x = compute(data, config) - y = transform(x, scale) - z = finalize(y, mode) - - def bar(): - result = None - x = compute(data, config) - y = transform(x, scale) - z = finalize(y, mode) - - def baz(): - if debug: - pass - a = process(item, key, idx) - b = convert(a, fmt, enc) - c = export(b, path, opts) - - def qux(): - result = None - a = process(item, key, idx) - b = convert(a, fmt, enc) - c = export(b, path, opts) - """ -) -_TWO_PAIR_RANGES = [(4, 30)] # overlaps all duplicate sequences - - -def _make_two_group_drop_extractor(monkeypatch, verbose=True): - """Two extraction groups; the combined check drops one, exercising line 1533. - - _has_call_to is patched with side_effect=[True, True, True, False]: - - calls 1-2: per-group checks for each group → both pass - - call 3: combined check for first group → kept - - call 4: combined check for second group → dropped - After the drop, extraction_groups still has one entry, so the inner - ``for _, g_edits, _ in extraction_groups`` loop runs once (line 1533). - """ - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic") as mock_anthropic, - patch( - "crispen.refactors.duplicate_extractor._has_call_to", - side_effect=[True, True, True, False], - ), - ): - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - # Six LLM calls: veto+extract+verify for each of the two groups. - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper1", - "placement": "module_level", - "helper_source": "def _helper1():\n pass\n", - "call_site_replacements": [ - " _helper1()\n", - " _helper1()\n", - ], - } - ), - _make_verify_response(True, []), - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper2", - "placement": "module_level", - "helper_source": "def _helper2():\n pass\n", - "call_site_replacements": [ - " _helper2()\n", - " _helper2()\n", - ], - } - ), - _make_verify_response(True, []), - ] - return DuplicateExtractor( - _TWO_PAIR_RANGES, source=_TWO_PAIR_SOURCE, verbose=verbose - ) - - -def test_two_groups_one_dropped_combined_check(monkeypatch, capsys): - """One of two groups is dropped by the combined call check; the other is kept.""" - de = _make_two_group_drop_extractor(monkeypatch, verbose=True) - assert de._new_source is not None - assert "DROPPED" in capsys.readouterr().err - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — successful extraction at module level -# --------------------------------------------------------------------------- - - -def test_successful_extraction_module_level(monkeypatch, tmp_path): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - import os - - def foo(): - if debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - - de = DuplicateExtractor([(12, 14)], source=source) - - assert de._new_source is not None - assert "_helper" in de._new_source - assert len(de.changes_made) == 1 - assert "'_helper'" in de.changes_made[0] - assert de.get_rewritten_source() == de._new_source - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — staticmethod placement -# --------------------------------------------------------------------------- - - -def test_staticmethod_placement(monkeypatch): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - class MyClass: - def foo(self): - if self.debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(self): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper = " @staticmethod\n def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "_helper", - "placement": "staticmethod:MyClass", - "helper_source": helper, - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - - de = DuplicateExtractor([(11, 13)], source=source) - - assert de._new_source is not None - - -def test_staticmethod_placement_zero_indent_helper_auto_indented(monkeypatch): - """0-indent helper with staticmethod: placement is auto-indented into the class.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - class MyClass: - def foo(self): - if self.debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(self): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - # LLM generates a 0-indent (module-level) def even though it requested - # staticmethod:MyClass placement. Without auto-indent this would end the - # class body at the docstring, making foo/bar nested inside the helper. - helper_zero_indent = "def _helper(self, data):\n return compute(data)\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "_helper", - "placement": "staticmethod:MyClass", - "helper_source": helper_zero_indent, - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - - de = DuplicateExtractor([(11, 13)], source=source) - - assert de._new_source is not None - # foo and bar must remain real class methods, not nested inside the helper. - import ast as _ast - - tree = _ast.parse(de._new_source) - class_def = next( - n - for n in _ast.walk(tree) - if isinstance(n, _ast.ClassDef) and n.name == "MyClass" - ) - top_level_methods = { - n.name for n in class_def.body if isinstance(n, _ast.FunctionDef) - } - assert "foo" in top_level_methods - assert "bar" in top_level_methods - assert "_helper" in top_level_methods - - -def test_cross_class_duplicates_use_module_level_placement(monkeypatch): - """Duplicates in different classes must be extracted as module-level functions.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - class ClassA: - def foo(self): - if self.debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - class ClassB: - def bar(self): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - responses = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - mock_client.messages.create.side_effect = responses - de = DuplicateExtractor([(3, 5)], source=source) - - assert de._new_source is not None - # The extraction call prompt should tell the LLM to use module_level - extract_prompt = mock_client.messages.create.call_args_list[1][1]["messages"][0][ - "content" - ] - assert "module_level" in extract_prompt - assert "staticmethod" not in extract_prompt - - -def test_cross_class_staticmethod_placement_rejected(monkeypatch): - """LLM returning staticmethod placement for a cross-class group is rejected.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - class ClassA: - def foo(self): - if self.debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - class ClassB: - def bar(self): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - # First extraction attempt: LLM ignores prompt and returns staticmethod - # placement for a cross-class group → rejected; second attempt: correct. - responses = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "_helper", - "placement": "staticmethod:ClassA", - "helper_source": ( - " @staticmethod\n def _helper(data):\n pass\n" - ), - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - mock_client.messages.create.side_effect = responses - de = DuplicateExtractor([(3, 5)], source=source) - - assert de._new_source is not None - # Three LLM calls: veto + two extraction attempts - assert mock_client.messages.create.call_count == 4 - - -def test_cross_class_staticmethod_placement_rejected_non_verbose(monkeypatch): - """Defensive cross-class check works when verbose=False (no print side-effect).""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - class ClassA: - def foo(self): - if self.debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - class ClassB: - def bar(self): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - responses = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "_helper", - "placement": "staticmethod:ClassA", - "helper_source": ( - " @staticmethod\n def _helper(data):\n pass\n" - ), - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - mock_client.messages.create.side_effect = responses - de = DuplicateExtractor([(3, 5)], source=source, verbose=False) - - assert de._new_source is not None - - -def test_same_class_module_level_placement_rejected(monkeypatch): - """module_level placement with self.() call sites is rejected and retried.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - class MyClass: - def foo(self): - if self.debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(self): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper_module = "def _helper(data):\n pass\n" - helper_static = " @staticmethod\n def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - responses = [ - _make_veto_response(True), - # First attempt: module_level placement but call sites use self._helper() - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper_module, - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - # Second attempt: correct staticmethod placement - _make_extract_response( - { - "function_name": "_helper", - "placement": "staticmethod:MyClass", - "helper_source": helper_static, - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - mock_client.messages.create.side_effect = responses - de = DuplicateExtractor([(11, 13)], source=source) - - assert de._new_source is not None - # veto + two extraction attempts + verify - assert mock_client.messages.create.call_count == 4 - - -def test_same_class_module_level_placement_rejected_non_verbose(monkeypatch): - """Same inconsistency rejection works when verbose=False.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - class MyClass: - def foo(self): - if self.debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(self): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper_static = " @staticmethod\n def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - responses = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(data):\n pass\n", - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - _make_extract_response( - { - "function_name": "_helper", - "placement": "staticmethod:MyClass", - "helper_source": helper_static, - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - mock_client.messages.create.side_effect = responses - de = DuplicateExtractor([(11, 13)], source=source, verbose=False) - - assert de._new_source is not None - - -def test_cross_class_module_level_self_call_rejected(monkeypatch): - """module_level with self.() call sites in a cross-class group is rejected.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - class ClassA: - def foo(self): - if self.debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - class ClassB: - def bar(self): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - responses = [ - _make_veto_response(True), - # First attempt: module_level but call sites use self._helper() - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - # Second attempt: correct call sites - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - mock_client.messages.create.side_effect = responses - de = DuplicateExtractor([(3, 5)], source=source) - - assert de._new_source is not None - assert mock_client.messages.create.call_count == 4 - - -def test_staticmethod_wrong_class_rejected(monkeypatch): - """LLM naming the wrong class in staticmethod:X is rejected and retried.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - class ClassA: - def setup(self): - pass - - class ClassB: - def foo(self): - if self.debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(self): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper_static = " @staticmethod\n def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - responses = [ - _make_veto_response(True), - # First attempt: LLM names the wrong class - _make_extract_response( - { - "function_name": "_helper", - "placement": "staticmethod:ClassA", - "helper_source": helper_static, - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - # Second attempt: correct class name - _make_extract_response( - { - "function_name": "_helper", - "placement": "staticmethod:ClassB", - "helper_source": helper_static, - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - mock_client.messages.create.side_effect = responses - de = DuplicateExtractor([(14, 16)], source=source) - - assert de._new_source is not None - assert mock_client.messages.create.call_count == 4 - - -def test_staticmethod_wrong_class_rejected_non_verbose(monkeypatch): - """Wrong-class staticmethod rejection works when verbose=False.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - class ClassA: - def setup(self): - pass - - class ClassB: - def foo(self): - if self.debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(self): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper_static = " @staticmethod\n def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - responses = [ - _make_veto_response(True), - _make_extract_response( - { - "function_name": "_helper", - "placement": "staticmethod:ClassA", - "helper_source": helper_static, - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - _make_extract_response( - { - "function_name": "_helper", - "placement": "staticmethod:ClassB", - "helper_source": helper_static, - "call_site_replacements": [ - " self._helper(data)\n", - " self._helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - mock_client.messages.create.side_effect = responses - de = DuplicateExtractor([(14, 16)], source=source, verbose=False) - - assert de._new_source is not None - - -def test_sequence_collector_class_scope(): - """_SequenceCollector sets class_scope for sequences inside class methods.""" - import libcst as cst - from libcst.metadata import MetadataWrapper - - from crispen.refactors.duplicate_extractor import _SequenceCollector - - source = textwrap.dedent( - """\ - x = 1 - y = 2 - z = 3 - - class MyClass: - def method(self): - a = 1 - b = 2 - c = 3 - """ - ) - lines = source.splitlines(keepends=True) - tree = cst.parse_module(source) - collector = _SequenceCollector(lines, max_seq_len=8) - MetadataWrapper(tree).visit(collector) - - module_seqs = [s for s in collector.sequences if s.class_scope is None] - class_seqs = [s for s in collector.sequences if s.class_scope == "MyClass"] - assert module_seqs, "expected module-level sequences with class_scope=None" - assert class_seqs, "expected class-method sequences with class_scope='MyClass'" - - -# --------------------------------------------------------------------------- -# _llm_veto / _llm_extract: loop continues past non-matching content blocks -# --------------------------------------------------------------------------- - - -def _make_seq_info(start: int, end: int, src: str = "") -> _SeqInfo: - return _SeqInfo( - stmts=[], - start_line=start, - end_line=end, - scope="foo", - source=src, - fingerprint="", - ) - - -def test_llm_veto_skips_non_matching_blocks(monkeypatch): - from crispen.refactors.duplicate_extractor import _llm_veto - - client = MagicMock() - non_matching = MagicMock() - non_matching.type = "text" # not tool_use → if condition False - matching = MagicMock() - matching.type = "tool_use" - matching.name = "evaluate_duplicate" - matching.input = {"is_valid_duplicate": True, "reason": "same"} - response = MagicMock() - response.content = [non_matching, matching] - client.messages.create.return_value = response - - group = [_make_seq_info(1, 3), _make_seq_info(5, 7)] - is_valid, reason, _ = _llm_veto(client, group) - assert is_valid is True - - -def test_llm_extract_skips_non_matching_blocks(monkeypatch): - from crispen.refactors.duplicate_extractor import _llm_extract - - client = MagicMock() - non_matching = MagicMock() - non_matching.type = "text" # not tool_use → if condition False - matching = MagicMock() - matching.type = "tool_use" - matching.name = "extract_helper" - matching.input = { - "function_name": "helper", - "placement": "module_level", - "helper_source": "def helper(): pass\n", - "call_site_replacements": ["helper()\n"], - } - response = MagicMock() - response.content = [non_matching, matching] - client.messages.create.return_value = response - - group = [_make_seq_info(1, 3)] - result = _llm_extract(client, group, "a = 1\n") - assert result is not None - assert result["function_name"] == "helper" - - -def test_llm_veto_with_timing_out(monkeypatch): - """_llm_veto appends result to _timing_out when provided.""" - from crispen.refactors.duplicate_extractor import _llm_veto - - client = MagicMock() - client.messages.create.return_value = _make_veto_response(True, "ok") - group = [_make_seq_info(1, 3), _make_seq_info(5, 7)] - timing: list = [] - _llm_veto(client, group, _timing_out=timing) - assert len(timing) == 1 - assert timing[0].tool_input == {"is_valid_duplicate": True, "reason": "ok"} - - -def test_llm_veto_func_match_with_timing_out(): - """_llm_veto_func_match appends result to _timing_out when provided.""" - client = MagicMock() - client.messages.create.return_value = _make_veto_func_match_response(True, "same") - seq = _make_seq_info(7, 9, " x = 1\n") - func = _FunctionInfo( - name="fn", - source="def fn(): pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=[], - ) - timing: list = [] - _llm_veto_func_match(client, seq, func, "source", _timing_out=timing) - assert len(timing) == 1 - assert timing[0].tool_input["is_valid_duplicate"] is True - - -def test_llm_generate_call_with_timing_out(): - """_llm_generate_call appends result to _timing_out when provided.""" - client = MagicMock() - client.messages.create.return_value = _make_call_gen_response(" fn(data)\n") - seq = _make_seq_info(7, 9, " y = 1\n") - func = _FunctionInfo( - name="fn", - source="def fn(val):\n pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=["val"], - ) - timing: list = [] - result = _llm_generate_call(client, seq, func, "source", _timing_out=timing) - assert result == " fn(data)\n" - assert len(timing) == 1 - - -# --------------------------------------------------------------------------- -# engine integration: CrispenAPIError propagates -def test_verbose_false_suppresses_stderr(monkeypatch): - # verbose=False must take all four if-self.verbose False branches without printing. - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - import os - - def foo(): - if debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - - de = DuplicateExtractor([(12, 14)], source=source, verbose=False) - - assert de._new_source is not None - assert "_helper" in de._new_source - - -# --------------------------------------------------------------------------- - - -def test_engine_propagates_api_error(tmp_path, monkeypatch): - from crispen.config import CrispenConfig - from crispen.engine import run_engine - - f = tmp_path / "code.py" - f.write_text(_DUP_SOURCE, encoding="utf-8") - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - monkeypatch.delenv("MOONSHOT_API_KEY", raising=False) - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) - monkeypatch.setattr("crispen.engine.load_config", lambda: CrispenConfig()) - - with pytest.raises(CrispenAPIError): - list(run_engine({str(f): _DUP_RANGES})) - - -# --------------------------------------------------------------------------- -# cli integration: CrispenAPIError → sys.exit(1) -# --------------------------------------------------------------------------- - - -def test_cli_exits_on_api_error(tmp_path, monkeypatch): - import io - from crispen.cli import main - from crispen.config import CrispenConfig - - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - monkeypatch.delenv("MOONSHOT_API_KEY", raising=False) - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) - monkeypatch.setattr("crispen.cli.load_config", lambda: CrispenConfig()) - monkeypatch.setattr("crispen.engine.load_config", lambda: CrispenConfig()) - - # Write file so engine can read it - f = tmp_path / "dup.py" - f.write_text(_DUP_SOURCE, encoding="utf-8") - - diff = textwrap.dedent( - f"""\ - --- a/{f} - +++ b/{f} - @@ -10,3 +10,3 @@ - - x = compute(data) - + x = compute(data) - y = transform(x) - z = finalize(y) - """ - ) - monkeypatch.setattr("sys.stdin", io.StringIO(diff)) - - with pytest.raises(SystemExit) as exc_info: - main() - assert exc_info.value.code == 1 - - -# --------------------------------------------------------------------------- -# _run_with_timeout: hard wall-clock timeout -# --------------------------------------------------------------------------- - - -def test_run_with_timeout_fires_on_slow_func(): - import threading - - barrier = threading.Event() - try: - with pytest.raises(_ApiTimeout): - _run_with_timeout(barrier.wait, timeout=0.01) - finally: - barrier.set() # allow the daemon thread to exit cleanly - - -# --------------------------------------------------------------------------- -# _analyze: veto timeout → group skipped -# --------------------------------------------------------------------------- - - -def test_veto_timeout_skips_group(monkeypatch): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - side_effect=_ApiTimeout("veto timed out"), - ), - ): - de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) - assert de._new_source is None - assert de.changes_made == [] - - -# --------------------------------------------------------------------------- -# _analyze: extract timeout → group skipped -# --------------------------------------------------------------------------- - - -def test_extract_timeout_skips_group(monkeypatch): - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - # First call (veto) returns success; second call (extract) times out. - side_effects = [(True, "same logic", ""), _ApiTimeout("extract timed out")] - - def _mock_run(func, timeout, *args, **kwargs): - result = side_effects.pop(0) - if isinstance(result, BaseException): - raise result - return result - - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - side_effect=_mock_run, - ), - ): - de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# _llm_veto_func_match unit tests -# --------------------------------------------------------------------------- - - -def _make_veto_func_match_response(is_valid: bool, reason: str = "test") -> MagicMock: - block = MagicMock() - block.type = "tool_use" - block.name = "evaluate_duplicate" - block.input = {"is_valid_duplicate": is_valid, "reason": reason} - resp = MagicMock() - resp.content = [block] - return resp - - -def test_llm_veto_func_match_accepted(): - client = MagicMock() - client.messages.create.return_value = _make_veto_func_match_response( - True, "same op" - ) - seq = _make_seq_info(7, 9, " x = 1\n") - func = _FunctionInfo( - name="fn", - source="def fn(): pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=[], - ) - is_valid, reason, _ = _llm_veto_func_match(client, seq, func, "source") - assert is_valid is True - assert reason == "same op" - - -def test_llm_veto_func_match_rejected(): - client = MagicMock() - client.messages.create.return_value = _make_veto_func_match_response( - False, "different" - ) - seq = _make_seq_info(7, 9, " x = 1\n") - func = _FunctionInfo( - name="fn", - source="def fn(): pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=[], - ) - is_valid, reason, _ = _llm_veto_func_match(client, seq, func, "source") - assert is_valid is False - - -def test_llm_veto_func_match_api_error(): - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_anthropic.APIError = Exception - client = MagicMock() - client.messages.create.side_effect = Exception("api error") - seq = _make_seq_info(7, 9, " x = 1\n") - func = _FunctionInfo( - name="fn", - source="def fn(): pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=[], - ) - with pytest.raises(CrispenAPIError): - _llm_veto_func_match(client, seq, func, "source") - - -def test_llm_veto_func_match_skips_non_matching_blocks(): - """Non-matching content block is skipped; matching block still found.""" - client = MagicMock() - non_matching = MagicMock() - non_matching.type = "text" # not tool_use → False branch of the if - matching = MagicMock() - matching.type = "tool_use" - matching.name = "evaluate_duplicate" - matching.input = {"is_valid_duplicate": True, "reason": "same"} - response = MagicMock() - response.content = [non_matching, matching] - client.messages.create.return_value = response - seq = _make_seq_info(7, 9, " x = 1\n") - func = _FunctionInfo( - name="fn", - source="def fn(): pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=[], - ) - is_valid, reason, _ = _llm_veto_func_match(client, seq, func, "source") - assert is_valid is True - - -# --------------------------------------------------------------------------- -# _generate_no_arg_call unit tests -# --------------------------------------------------------------------------- - - -def test_generate_no_arg_call_indented(): - seq = _make_seq_info(7, 9, " x = 1\n y = 2\n") - func = _FunctionInfo( - name="setup", - source="def setup(): pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=[], - ) - result = _generate_no_arg_call(seq, func) - assert result == " setup()\n" - - -def test_generate_no_arg_call_no_indent(): - seq = _make_seq_info(1, 2, "x = 1\ny = 2\n") - func = _FunctionInfo( - name="setup", - source="def setup(): pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=[], - ) - result = _generate_no_arg_call(seq, func) - assert result == "setup()\n" - - -# --------------------------------------------------------------------------- -# _llm_generate_call unit tests -# --------------------------------------------------------------------------- - - -def _make_call_gen_response(replacement: str) -> MagicMock: - block = MagicMock() - block.type = "tool_use" - block.name = "generate_call" - block.input = {"replacement": replacement} - resp = MagicMock() - resp.content = [block] - return resp - - -def test_llm_generate_call_success(): - client = MagicMock() - client.messages.create.return_value = _make_call_gen_response( - " _process(data)\n" - ) - seq = _make_seq_info(7, 9, " y = 1\n") - func = _FunctionInfo( - name="_process", - source="def _process(val):\n pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=["val"], - ) - result = _llm_generate_call(client, seq, func, "source") - assert result == " _process(data)\n" - - -def test_llm_generate_call_api_error(): - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_anthropic.APIError = Exception - client = MagicMock() - client.messages.create.side_effect = Exception("api error") - seq = _make_seq_info(7, 9, " y = 1\n") - func = _FunctionInfo( - name="_process", - source="def _process(val):\n pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=["val"], - ) - with pytest.raises(CrispenAPIError): - _llm_generate_call(client, seq, func, "source") - - -def test_llm_generate_call_skips_non_matching_blocks(): - """Non-matching content block is skipped; matching block still found.""" - client = MagicMock() - non_matching = MagicMock() - non_matching.type = "text" # not tool_use → False branch of the if - matching = MagicMock() - matching.type = "tool_use" - matching.name = "generate_call" - matching.input = {"replacement": " _process(data)\n"} - response = MagicMock() - response.content = [non_matching, matching] - client.messages.create.return_value = response - seq = _make_seq_info(7, 9, " y = 1\n") - func = _FunctionInfo( - name="_process", - source="def _process(val):\n pass\n", - scope="", - body_source=" pass\n", - body_stmt_count=1, - params=["val"], - ) - result = _llm_generate_call(client, seq, func, "source") - assert result == " _process(data)\n" - - -# --------------------------------------------------------------------------- -# Function-match integration fixtures -# --------------------------------------------------------------------------- - -# _setup() has no params; called by main() → in func_body_fps. -# foo.body fingerprint == _setup.body fingerprint. -# Diff range (2, 9) covers both _setup.body (2-4) AND foo.body (7-9). -# _setup.body hits the func.name==seq.scope True branch (skipped). -# foo.body hits the False branch and proceeds to veto → replace. -_FUNC_MATCH_SOURCE = textwrap.dedent( - """\ - def _setup(): - x = compute(data) - y = transform(x) - z = finalize(y) - - def foo(): - x = compute(data) - y = transform(x) - z = finalize(y) - - def main(): - _setup() - """ -) -_FUNC_MATCH_RANGES = [(2, 9)] # covers _setup.body AND foo.body - -# _process(val) has one param; called by main() → in func_body_fps. -# foo.body fingerprint == _process.body fingerprint (names normalized). -# Diff range covers foo.body only. -_FUNC_MATCH_PARAM_SOURCE = textwrap.dedent( - """\ - def _process(val): - y = transform(val) - z = finalize(y) - return z - - def foo(): - y = transform(data) - z = finalize(y) - return z - - def main(): - _process(data) - """ -) -_FUNC_MATCH_PARAM_RANGES = [(6, 9)] # overlaps foo.body only - -# Source with a function-match AND an independent duplicate group. -# bar/baz use an if-else structure so no sub-window of their bodies matches -# _setup's 3-chained-assignment fingerprint. -_FUNC_MATCH_THEN_DUP_SOURCE = textwrap.dedent( - """\ - def _setup(): - x = compute(data) - y = transform(x) - z = finalize(y) - - def foo(): - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(): - a = setup(items) - if condition: - result = process(items) - else: - result = fallback(items) - store(result) - - def baz(): - if quick_check: - pass - if condition: - result = process(items) - else: - result = fallback(items) - store(result) - - def main(): - _setup() - """ -) -_FUNC_MATCH_THEN_DUP_RANGES = [(2, 30)] # covers foo, bar, baz bodies - - -# --------------------------------------------------------------------------- -# Function-match integration tests -# --------------------------------------------------------------------------- - - -def test_func_match_no_arg_replaces_body(monkeypatch): - """No-param module-level function: algorithmic replacement, no call-gen LLM.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - return_value=(True, "same operation", ""), - ), - ): - de = DuplicateExtractor( - _FUNC_MATCH_RANGES, source=_FUNC_MATCH_SOURCE, verbose=True - ) - assert de._new_source is not None - assert "_setup" in de.changes_made[0] - - -def test_func_match_verbose_false(monkeypatch): - """verbose=False covers all False branches of new if-self.verbose guards.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - return_value=(True, "same operation", ""), - ), - ): - de = DuplicateExtractor( - _FUNC_MATCH_RANGES, source=_FUNC_MATCH_SOURCE, verbose=False - ) - assert de._new_source is not None - - -def test_func_match_veto_rejects(monkeypatch): - """Veto rejects func match → no replacement.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - return_value=(False, "different", ""), - ), - ): - de = DuplicateExtractor( - _FUNC_MATCH_RANGES, source=_FUNC_MATCH_SOURCE, verbose=True - ) - assert de._new_source is None - - -def test_func_match_veto_timeout(monkeypatch): - """Veto times out → seq skipped; subsequent dup group also times out.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - side_effect=_ApiTimeout("timed out"), - ), - ): - de = DuplicateExtractor( - _FUNC_MATCH_RANGES, source=_FUNC_MATCH_SOURCE, verbose=True - ) - assert de._new_source is None - - -def test_func_match_verify_fails(monkeypatch): - """_verify_extraction returns False → func match skipped; dup group veto rejects.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - # Call 1: func match veto → (True, "ok") - # Call 2: dup group veto → (False, "different") so extract is never called - side_effects = [(True, "ok", ""), (False, "different", "")] - - def _mock_run(func, timeout, *args, **kwargs): - return side_effects.pop(0) - - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - side_effect=_mock_run, - ), - patch( - "crispen.refactors.duplicate_extractor._verify_extraction", - return_value=False, - ), - ): - de = DuplicateExtractor(_FUNC_MATCH_RANGES, source=_FUNC_MATCH_SOURCE) - assert de._new_source is None - - -def test_func_match_param_call_gen_success(monkeypatch): - """Parametrised function: LLM generates call expression successfully.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - # Call 1: func match veto → (True, "ok") - # Call 2: _llm_generate_call → replacement string - side_effects: list = [(True, "ok", ""), " _process(data)\n"] - - def _mock_run(func, timeout, *args, **kwargs): - return side_effects.pop(0) - - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - side_effect=_mock_run, - ), - ): - de = DuplicateExtractor( - _FUNC_MATCH_PARAM_RANGES, - source=_FUNC_MATCH_PARAM_SOURCE, - verbose=True, - ) - assert de._new_source is not None - - -def test_func_match_param_call_gen_timeout(monkeypatch): - """Call generation times out → seq skipped; dup group veto rejects.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - # Call 1: func match veto → (True, "ok") - # Call 2: _llm_generate_call → timeout - # Call 3: dup group veto → (False, "reject") so no extract called - side_effects: list = [ - (True, "ok", ""), - _ApiTimeout("timed out"), - (False, "reject", ""), - ] - - def _mock_run(func, timeout, *args, **kwargs): - result = side_effects.pop(0) - if isinstance(result, BaseException): - raise result - return result - - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - side_effect=_mock_run, - ), - ): - de = DuplicateExtractor( - _FUNC_MATCH_PARAM_RANGES, - source=_FUNC_MATCH_PARAM_SOURCE, - verbose=True, - ) - assert de._new_source is None - - -def test_func_match_then_dup_extract(monkeypatch): - """Func match succeeds; remaining dup group triggers standard veto/extract.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - extraction_dict = { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper():\n pass\n", - "call_site_replacements": [" _helper()\n", " _helper()\n"], - } - # Call 1: func match veto → (True, "ok", "") - # Call 2: dup group veto → (True, "ok", "") - # Call 3: dup group extract → extraction dict - # Call 4: LLM verify → (True, []) - side_effects: list = [ - (True, "ok", ""), - (True, "ok", ""), - extraction_dict, - (True, []), - ] - - def _mock_run(func, timeout, *args, **kwargs): - return side_effects.pop(0) - - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - side_effect=_mock_run, - ), - ): - de = DuplicateExtractor( - _FUNC_MATCH_THEN_DUP_RANGES, - source=_FUNC_MATCH_THEN_DUP_SOURCE, - ) - assert de._new_source is not None - # One func-match change + one dup-extract change - assert len(de.changes_made) == 2 - - -# --------------------------------------------------------------------------- -# match_functions=False -# --------------------------------------------------------------------------- - - -def test_match_functions_false_skips_func_match_pass(monkeypatch): - """match_functions=False: func-match veto never called even when match exists.""" - from crispen.refactors.duplicate_extractor import _llm_veto_func_match - - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - veto_func_match_called: list = [] - - def _mock_run_with_timeout(fn, timeout, *args, **kwargs): - if fn is _llm_veto_func_match: - veto_func_match_called.append(True) - # Reject any extraction-pass LLM call so no new source is produced. - return (False, "rejected", "") - - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - side_effect=_mock_run_with_timeout, - ), - ): - de = DuplicateExtractor( - _FUNC_MATCH_RANGES, - source=_FUNC_MATCH_SOURCE, - verbose=False, - match_functions=False, - ) - assert veto_func_match_called == [] - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# DuplicateExtractor — name collision guard -# --------------------------------------------------------------------------- - -# Source that already defines _helper AND has duplicate blocks. -_COLLISION_SOURCE = textwrap.dedent( - """\ - def _helper(x): - return x - - def foo(): - if debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def bar(): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ -) -_COLLISION_RANGES = [(12, 14)] # overlaps bar's duplicate block - - -def test_extraction_name_collision_skipped(monkeypatch, capsys): - # LLM returns function_name="_helper", which is already defined → skipped. - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(x, y):\n pass\n", - "call_site_replacements": [ - " _helper(data, x)\n", - " _helper(data, x)\n", - ], - } - ), - ] - de = DuplicateExtractor( - _COLLISION_RANGES, - source=_COLLISION_SOURCE, - verbose=True, - extraction_retries=0, - llm_verify_retries=0, - ) - - assert de._new_source is None - assert de.changes_made == [] - err = capsys.readouterr().err - assert "name collision" in err - assert "_helper" in err - - -def test_extraction_name_collision_silent(monkeypatch, capsys): - # Same collision, verbose=False → no stderr output. - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(x, y):\n pass\n", - "call_site_replacements": [ - " _helper(data, x)\n", - " _helper(data, x)\n", - ], - } - ), - ] - de = DuplicateExtractor( - _COLLISION_RANGES, - source=_COLLISION_SOURCE, - verbose=False, - extraction_retries=0, - llm_verify_retries=0, - ) - - assert de._new_source is None - assert de.changes_made == [] - err = capsys.readouterr().err - assert "name collision" not in err - - -# --------------------------------------------------------------------------- -# _strip_helper_docstring -# --------------------------------------------------------------------------- - - -def test_strip_helper_docstring_with_docstring(): - source = 'def _helper(x):\n """Strip me."""\n return x\n' - result = _strip_helper_docstring(source) - assert '"""Strip me."""' not in result - assert "return x" in result - - -def test_strip_helper_docstring_no_docstring(): - source = "def _helper(x):\n return x\n" - result = _strip_helper_docstring(source) - assert result == source - - -def test_strip_helper_docstring_parse_error(): - bad = "def f(:\n pass\n" - result = _strip_helper_docstring(bad) - assert result == bad - - -def test_strip_helper_docstring_non_function(): - source = "x = 1\n" - result = _strip_helper_docstring(source) - assert result == source - - -def test_strip_helper_docstring_docstring_only_body(): - # Function whose body is only a docstring — don't strip (would leave empty body). - source = 'def _helper():\n """Only doc."""\n' - result = _strip_helper_docstring(source) - assert '"""Only doc."""' in result - - -# --------------------------------------------------------------------------- -# _collect_ast_store_names -# --------------------------------------------------------------------------- - - -def test_collect_ast_store_names_simple_name(): - import ast - - node = ast.parse("x = 1").body[0].targets[0] - names: list = [] - _collect_ast_store_names(node, names) - assert names == ["x"] - - -def test_collect_ast_store_names_tuple(): - import ast - - node = ast.parse("a, b = 1, 2").body[0].targets[0] - names: list = [] - _collect_ast_store_names(node, names) - assert set(names) == {"a", "b"} - - -def test_collect_ast_store_names_nested_tuple(): - import ast - - node = ast.parse("(a, (b, c)) = x").body[0].targets[0] - names: list = [] - _collect_ast_store_names(node, names) - assert set(names) == {"a", "b", "c"} - - -def test_collect_ast_store_names_non_name_non_tuple_noop(): - # ast.Attribute target (e.g. self.x) → nothing collected. - import ast - - node = ast.parse("self.x = 1").body[0].targets[0] - names: list = [] - _collect_ast_store_names(node, names) - assert names == [] - - -# --------------------------------------------------------------------------- -# _scope_end_line -# --------------------------------------------------------------------------- - - -def _make_source_lines(src: str): - return src.splitlines(keepends=True) - - -def test_scope_end_line_module_returns_full_length(): - lines = _make_source_lines("x = 1\ny = 2\n") - assert _scope_end_line(lines, "", 1) == len(lines) - - -def test_scope_end_line_function_scope(): - src = "def foo():\n x = 1\n y = 2\n\ndef bar():\n z = 3\n" - lines = _make_source_lines(src) - # Block ends at line 2 (inside foo). foo ends at line 3. - assert _scope_end_line(lines, "foo", 2) == 3 - - -def test_scope_end_line_does_not_bleed_into_next_function(): - src = "def foo():\n x = 1\n\ndef bar():\n x = 2\n" - lines = _make_source_lines(src) - # Searching for `x` after line 2 should stop at end of foo (line 2), not - # reach bar where `x` also appears. - end = _scope_end_line(lines, "foo", 2) - assert end == 2 # foo ends at line 2; bar's x is excluded - - -def test_scope_end_line_picks_innermost_matching_scope(): - # Two functions named "inner" — one nested inside outer, one at module level. - src = ( - "def outer():\n" - " def inner():\n" - " a = 1\n" - " inner()\n" - "\n" - "def inner():\n" - " b = 2\n" - ) - lines = _make_source_lines(src) - # Block at line 3 is inside the nested inner (lines 2-3). That is the - # smallest matching span, so end_lineno == 3 is returned. - assert _scope_end_line(lines, "inner", 3) == 3 - - -def test_scope_end_line_class_scope(): - src = "class Foo:\n x = 1\n y = 2\n\nclass Bar:\n x = 3\n" - lines = _make_source_lines(src) - assert _scope_end_line(lines, "Foo", 2) == 3 - - -def test_scope_end_line_no_match_returns_full_length(): - src = "def foo():\n x = 1\n" - lines = _make_source_lines(src) - # Scope name doesn't match any definition. - assert _scope_end_line(lines, "bar", 1) == len(lines) - - -def test_scope_end_line_syntax_error_returns_full_length(): - lines = _make_source_lines("def (\n x = 1\n") - assert _scope_end_line(lines, "foo", 1) == len(lines) - - -# --------------------------------------------------------------------------- -# _replace_unused_in_target -# --------------------------------------------------------------------------- - - -def test_replace_unused_in_target_name_used(): - import ast - - target = ast.parse("result = 1").body[0].targets[0] - new_t, all_r, any_r = _replace_unused_in_target(target, "print(result)\n") - assert all_r is False and any_r is False - assert ast.unparse(new_t) == "result" - - -def test_replace_unused_in_target_name_unused(): - import ast - - target = ast.parse("result = 1").body[0].targets[0] - new_t, all_r, any_r = _replace_unused_in_target(target, "return None\n") - assert all_r is True and any_r is True - assert ast.unparse(new_t) == "_" - - -def test_replace_unused_in_target_tuple_all_unused(): - import ast - - target = ast.parse("a, b = 1").body[0].targets[0] - new_t, all_r, any_r = _replace_unused_in_target(target, "return None\n") - assert all_r is True and any_r is True - assert ast.unparse(new_t) == "(_, _)" - - -def test_replace_unused_in_target_tuple_some_unused(): - import ast - - target = ast.parse("a, b = 1").body[0].targets[0] - new_t, all_r, any_r = _replace_unused_in_target(target, "print(a)\n") - assert all_r is False and any_r is True - assert ast.unparse(new_t) == "(a, _)" - - -def test_replace_unused_in_target_tuple_all_used(): - import ast - - target = ast.parse("a, b = 1").body[0].targets[0] - new_t, all_r, any_r = _replace_unused_in_target(target, "print(a, b)\n") - assert all_r is False and any_r is False - - -def test_replace_unused_in_target_attribute_treated_as_used(): - import ast - - target = ast.parse("self.x = 1").body[0].targets[0] - new_t, all_r, any_r = _replace_unused_in_target(target, "return None\n") - assert all_r is False and any_r is False - - -# --------------------------------------------------------------------------- -# _strip_unused_call_assignments -# --------------------------------------------------------------------------- - - -def test_strip_unused_call_assignments_removes_unused_single(): - # `result` never appears after the block → assignment stripped. - replacement = " result = _helper(x, y)\n" - following = [" do_something()\n", " return z\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == " _helper(x, y)\n" - - -def test_strip_unused_call_assignments_keeps_used_single(): - # `result` is referenced after the block → assignment kept. - replacement = " result = _helper(x, y)\n" - following = [" print(result)\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == replacement - - -def test_strip_unused_call_assignments_removes_unused_tuple(): - # Both names unused after the block → assignment stripped entirely. - replacement = " a, b = _helper(x)\n" - following = [" return None\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == " _helper(x)\n" - - -def test_strip_unused_call_assignments_partial_tuple_replaces_with_underscore(): - # One name used, one unused → replace unused with _. - replacement = " a, b = _helper(x)\n" - following = [" print(a)\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == " (a, _) = _helper(x)\n" - - -def test_strip_unused_call_assignments_attribute_target_unchanged(): - # Target is an attribute (self.x = call()) → treated as used → left unchanged. - replacement = " self.result = _helper(x)\n" - following = [] - out = _strip_unused_call_assignments(replacement, following) - assert out == replacement - - -def test_strip_unused_call_assignments_non_call_rhs_unchanged(): - # RHS is not a Call → leave unchanged. - replacement = " result = x + y\n" - following = [" return None\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == replacement - - -def test_strip_unused_call_assignments_chained_all_unused_stripped(): - # Chained assignment where every name is unused → stripped to just the call. - replacement = " a = b = _helper(x)\n" - following = [" return None\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == " _helper(x)\n" - - -def test_strip_unused_call_assignments_chained_some_used_unchanged(): - # Chained assignment where one name is used → left unchanged. - replacement = " a = b = _helper(x)\n" - following = [" print(a)\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == replacement - - -def test_strip_unused_call_assignments_chained_no_names_unchanged(): - # Chained assignment whose targets yield no names (e.g. attributes) → unchanged. - replacement = " self.a = self.b = _helper(x)\n" - following = [] - out = _strip_unused_call_assignments(replacement, following) - assert out == replacement - - -def test_strip_unused_call_assignments_no_assignment_unchanged(): - # Replacement is already just a call → returned as-is. - replacement = " _helper(x, y)\n" - following = [" return None\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == replacement - - -def test_strip_unused_call_assignments_syntax_error_unchanged(): - # Unparseable replacement → returned unchanged. - replacement = " def (\n" - following = [] - out = _strip_unused_call_assignments(replacement, following) - assert out == replacement - - -def test_strip_unused_call_assignments_multiline_replacement(): - # Multi-statement replacement: only the unused assignment is stripped. - replacement = " result = _helper(x)\n do_other()\n" - following = [" return None\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == " _helper(x)\n do_other()\n" - - -def test_strip_unused_call_assignments_preserves_indentation(): - # Indentation of stripped replacement matches original block indent. - replacement = " result = _helper(x)\n" - following = [] - out = _strip_unused_call_assignments(replacement, following) - assert out == " _helper(x)\n" - - -def test_strip_unused_call_assignments_leading_blank_line(): - # Replacement with a blank leading line: indent is read from first content line. - replacement = "\n result = _helper(x)\n" - following = [] - out = _strip_unused_call_assignments(replacement, following) - assert out == "\n _helper(x)\n" - - -def test_strip_unused_call_assignments_await_unused_stripped(): - # `result = await _helper(x)` and `result` never used → strip assignment. - replacement = " result = await _helper(x)\n" - following = [" return None\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == " await _helper(x)\n" - - -def test_strip_unused_call_assignments_await_used_kept(): - # `result = await _helper(x)` and `result` is used → keep assignment. - replacement = " result = await _helper(x)\n" - following = [" print(result)\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == replacement - - -def test_strip_unused_call_assignments_await_tuple_unused_stripped(): - # `a, b = await _helper(x)` and neither name is used → strip assignment. - replacement = " a, b = await _helper(x)\n" - following = [" return None\n"] - out = _strip_unused_call_assignments(replacement, following) - assert out == " await _helper(x)\n" - - -def test_strip_unused_call_assignments_await_non_call_unchanged(): - # `result = await some_awaitable` (not a call) → left unchanged. - replacement = " result = await some_awaitable\n" - following = [] - out = _strip_unused_call_assignments(replacement, following) - assert out == replacement - - -# --------------------------------------------------------------------------- -# Re-strip with candidate following lines (end-to-end) -# --------------------------------------------------------------------------- - - -def test_restrip_drops_assignment_unused_only_after_all_call_sites_replaced( - monkeypatch, -): - # Regression: when two call sites reference the same variable name, the - # per-call-site strip (which uses original following lines) sees the name - # in the other call site's original block and keeps the assignment. After - # all replacements are assembled the variable is truly unused, so the - # re-strip pass must drop it. - # - # Source: test_f has two identical 2-line blocks. - # LLM returns: - # - call site 1 replacement: ``data = assert_error(result)`` - # - call site 2 replacement: ``assert_error(result2)`` (no assignment) - # After initial per-call-site strip, call site 1 keeps the assignment - # because "data" appears in the original following source (inside call - # site 2's original block). The re-strip must then drop it. - # Using function parameters avoids the SequenceCollector merging the - # assignment lines into the duplicate block. - # Use 3-statement blocks (weight=3 ≥ min_weight) so the SequenceCollector - # finds the duplicate group. Mirroring the real lever-mcp pattern: - # json.loads + two asserts. Both result and result2 are function - # parameters so the SequenceCollector cannot absorb the assignment lines - # into the duplicate block. - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - source = textwrap.dedent( - """\ - def test_f(result, result2): - rd = json.loads(result) - assert rd["value"] is None - assert "error" in rd - rd = json.loads(result2) - assert rd["value"] is None - assert "error" in rd - """ - ) - helper = textwrap.dedent( - """\ - def assert_error_result(result): - rd = json.loads(result) - assert rd["value"] is None - assert "error" in rd - """ - ) - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "identical blocks"), - _make_extract_response( - { - "function_name": "assert_error_result", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - # LLM assigns the return value at call site 1 … - " rd = assert_error_result(result)\n", - # … but not at call site 2 (helper returns None). - " assert_error_result(result2)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor([(2, 4), (5, 7)], source=source) - - assert de._new_source is not None - # The re-strip must have dropped the unused assignment at call site 1. - assert "rd = assert_error_result(result)" not in de._new_source - assert "assert_error_result(result)" in de._new_source - assert "assert_error_result(result2)" in de._new_source - - -# --------------------------------------------------------------------------- -# _SequenceCollector: min_weight parameter # --------------------------------------------------------------------------- - - -def test_sequence_collector_min_weight_filters_light_sequences(): - # A single assignment has weight 1. With min_weight=2 it should be excluded. - source = "def f():\n a = 1\n b = 2\n" - source_lines = source.splitlines(keepends=True) - tree = cst.parse_module(source) - from libcst.metadata import MetadataWrapper - - collector = _SequenceCollector(source_lines, max_seq_len=2, min_weight=2) - MetadataWrapper(tree).visit(collector) - # Single-statement sequences (weight=1) should be filtered out - single_stmt_seqs = [s for s in collector.sequences if len(s.stmts) == 1] - assert single_stmt_seqs == [] - - -# --------------------------------------------------------------------------- -# DuplicateExtractor: helper_docstrings config option -# --------------------------------------------------------------------------- - - -def test_duplicate_extractor_helper_docstrings_false_strips_docstring( - monkeypatch, capsys -): - """When helper_docstrings=False, the LLM-returned docstring is stripped.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_shared", - "placement": "module_level", - "helper_source": ( - "def _shared(data):\n" - ' """LLM added a docstring."""\n' - " pass\n" - ), - "call_site_replacements": [ - " _shared(data)\n", - " _shared(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor( - _DUP_RANGES, source=_DUP_SOURCE, verbose=False, helper_docstrings=False - ) - - assert de._new_source is not None - assert '"""LLM added a docstring."""' not in de._new_source - - -def test_duplicate_extractor_helper_docstrings_true_keeps_docstring( - monkeypatch, capsys -): - """When helper_docstrings=True, the LLM-returned docstring is preserved.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_shared", - "placement": "module_level", - "helper_source": ( - "def _shared(data):\n" - ' """Keep this docstring."""\n' - " pass\n" - ), - "call_site_replacements": [ - " _shared(data)\n", - " _shared(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor( - _DUP_RANGES, source=_DUP_SOURCE, verbose=False, helper_docstrings=True - ) - - assert de._new_source is not None - assert '"""Keep this docstring."""' in de._new_source - - -# --------------------------------------------------------------------------- -# DuplicateExtractor: model config option (passed to API) -# --------------------------------------------------------------------------- - - -def test_duplicate_extractor_custom_model_used(monkeypatch): - """Custom model string is passed to the Anthropic API.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.return_value = _make_veto_response(False, "no") - DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, model="claude-opus-4-6") - # Verify the custom model was passed - call_kwargs = mock_client.messages.create.call_args_list[0][1] - assert call_kwargs["model"] == "claude-opus-4-6" - - -# --------------------------------------------------------------------------- -# _seq_ends_with_return -# --------------------------------------------------------------------------- - - -def test_seq_ends_with_return_true(): - assert ( - _seq_ends_with_return(_make_seq_with_source(" x = 1\n return x\n")) - is True - ) - - -def test_seq_ends_with_return_false_no_return(): - assert ( - _seq_ends_with_return(_make_seq_with_source(" x = 1\n y = 2\n")) is False - ) - - -def test_seq_ends_with_return_syntax_error(): - assert _seq_ends_with_return(_make_seq_with_source(" (\n")) is False - - -def test_seq_ends_with_return_empty_body(): - # Pure whitespace → ast.parse produces an empty module body. - assert _seq_ends_with_return(_make_seq_with_source(" \n")) is False - - -def test_seq_ends_with_return_bare_return(): - # Bare `return` is equivalent to returning None — not flagged. - assert ( - _seq_ends_with_return(_make_seq_with_source(" x = 1\n return\n")) is False - ) - - -def test_seq_ends_with_return_return_none(): - # Explicit `return None` is also equivalent to implicit None — not flagged. - assert ( - _seq_ends_with_return(_make_seq_with_source(" x = 1\n return None\n")) - is False - ) - - -# --------------------------------------------------------------------------- -# _seq_source_contains_yield -# --------------------------------------------------------------------------- - - -def test_seq_source_contains_yield_async_with_yield(): - # The exact pattern that triggered the bug: async with ... as c: yield c - src = " async with Client(mcp) as c:\n yield c\n" - assert _seq_source_contains_yield(src) is True - - -def test_seq_source_contains_yield_plain_yield(): - assert _seq_source_contains_yield(" yield x\n") is True - - -def test_seq_source_contains_yield_from(): - assert _seq_source_contains_yield(" yield from something()\n") is True - - -def test_seq_source_contains_yield_no_yield(): - assert _seq_source_contains_yield(" x = 1\n y = 2\n") is False - - -def test_seq_source_contains_yield_nested_funcdef_not_counted(): - # yield inside a nested def must NOT trigger the guard - src = " def inner():\n yield 1\n" - assert _seq_source_contains_yield(src) is False - - -def test_seq_source_contains_yield_syntax_error(): - assert _seq_source_contains_yield(" (\n") is False - - -def test_collector_skips_yield_sequences(): - # Sequences whose source contains yield should never be collected. - source = textwrap.dedent( - """\ - async def make_client(): - x = setup() - async with Client(x) as c: - yield c - - async def make_client2(): - x = setup() - async with Client(x) as c: - yield c - """ - ) - seqs = _collect_sequences(source) - for seq in seqs: - assert not _seq_source_contains_yield(seq.source) - - -# --------------------------------------------------------------------------- -# _replacement_contains_return -# --------------------------------------------------------------------------- - - -def test_replacement_contains_return_true(): - assert _replacement_contains_return(" return x\n") is True - - -def test_replacement_contains_return_false(): - assert _replacement_contains_return(" _helper()\n") is False - - -def test_replacement_contains_return_syntax_error(): - # Unclosed paren inside the wrapper → SyntaxError → False. - assert _replacement_contains_return(" (\n") is False - - -# --------------------------------------------------------------------------- -# _replacement_steals_post_block_line -# --------------------------------------------------------------------------- - - -def _make_steal_seq(end_line: int) -> _SeqInfo: - return _SeqInfo( - stmts=[], start_line=1, end_line=end_line, scope="f", source="", fingerprint="" - ) - - -def test_replacement_steals_post_block_at_eof(): - # Block is the last line of the file — no post-block line exists. - source_lines = ["x = 1\n"] - seq = _make_steal_seq(1) # next_idx=1 >= len=1 → skip - assert not _replacement_steals_post_block_line( - [seq], ["y = helper()\n"], source_lines - ) - - -def test_replacement_steals_post_block_blank_after(): - # Post-block line is blank but there is a non-blank line further down. - # The check must scan past the blank to find the real post-block code. - source_lines = ["x = 1\n", "\n", "y = 2\n"] - seq = _make_steal_seq(1) # next_idx=1 → "\n" → scan → next_idx=2 → "y = 2" - assert _replacement_steals_post_block_line([seq], ["y = 2\n"], source_lines) - - -def test_replacement_steals_post_block_blank_after_no_match(): - # Blank after block, but replacement doesn't steal the non-blank post-block line. - source_lines = ["x = 1\n", "\n", "y = 2\n"] - seq = _make_steal_seq(1) - assert not _replacement_steals_post_block_line( - [seq], ["z = helper()\n"], source_lines - ) - - -def test_replacement_steals_post_block_all_blank_after(): - # Only blank lines follow the block — no real post-block line to steal. - source_lines = ["x = 1\n", "\n", "\n"] - seq = _make_steal_seq(1) - assert not _replacement_steals_post_block_line( - [seq], ["z = helper()\n"], source_lines - ) - - -def test_replacement_steals_post_block_no_match(): - # Replacement last line doesn't match post-block line. - source_lines = ["x = 1\n", "y = 2\n"] - seq = _make_steal_seq(1) # next_idx=1 → "y = 2" - assert not _replacement_steals_post_block_line( - [seq], ["z = helper()\n"], source_lines - ) - - -def test_replacement_steals_post_block_match(): - # Replacement last line matches post-block line → steal detected. - source_lines = ["x = 1\n", "y = 2\n"] - seq = _make_steal_seq(1) # next_idx=1 → "y = 2" - assert _replacement_steals_post_block_line( - [seq], ["z = helper()\ny = 2\n"], source_lines - ) - - -# --------------------------------------------------------------------------- -# _helper_imports_local_name -# --------------------------------------------------------------------------- - - -def test_helper_imports_local_name_true(): - helper = "def _h():\n import mock_client\n mock_client.run()\n" - original = "def test(mock_client):\n mock_client.run()\n" - assert _helper_imports_local_name(helper, original) is True - - -def test_helper_imports_local_name_already_imported_in_original(): - # mock_client is already a top-level import → not a local-only name. - helper = "def _h():\n import mock_client\n mock_client.run()\n" - original = "import mock_client\ndef test(x):\n mock_client.run()\n" - assert _helper_imports_local_name(helper, original) is False - - -def test_helper_imports_local_name_no_imports_in_helper(): - helper = "def _h():\n pass\n" - original = "def test(mock_client):\n pass\n" - assert _helper_imports_local_name(helper, original) is False - - -def test_helper_imports_local_name_syntax_error_helper(): - assert _helper_imports_local_name("def (:\n", "def test(x):\n pass\n") is False - - -def test_helper_imports_local_name_syntax_error_original(): - assert _helper_imports_local_name("def _h():\n import x\n", "(:\n") is False - - -def test_helper_imports_local_name_from_import_in_helper(): - # "from X import Y" in helper: the tracked name is "Y", not "X". - # If "Y" is a param in the original, it is flagged. - helper = "def _h():\n from pkg import mock_client\n mock_client.run()\n" - original = "def test(mock_client):\n mock_client.run()\n" - assert _helper_imports_local_name(helper, original) is True - - -def test_helper_imports_local_name_from_import_in_original(): - # Top-level "from pkg import something" in the original covers the branch - # in the orig_top_imports loop and prevents false-positive flagging. - helper = "def _h():\n import something\n something.run()\n" - original = "from pkg import something\ndef test(x):\n something.run()\n" - assert _helper_imports_local_name(helper, original) is False - - -def test_helper_imports_local_name_vararg(): - # Function with *args: vararg name tracked as potential local. - helper = "def _h():\n import args\n" - original = "def test(*args):\n pass\n" - assert _helper_imports_local_name(helper, original) is True - - -def test_helper_imports_local_name_kwarg(): - # Function with **kwargs: kwarg name tracked as potential local. - helper = "def _h():\n import kwargs\n" - original = "def test(**kwargs):\n pass\n" - assert _helper_imports_local_name(helper, original) is True - - -# --------------------------------------------------------------------------- -# Integration: block-ends-with-return guard -# --------------------------------------------------------------------------- - -_RETURN_BLOCK_SOURCE = textwrap.dedent( - """\ - def foo(): - if debug: - pass - x = compute(data) - y = transform(x) - return y - - def bar(): - result = None - x = compute(data) - y = transform(x) - return y - """ -) -_RETURN_BLOCK_RANGES = [(10, 12)] # overlaps bar's duplicate block (x/y/return lines) - - -def _make_return_block_extract_response(): - return _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": ( - "def _helper():\n" - " x = compute(data)\n" - " y = transform(x)\n" - " return y\n" - ), - # replacement drops the return — this is the bug being guarded - "call_site_replacements": [ - " _helper()\n", - " _helper()\n", - ], - } - ) - - -def test_block_ends_with_return_guard_skips(monkeypatch, capsys): - """Extraction rejected when block ends with return but replacement omits it.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_return_block_extract_response(), - ] - de = DuplicateExtractor( - _RETURN_BLOCK_RANGES, - source=_RETURN_BLOCK_SOURCE, - extraction_retries=0, - llm_verify_retries=0, - ) - assert de._new_source is None - assert "block ends with return but replacement omits it" in capsys.readouterr().err - - -def test_block_ends_with_return_guard_skips_silent(monkeypatch): - """verbose=False: extraction rejected with no stderr output.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_return_block_extract_response(), - ] - de = DuplicateExtractor( - _RETURN_BLOCK_RANGES, - source=_RETURN_BLOCK_SOURCE, - verbose=False, - extraction_retries=0, - llm_verify_retries=0, - ) - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# Integration: helper-imports-local-name guard +# DuplicateExtractor — final combined call check # --------------------------------------------------------------------------- -_PARAM_DUP_SOURCE = textwrap.dedent( - """\ - def test_a(mock_client): - if debug: - pass - x = compute(data) - y = transform(x) - z = finalize(y) - - def test_b(mock_client): - result = None - x = compute(data) - y = transform(x) - z = finalize(y) - """ -) -_PARAM_DUP_RANGES = [(10, 12)] # overlaps test_b's duplicate block - - -def _make_import_local_extract_response(): - return _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - # helper imports mock_client instead of taking it as a parameter - "helper_source": ( - "def _helper():\n" - " import mock_client\n" - " x = compute(data)\n" - " y = transform(x)\n" - " z = finalize(y)\n" - ), - "call_site_replacements": [ - " _helper()\n", - " _helper()\n", - ], - } - ) - - -def test_helper_imports_local_guard_skips(monkeypatch, capsys): - """Extraction rejected when helper imports a name that is a param in original.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_import_local_extract_response(), - ] - de = DuplicateExtractor( - _PARAM_DUP_RANGES, - source=_PARAM_DUP_SOURCE, - extraction_retries=0, - llm_verify_retries=0, - ) - assert de._new_source is None - assert "helper imports a name that is a parameter/local" in capsys.readouterr().err - - -def test_helper_imports_local_guard_skips_silent(monkeypatch): - """verbose=False: extraction rejected with no stderr output.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True), - _make_import_local_extract_response(), - ] - de = DuplicateExtractor( - _PARAM_DUP_RANGES, - source=_PARAM_DUP_SOURCE, - verbose=False, - extraction_retries=0, - llm_verify_retries=0, - ) - assert de._new_source is None - # --------------------------------------------------------------------------- -# _lift_and_dedup_imports +# DuplicateExtractor — helper defined in per-group candidate but missing from +# combined output (insertion blocked by overlapping blank-line replacement) # --------------------------------------------------------------------------- -def test_lift_and_dedup_no_changes_needed(): - src = "import os\nfrom typing import Any, Dict\nx = 1\n" - assert _lift_and_dedup_imports(src) == src - - -def test_lift_and_dedup_exact_from_duplicate(): - src = "from typing import Any\nfrom typing import Any\n" - assert _lift_and_dedup_imports(src) == "from typing import Any\n" - - -def test_lift_and_dedup_partial_overlap_adds_new_names(): - # Original F811 trigger: helper adds Any+Dict+Optional, file had Any+Dict - src = "from typing import Any, Dict\nfrom typing import Any, Dict, Optional\n" - assert _lift_and_dedup_imports(src) == "from typing import Any, Dict, Optional\n" - - -def test_lift_and_dedup_second_adds_only_new_names(): - src = "from typing import Any\nfrom typing import Optional\n" - assert _lift_and_dedup_imports(src) == "from typing import Any, Optional\n" - - -def test_lift_and_dedup_multiple_modules_independent(): - src = ( - "from typing import Any\n" - "from os.path import join\n" - "from typing import Dict\n" - "from os.path import exists\n" - ) - result = _lift_and_dedup_imports(src) - assert result == "from typing import Any, Dict\nfrom os.path import join, exists\n" - - -def test_lift_and_dedup_plain_import_deduped(): - # Unlike the old _dedup_from_imports, plain 'import X' dups are now removed - src = "import os\nimport os\n" - assert _lift_and_dedup_imports(src) == "import os\n" - - -def test_lift_and_dedup_skips_multiline_parens(): - src = "from typing import (\n Any,\n Dict,\n)\nfrom typing import Any\n" - # Paren form not matched; single-line import stands alone — no change - assert _lift_and_dedup_imports(src) == src - - -def test_lift_and_dedup_skips_wildcard(): - src = "from typing import *\nfrom typing import *\n" - assert _lift_and_dedup_imports(src) == src - - -def test_lift_and_dedup_skips_commented_import_line(): - # Inline comment prevents matching; both lines are left alone - src = "from typing import Any # noqa\nfrom typing import Any\n" - assert _lift_and_dedup_imports(src) == src - - -def test_lift_and_dedup_skips_indented_imports(): - # Indented imports (TYPE_CHECKING blocks, try/except, etc.) are not touched - src = " from typing import Any\n from typing import Dict\n" - assert _lift_and_dedup_imports(src) == src - - -def test_lift_and_dedup_empty_names_skipped(): - # Malformed import with no names: left unchanged - src = "from typing import ,\nfrom typing import ,\n" - assert _lift_and_dedup_imports(src) == src - - -def test_lift_and_dedup_non_import_lines_preserved(): - src = "from typing import Any\nx = 1\nfrom typing import Dict\ny = 2\n" - result = _lift_and_dedup_imports(src) - assert result == "from typing import Any, Dict\nx = 1\ny = 2\n" - - -def test_lift_and_dedup_lifts_misplaced_existing_module(): - # Helper inserted before second_fn lands after def first_fn → misplaced - # The import merges into the block and the misplaced copy is removed. - src = ( - "from typing import Any\n" - "\n" - "def first_fn():\n" - " pass\n" - "\n" - "from typing import Optional\n" # misplaced — helper preamble - "def _helper():\n" - " pass\n" - "\n" - "def second_fn():\n" - " pass\n" - ) - result = _lift_and_dedup_imports(src) - assert result == ( - "from typing import Any, Optional\n" - "\n" - "def first_fn():\n" - " pass\n" - "\n" - "def _helper():\n" - " pass\n" - "\n" - "def second_fn():\n" - " pass\n" - ) - - -def test_lift_and_dedup_lifts_misplaced_new_module(): - # Helper introduces a brand-new import mid-file → moved to after block. - src = ( - "from typing import Any\n" - "\n" - "def first_fn():\n" - " pass\n" - "\n" - "from collections import OrderedDict\n" # misplaced — new module - "def _helper():\n" - " pass\n" - "\n" - "def second_fn():\n" - " pass\n" - ) - result = _lift_and_dedup_imports(src) - assert result == ( - "from typing import Any\n" - "from collections import OrderedDict\n" # lifted after last block import - "\n" - "def first_fn():\n" - " pass\n" - "\n" - "def _helper():\n" - " pass\n" - "\n" - "def second_fn():\n" - " pass\n" - ) - - -def test_lift_and_dedup_lifts_misplaced_plain_import_new_module(): - # Covers: misplaced plain 'import X' (i >= first_funcdef_idx branch) and - # the new_plain_modules emission path inside _emit_new_imports. - src = ( - "from typing import Any\n" - "\n" - "def first_fn():\n" - " pass\n" - "\n" - "import os\n" # misplaced plain import — new module - "def _helper():\n" - " pass\n" - ) - result = _lift_and_dedup_imports(src) - assert result == ( - "from typing import Any\n" - "import os\n" # lifted after last block import - "\n" - "def first_fn():\n" - " pass\n" - "\n" - "def _helper():\n" - " pass\n" - ) - - -def test_lift_and_dedup_sorts_new_imports_by_pep8_section(): - # New lifted imports are sorted future→stdlib→third-party→local regardless - # of the order they were encountered. - src = ( - "from typing import Any\n" # block stdlib import - "\n" - "def first_fn():\n" - " pass\n" - "\n" - "import requests\n" # misplaced third-party - "from collections import OrderedDict\n" # misplaced stdlib - "def _helper():\n" - " pass\n" - ) - result = _lift_and_dedup_imports(src) - assert result == ( - "from typing import Any\n" - "from collections import OrderedDict\n" # stdlib before third-party - "import requests\n" - "\n" - "def first_fn():\n" - " pass\n" - "\n" - "def _helper():\n" - " pass\n" - ) - - -def test_lift_and_dedup_blank_lines_in_block_dropped(): - # Blank lines between import lines in the block are removed when the block - # is rebuilt — covers the blank-line-dropping branch in pass 5. - src = ( - "import os\n" - "\n" # blank between block imports → dropped on rebuild - "from typing import Any\n" - "from typing import Dict\n" # duplicate module → merged - "x = 1\n" - ) - result = _lift_and_dedup_imports(src) - # PEP 8 sort: both are stdlib (group 1); from_order precedes plain_order in - # all_final_imports so stable sort keeps 'from typing' before 'import os'. - assert result == ("from typing import Any, Dict\n" "import os\n" "x = 1\n") - - -def test_lift_and_dedup_no_block_imports_inserts_before_first_funcdef(): - # File has no imports at all; helper adds one mid-file → moved to very top. - src = ( - "def first_fn():\n" - " pass\n" - "\n" - "from collections import OrderedDict\n" # misplaced - "def _helper():\n" - " pass\n" - "\n" - "def second_fn():\n" - " pass\n" - ) - result = _lift_and_dedup_imports(src) - assert result == ( - "from collections import OrderedDict\n" # inserted before first funcdef - "def first_fn():\n" - " pass\n" - "\n" - "def _helper():\n" - " pass\n" - "\n" - "def second_fn():\n" - " pass\n" - ) - - # --------------------------------------------------------------------------- -# New behaviour: veto notes, algorithmic retry, LLM verify step +# _llm_veto / _llm_extract: loop continues past non-matching content blocks # --------------------------------------------------------------------------- -def _make_veto_response_with_notes( - is_valid: bool, reason: str, notes: str -) -> MagicMock: - block = MagicMock() - block.type = "tool_use" - block.name = "evaluate_duplicate" - block.input = { - "is_valid_duplicate": is_valid, - "reason": reason, - "extraction_notes": notes, - } - resp = MagicMock() - resp.content = [block] - return resp - - -def test_veto_notes_passed_to_extract(monkeypatch): - """extraction_notes from veto are forwarded to the extract prompt.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response_with_notes(True, "same logic", "watch out for x"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE) - - assert de._new_source is not None - extract_call = mock_client.messages.create.call_args_list[1] - extract_prompt = extract_call.kwargs["messages"][0]["content"] - assert "watch out for x" in extract_prompt - - -def test_extraction_retry_on_alg_failure_verbose(monkeypatch, capsys): - """First extract has wrong call count -> retry -> second succeeds. verbose=True.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [" _helper(data)\n"], # wrong count - } - ), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor( - _DUP_RANGES, source=_DUP_SOURCE, verbose=True, extraction_retries=1 - ) - - assert de._new_source is not None - err = capsys.readouterr().err - assert "retrying" in err - - -def test_extraction_retry_on_alg_failure_silent(monkeypatch): - """First extract has wrong call count -> retry -> second succeeds. verbose=False.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [" _helper(data)\n"], # wrong count - } - ), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor( - _DUP_RANGES, source=_DUP_SOURCE, verbose=False, extraction_retries=1 - ) - - assert de._new_source is not None - - -def test_llm_verify_timeout_verbose(monkeypatch, capsys): - """Verify times out (verbose=True) -> extraction is accepted and logged.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - from crispen.refactors.duplicate_extractor import _llm_verify_extraction - - extraction_dict = { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(data):\n pass\n", - "call_site_replacements": [" _helper(data)\n", " _helper(data)\n"], - } - side_effects: list = [(True, "same logic", ""), extraction_dict] - - def _mock_run(func, timeout, *args, **kwargs): - if func is _llm_verify_extraction: - raise _ApiTimeout("verify timed out") - return side_effects.pop(0) - - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - side_effect=_mock_run, - ), - ): - de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, verbose=True) - - assert de._new_source is not None - err = capsys.readouterr().err - assert "verify timed out" in err - - -def test_llm_verify_rejects_then_retries_verbose(monkeypatch, capsys): - """Verify rejects first attempt; retry extract passes. verbose=True.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(False, ["wrong variable name"]), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor( - _DUP_RANGES, source=_DUP_SOURCE, verbose=True, llm_verify_retries=1 - ) - - assert de._new_source is not None - err = capsys.readouterr().err - assert "REJECTED" in err - assert "wrong variable name" in err - assert "retrying" in err - - -def test_llm_verify_rejects_then_retries_silent(monkeypatch): - """Verify rejects first attempt; retry extract passes. verbose=False.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(False, ["wrong variable name"]), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor( - _DUP_RANGES, source=_DUP_SOURCE, verbose=False, llm_verify_retries=1 - ) - - assert de._new_source is not None - - -def test_llm_verify_exhausted_skips_group(monkeypatch): - """All verify attempts fail -> group skipped.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(False, ["issue"]), - ] - de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, llm_verify_retries=0) - - assert de._new_source is None - - -def test_llm_verify_timeout_silent(monkeypatch): - """Verify times out (verbose=False) -> extraction is accepted silently.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - from crispen.refactors.duplicate_extractor import _llm_verify_extraction - - extraction_dict = { - "function_name": "_helper", - "placement": "module_level", - "helper_source": "def _helper(data):\n pass\n", - "call_site_replacements": [" _helper(data)\n", " _helper(data)\n"], - } - side_effects: list = [(True, "same logic", ""), extraction_dict] - - def _mock_run(func, timeout, *args, **kwargs): - if func is _llm_verify_extraction: - raise _ApiTimeout("verify timed out") - return side_effects.pop(0) - - with ( - patch("crispen.llm_client.anthropic.Anthropic"), - patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", - side_effect=_mock_run, - ), - ): - de = DuplicateExtractor(_DUP_RANGES, source=_DUP_SOURCE, verbose=False) - - assert de._new_source is not None - - -# --------------------------------------------------------------------------- -# Underscore enforcement on extracted helper names # --------------------------------------------------------------------------- -def test_llm_name_without_underscore_is_prefixed(monkeypatch): - """LLM returns a name without a leading '_'; extractor prepends one.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "helper", # no underscore - "placement": "module_level", - "helper_source": "def helper(data):\n pass\n", - "call_site_replacements": [ - " helper(data)\n", - " helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - de = DuplicateExtractor( - _DUP_RANGES, source=_DUP_SOURCE, extraction_retries=0, llm_verify_retries=0 - ) - - assert de._new_source is not None - assert "def _helper(" in de._new_source - assert "def helper(" not in de._new_source - assert "_helper(data)" in de._new_source - - # --------------------------------------------------------------------------- # _would_create_proxy_wrappers # --------------------------------------------------------------------------- -def _make_proxy_seq(stmts_count: int, scope: str, class_scope=None) -> _SeqInfo: - """Build a _SeqInfo with a synthetic stmts list of the given length.""" - return _SeqInfo( - stmts=[None] * stmts_count, # type: ignore[list-item] - start_line=1, - end_line=stmts_count, - scope=scope, - source="", - fingerprint="", - class_scope=class_scope, - ) - - -def _make_proxy_func( - name: str, body_stmt_count: int, scope: str = "" -) -> _FunctionInfo: - return _FunctionInfo( - name=name, - source=f"def {name}(): pass\n", - scope=scope, - body_source=" pass\n", - body_stmt_count=body_stmt_count, - params=[], - ) - - -def test_would_create_proxy_wrappers_false_single_full_body(): - """Single-member group where the seq covers the entire function body. - - All members are proxies, so extraction is still worthwhile → False. - """ - seq = _make_proxy_seq(3, scope="foo") - func = _make_proxy_func("foo", body_stmt_count=3, scope="") - assert _would_create_proxy_wrappers([seq], [func]) is False - - -def test_would_create_proxy_wrappers_false_all_full_bodies(): - """All group members cover entire function bodies → False. - - When every member becomes a proxy the group is all-or-nothing: extracting - a shared helper is still worthwhile, so the guard should not block it. - """ - seq1 = _make_proxy_seq(3, scope="process", class_scope="ClassA") - seq2 = _make_proxy_seq(3, scope="process", class_scope="ClassB") - func1 = _make_proxy_func("process", body_stmt_count=3, scope="ClassA") - func2 = _make_proxy_func("process", body_stmt_count=3, scope="ClassB") - assert _would_create_proxy_wrappers([seq1, seq2], [func1, func2]) is False - - -def test_would_create_proxy_wrappers_false_partial_body(): - """A seq that covers only part of a function body → False.""" - seq = _make_proxy_seq(2, scope="foo") - func = _make_proxy_func("foo", body_stmt_count=4, scope="") - assert _would_create_proxy_wrappers([seq], [func]) is False - - -def test_would_create_proxy_wrappers_false_module_scope(): - """A seq at module scope (not inside a function) is never a proxy → False.""" - seq = _make_proxy_seq(3, scope="") - func = _make_proxy_func("foo", body_stmt_count=3, scope="") - assert _would_create_proxy_wrappers([seq], [func]) is False - - -def test_would_create_proxy_wrappers_false_no_matching_func(): - """No function with matching name → False.""" - seq = _make_proxy_seq(3, scope="foo") - func = _make_proxy_func("bar", body_stmt_count=3, scope="") - assert _would_create_proxy_wrappers([seq], [func]) is False - - -def test_would_create_proxy_wrappers_false_scope_mismatch(): - """Seq in class method but func is module-level with same name → False.""" - seq = _make_proxy_seq(3, scope="foo", class_scope="MyClass") - func = _make_proxy_func("foo", body_stmt_count=3, scope="") - assert _would_create_proxy_wrappers([seq], [func]) is False - - -def test_would_create_proxy_wrappers_group_with_one_proxy(): - """A group with multiple seqs, one of which covers an entire body → True.""" - seq_partial = _make_proxy_seq(2, scope="foo") - seq_full = _make_proxy_seq(3, scope="bar") - func_foo = _make_proxy_func("foo", body_stmt_count=5, scope="") - func_bar = _make_proxy_func("bar", body_stmt_count=3, scope="") - assert ( - _would_create_proxy_wrappers([seq_partial, seq_full], [func_foo, func_bar]) - is True - ) - - # DuplicateExtractor: proxy wrapper guard skips groups without LLM calls - - -_PROXY_SOURCE = textwrap.dedent( - """\ - def foo(): - setup = prepare(data) - x = compute(data) - y = transform(x) - z = finalize(y) - return setup, z - - def bar(): - x = compute(data) - y = transform(x) - z = finalize(y) - """ -) -# overlaps foo: foo has 5 stmts but duplicate block is only 3 of them (not a proxy); -# bar has 3 stmts = its entire body (would become a proxy) → mixed → guard fires. -_PROXY_RANGES = [(1, 11)] - - -def test_proxy_wrapper_guard_skips_group_verbose(monkeypatch, capsys): - """Groups that would leave a function as a trivial proxy are skipped (verbose).""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic.Anthropic"): - de = DuplicateExtractor(_PROXY_RANGES, source=_PROXY_SOURCE, verbose=True) - - assert de._new_source is None - captured = capsys.readouterr() - assert "trivial proxy wrapper" in captured.err - - -def test_proxy_wrapper_guard_skips_group_silent(monkeypatch): - """Groups that would leave a trivial proxy are skipped with verbose=False.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic.Anthropic"): - de = DuplicateExtractor(_PROXY_RANGES, source=_PROXY_SOURCE, verbose=False) - - assert de._new_source is None - - -# --------------------------------------------------------------------------- -# _timing_out parameter: ensures timing list is populated by helper functions -# --------------------------------------------------------------------------- - - -def test_llm_verify_extraction_with_timing_out(): - """_llm_verify_extraction appends result to _timing_out when provided.""" - from crispen.refactors.duplicate_extractor import _llm_verify_extraction - - client = MagicMock() - client.messages.create.return_value = _make_verify_response(True, []) - group = [_make_seq_info(1, 3), _make_seq_info(5, 7)] - timing: list = [] - is_correct, issues = _llm_verify_extraction( - client, - group, - "def _helper(): pass\n", - [" _helper()\n", " _helper()\n"], - "a = 1\nb = 2\n", - _timing_out=timing, - ) - assert is_correct is True - assert len(timing) == 1 - - -def test_llm_verify_extraction_without_timing_out(): - """_llm_verify_extraction works correctly when _timing_out is None.""" - from crispen.refactors.duplicate_extractor import _llm_verify_extraction - - client = MagicMock() - client.messages.create.return_value = _make_verify_response(True, []) - group = [_make_seq_info(1, 3), _make_seq_info(5, 7)] - is_correct, issues = _llm_verify_extraction( - client, - group, - "def _helper(): pass\n", - [" _helper()\n", " _helper()\n"], - "a = 1\nb = 2\n", - ) - assert is_correct is True - assert issues == [] - - -def test_func_match_veto_timing_recorded(monkeypatch): - """When func-match veto accepts, record_llm_call is invoked for the veto call.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - # veto accepts → call-gen runs (func has params) → done (no dup groups) - mock_client.messages.create.side_effect = [ - _make_veto_func_match_response(True, "same"), - _make_call_gen_response(" _process(data)\n"), - ] - de = DuplicateExtractor( - _FUNC_MATCH_PARAM_RANGES, - source=_FUNC_MATCH_PARAM_SOURCE, - ) - # record_llm_call ran for veto (the timing branch was True) - assert de.stats.llm_elapsed_by_category.get("veto", 0) >= 0 - - -def test_func_match_call_gen_timing_recorded(monkeypatch): - """When func-match call-gen runs, record_llm_call is invoked for the edit call.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - # veto accepts → call-gen runs → done (no dup groups) - mock_client.messages.create.side_effect = [ - _make_veto_func_match_response(True, "same"), - _make_call_gen_response(" _process(data)\n"), - ] - de = DuplicateExtractor( - _FUNC_MATCH_PARAM_RANGES, - source=_FUNC_MATCH_PARAM_SOURCE, - ) - assert de.stats.llm_edit_calls >= 1 - - -def test_func_match_veto_detailed_timing_suffix(monkeypatch, capsys): - """timing='detailed' prints timing suffix after func-match veto result.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_func_match_response(True, "same"), - _make_call_gen_response(" _process(data)\n"), - ] - DuplicateExtractor( - _FUNC_MATCH_PARAM_RANGES, - source=_FUNC_MATCH_PARAM_SOURCE, - verbose=True, - timing="detailed", - ) - err = capsys.readouterr().err - assert "ACCEPTED" in err - assert "[" in err # timing suffix present - - -def test_func_match_replacement_detailed_timing_suffix(monkeypatch, capsys): - """timing='detailed' prints timing suffix after func-match replacement line.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_func_match_response(True, "same"), - _make_call_gen_response(" _process(data)\n"), - ] - DuplicateExtractor( - _FUNC_MATCH_PARAM_RANGES, - source=_FUNC_MATCH_PARAM_SOURCE, - verbose=True, - timing="detailed", - ) - err = capsys.readouterr().err - assert "replacing" in err - assert "[" in err # timing suffix on replacement line - - -def test_dup_veto_detailed_timing_suffix(monkeypatch, capsys): - """timing='detailed' prints timing suffix after dup-group veto result.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.return_value = _make_veto_response( - False, "different logic" - ) - DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - verbose=True, - timing="detailed", - ) - err = capsys.readouterr().err - assert "VETOED" in err - assert "[" in err # timing suffix present - - -def test_verify_detailed_timing_suffix(monkeypatch, capsys): - """timing='detailed' prints timing suffix after verify result.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - verbose=True, - timing="detailed", - ) - err = capsys.readouterr().err - assert "verify ACCEPTED" in err - assert "[" in err # timing suffix present - - -def test_extraction_detailed_timing_message(monkeypatch, capsys): - """timing='detailed' prints extraction timing message after extraction call.""" - monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") - helper = "def _helper(data):\n pass\n" - with patch("crispen.llm_client.anthropic") as mock_anthropic: - mock_client = MagicMock() - mock_anthropic.Anthropic.return_value = mock_client - mock_anthropic.APIError = Exception - mock_client.messages.create.side_effect = [ - _make_veto_response(True, "same logic"), - _make_extract_response( - { - "function_name": "_helper", - "placement": "module_level", - "helper_source": helper, - "call_site_replacements": [ - " _helper(data)\n", - " _helper(data)\n", - ], - } - ), - _make_verify_response(True, []), - ] - DuplicateExtractor( - _DUP_RANGES, - source=_DUP_SOURCE, - verbose=True, - timing="detailed", - ) - err = capsys.readouterr().err - assert "→ extraction [" in err diff --git a/tests/test_engine.py b/tests/test_engine.py index c9c6279..be562a2 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,4030 +1,7 @@ -"""Tests for the engine module.""" - -import textwrap -import threading -from unittest.mock import patch - -import libcst as cst -import pytest - -from crispen.config import CrispenConfig -from crispen.engine import ( - _EXCLUDED_DIR_NAMES, - _LLM_REFACTOR_KEYS, - _add_fl_context, - _apply_tuple_dataclass, - _blocked_private_scopes, - _build_alias_map, - _build_patch_map, - _categorize_into_stats, - _collect_assignment_names, - _collect_code_referenced_names, - _collect_imported_names, - _collect_top_level_names, - _compute_qname, - _file_to_module, - _find_outside_callers, - _find_repo_root, - _has_callers_outside_ranges, - _module_path_for_file, - _patch_inline_imports_after_test_deletion, - _redirect_inline_module_imports, - _should_run, - _visit_with_timeout, - run_engine, -) - -from crispen.errors import CrispenAPIError -from crispen.file_limiter.runner import FileLimiterResult -from crispen.refactors.base import Refactor -from crispen.stats import RunStats - - -def _run(changed): - return list(run_engine(changed, config=CrispenConfig(min_tuple_size=3))) - - -# --------------------------------------------------------------------------- -# Config header printed to stderr -# --------------------------------------------------------------------------- - - -def test_config_header_printed_when_llm_refactors_enabled(tmp_path, capsys): - f = tmp_path / "simple.py" - f.write_text("x = 1\n", encoding="utf-8") - list(run_engine({str(f): [(1, 1)]}, config=CrispenConfig())) - err = capsys.readouterr().err - assert "--- crispen ---" in err - assert "provider:" in err - assert "model:" in err - - -def test_config_header_suppressed_when_all_llm_refactors_disabled(tmp_path, capsys): - f = tmp_path / "simple.py" - f.write_text("x = 1\n", encoding="utf-8") - cfg = CrispenConfig(disabled_refactors=list(_LLM_REFACTOR_KEYS)) - list(run_engine({str(f): [(1, 1)]}, config=cfg)) - assert "--- crispen ---" not in capsys.readouterr().err - - -def test_config_header_suppressed_when_changed_empty(capsys): - list(run_engine({}, config=CrispenConfig())) - assert "--- crispen ---" not in capsys.readouterr().err - - -# --------------------------------------------------------------------------- -# File not found -# --------------------------------------------------------------------------- - - -def test_skip_missing_file(tmp_path): - missing = str(tmp_path / "nonexistent.py") - msgs = _run({missing: [(1, 10)]}) - assert len(msgs) == 1 - assert "SKIP" in msgs[0] - assert "file not found" in msgs[0] - - -# --------------------------------------------------------------------------- -# No changes produced -# --------------------------------------------------------------------------- - - -def test_no_changes_no_messages(tmp_path): - f = tmp_path / "simple.py" - f.write_text("x = 1\n", encoding="utf-8") - msgs = _run({str(f): [(1, 1)]}) - assert msgs == [] - - -# --------------------------------------------------------------------------- -# Successful transformation — writes file back -# --------------------------------------------------------------------------- - - -def test_applies_refactor_and_writes(tmp_path): - source = textwrap.dedent( - """\ - if not x: - a() - else: - b() - """ - ) - f = tmp_path / "code.py" - f.write_text(source, encoding="utf-8") - msgs = _run({str(f): [(1, 4)]}) - assert any("IfNotElse" in m for m in msgs) - assert "if x:" in f.read_text(encoding="utf-8") - - -def test_rewritten_source_used_when_available(tmp_path): - """get_rewritten_source() is preferred over new_tree.code when non-None.""" - rewritten = "x = 999 # rewritten\n" - - class _RewritingRefactor(Refactor): - @classmethod - def name(cls): - return "Rewriter" - - def get_rewritten_source(self): - return rewritten - - def get_changes(self): - return ["Rewriter: rewrote the file"] - - f = tmp_path / "code.py" - f.write_text("x = 1\n", encoding="utf-8") - with patch("crispen.engine._REFACTORS", [_RewritingRefactor]): - msgs = _run({str(f): [(1, 1)]}) - assert any("Rewriter" in m for m in msgs) - assert f.read_text(encoding="utf-8") == rewritten - - -# --------------------------------------------------------------------------- -# Parse error -# --------------------------------------------------------------------------- - - -def test_skip_parse_error(tmp_path): - f = tmp_path / "bad.py" - f.write_text("def f(:\n pass\n", encoding="utf-8") - msgs = _run({str(f): [(1, 2)]}) - assert any("parse error" in m for m in msgs) - - -# --------------------------------------------------------------------------- -# Transform error -# --------------------------------------------------------------------------- - - -class _RaisingTransformer(Refactor): - """A Refactor subclass that always raises during tree traversal.""" - - @classmethod - def name(cls): - return "RaisingRefactor" - - def leave_Module(self, original_node, updated_node): - raise RuntimeError("intentional transform error") - - -def test_skip_transform_error(tmp_path): - f = tmp_path / "code.py" - f.write_text("x = 1\n", encoding="utf-8") - with patch("crispen.engine._REFACTORS", [_RaisingTransformer]): - msgs = _run({str(f): [(1, 1)]}) - assert any("transform error" in m for m in msgs) - - -# --------------------------------------------------------------------------- -# CrispenAPIError propagates through engine -# --------------------------------------------------------------------------- - - -class _CrispenApiErrorRefactor(Refactor): - @classmethod - def name(cls): - return "ApiErrorRefactor" - - def leave_Module(self, original_node, updated_node): - raise CrispenAPIError("test api error") - - -def test_crispen_api_error_propagates(tmp_path): - f = tmp_path / "code.py" - f.write_text("x = 1\n", encoding="utf-8") - with patch("crispen.engine._REFACTORS", [_CrispenApiErrorRefactor]): - with pytest.raises(CrispenAPIError): - list(run_engine({str(f): [(1, 1)]})) - - -# --------------------------------------------------------------------------- -# TupleDataclass transform error: td is None (covers 290->293 branch) -# --------------------------------------------------------------------------- - - -def test_tuple_dataclass_transform_error_handled(tmp_path): - f = tmp_path / "code.py" - f.write_text("x = 1\n", encoding="utf-8") - - class _FailingTD: - def __init__(self, *a, **kw): - raise RuntimeError("simulated TupleDataclass failure") - - with patch("crispen.engine.TupleDataclass", _FailingTD): - msgs = _run({str(f): [(1, 1)]}) - assert any("TupleDataclass" in m and "transform error" in m for m in msgs) - - -# --------------------------------------------------------------------------- -# _find_repo_root -# --------------------------------------------------------------------------- - - -def test_find_repo_root_finds_git(tmp_path): - (tmp_path / ".git").mkdir() - subdir = tmp_path / "src" - subdir.mkdir() - f = subdir / "code.py" - f.write_text("x = 1\n") - root = _find_repo_root({str(f): [(1, 1)]}) - assert root == str(tmp_path) - - -def test_find_repo_root_not_found(tmp_path): - f = tmp_path / "code.py" - f.write_text("x = 1\n") - root = _find_repo_root({str(f): [(1, 1)]}) - assert root is None - - -# --------------------------------------------------------------------------- -# _file_to_module and _compute_qname -# --------------------------------------------------------------------------- - - -def test_file_to_module_regular_file(tmp_path): - f = tmp_path / "mypkg" / "service.py" - f.parent.mkdir() - f.write_text("x = 1\n") - assert _file_to_module(str(tmp_path), str(f)) == "mypkg.service" - - -def test_file_to_module_init(tmp_path): - f = tmp_path / "mypkg" / "__init__.py" - f.parent.mkdir() - f.write_text("") - assert _file_to_module(str(tmp_path), str(f)) == "mypkg" - - -def test_compute_qname(tmp_path): - f = tmp_path / "pkg" / "mod.py" - f.parent.mkdir() - f.write_text("") - assert _compute_qname(str(tmp_path), str(f), "my_func") == "pkg.mod.my_func" - - -# --------------------------------------------------------------------------- -# _build_alias_map -# --------------------------------------------------------------------------- - - -def test_build_alias_map_identity_only(tmp_path): - # No __init__.py in tmp_path → only identity mapping returned. - alias_map = _build_alias_map(str(tmp_path), {"a.b.func"}) - assert alias_map == {"a.b.func": "a.b.func"} - - -def test_build_alias_map_with_reexport(tmp_path): - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("from mypkg.service import get_user\n") - alias_map = _build_alias_map(str(tmp_path), {"mypkg.service.get_user"}) - assert "mypkg.get_user" in alias_map - assert alias_map["mypkg.get_user"] == "mypkg.service.get_user" - - -def test_build_alias_map_star_import_skipped(tmp_path): - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("from mypkg.service import *\n") - alias_map = _build_alias_map(str(tmp_path), {"mypkg.service.get_user"}) - # Star import does not create an alias - assert "mypkg.get_user" not in alias_map - - -def test_build_alias_map_ambiguous_name_skipped(tmp_path): - # Two canonical qnames share the same function name → alias is ambiguous. - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("from mypkg.service import get_user\n") - alias_map = _build_alias_map( - str(tmp_path), - {"mypkg.service.get_user", "mypkg.other.get_user"}, - ) - # Ambiguous: skip adding the alias - assert "mypkg.get_user" not in alias_map - - -def test_build_alias_map_invalid_init_skipped(tmp_path): - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("def f(:\n pass\n") # invalid Python - alias_map = _build_alias_map(str(tmp_path), {"mypkg.service.get_user"}) - # Gracefully skips the unreadable __init__.py - assert alias_map == {"mypkg.service.get_user": "mypkg.service.get_user"} - - -# --------------------------------------------------------------------------- -# _find_outside_callers -# --------------------------------------------------------------------------- - - -def test_find_outside_callers_empty_qnames(tmp_path): - result = _find_outside_callers(str(tmp_path), set(), set()) - assert result == set() - - -def test_find_outside_callers_no_outside_py_files(tmp_path): - f = tmp_path / "code.py" - f.write_text("x = 1\n") - result = _find_outside_callers(str(tmp_path), {"pkg.func"}, {str(f.resolve())}) - # All .py files are in the diff → nothing to scan outside - assert result == set() - - -def test_find_outside_callers_finds_caller(tmp_path): - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("") - service = pkg / "service.py" - service.write_text("def get_user():\n return (1, 2, 3)\n") - outside = tmp_path / "outside.py" - outside.write_text("from mypkg.service import get_user\nget_user()\n") - - qname = "mypkg.service.get_user" - diff_files = {str(service.resolve())} - result = _find_outside_callers(str(tmp_path), {qname}, diff_files) - assert qname in result - - -def test_find_outside_callers_no_match(tmp_path): - outside = tmp_path / "other.py" - outside.write_text("x = 1\n") - qname = "mypkg.service.get_user" - result = _find_outside_callers(str(tmp_path), {qname}, set()) - assert qname not in result - - -# --------------------------------------------------------------------------- -# Cross-file integration: public function + caller both in diff -# --------------------------------------------------------------------------- - - -def _make_pkg(root, name): - pkg = root / name - pkg.mkdir(exist_ok=True) - (pkg / "__init__.py").write_text("", encoding="utf-8") - return pkg - - -def test_cross_file_transforms_public_func_and_caller(tmp_path): - pkg = _make_pkg(tmp_path, "mypkg") - - service = pkg / "service.py" - service.write_text( - "def get_user():\n return (name, age, score)\n", encoding="utf-8" - ) - - api = pkg / "api.py" - api.write_text( - "from mypkg.service import get_user\n" - "def main():\n" - " a, b, c = get_user()\n", - encoding="utf-8", - ) - - changed = {str(service): [(1, 2)], str(api): [(1, 4)]} - msgs = list( - run_engine( - changed, - _repo_root=str(tmp_path), - config=CrispenConfig(min_tuple_size=3), - ) - ) - - assert any("TupleDataclass" in m for m in msgs) - assert any("CallerUpdater" in m for m in msgs) - - service_text = service.read_text(encoding="utf-8") - assert "GetUserResult(" in service_text - assert "@dataclass" in service_text - - api_text = api.read_text(encoding="utf-8") - assert "_ = get_user()" in api_text - assert "_.name" in api_text - - -# --------------------------------------------------------------------------- -# Cross-file: outside callers block the transform -# --------------------------------------------------------------------------- - - -def test_cross_file_skips_when_outside_caller_exists(tmp_path): - pkg = _make_pkg(tmp_path, "mypkg") - - service = pkg / "service.py" - service.write_text( - "def get_user():\n return (name, age, score)\n", encoding="utf-8" - ) - - # This file is NOT in the diff but calls get_user. - outside = pkg / "outside.py" - outside.write_text( - "from mypkg.service import get_user\na, b, c = get_user()\n", - encoding="utf-8", - ) - - changed = {str(service): [(1, 2)]} - msgs = list( - run_engine( - changed, - _repo_root=str(tmp_path), - config=CrispenConfig(min_tuple_size=3), - ) - ) - - assert any("callers exist outside the diff" in m for m in msgs) - assert "return (name, age, score)" in service.read_text(encoding="utf-8") - - -# --------------------------------------------------------------------------- -# _build_alias_map: skip non-SimpleStatementLine and non-ImportFrom branches -# --------------------------------------------------------------------------- - - -def test_build_alias_map_skips_compound_statement(tmp_path): - # A function definition is a compound statement, not SimpleStatementLine (line 76). - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("def helper():\n pass\n") - alias_map = _build_alias_map(str(tmp_path), {"mypkg.service.get_user"}) - assert alias_map == {"mypkg.service.get_user": "mypkg.service.get_user"} - - -def test_build_alias_map_skips_non_import_in_simple_stmt(tmp_path): - # An assignment in SimpleStatementLine is not ImportFrom (line 79). - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("__version__ = '1.0'\n") - alias_map = _build_alias_map(str(tmp_path), {"mypkg.service.get_user"}) - assert alias_map == {"mypkg.service.get_user": "mypkg.service.get_user"} - - -# --------------------------------------------------------------------------- -# _find_outside_callers: call resolves but qname not in targets (118->117) -# --------------------------------------------------------------------------- - - -def test_find_outside_callers_call_qname_not_target(tmp_path): - # outside file calls other_func (resolves to mypkg.other.other_func), - # but target is mypkg.service.get_user → hits the 118->117 branch. - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("") - (pkg / "other.py").write_text("def other_func(): pass\n") - caller = tmp_path / "caller.py" - caller.write_text("from mypkg.other import other_func\nother_func()\n") - - result = _find_outside_callers(str(tmp_path), {"mypkg.service.get_user"}, set()) - assert "mypkg.service.get_user" not in result - - -# --------------------------------------------------------------------------- -# _find_outside_callers: FullRepoManager build failure (143-145) -# --------------------------------------------------------------------------- - - -def test_find_outside_callers_manager_build_fails(tmp_path): - (tmp_path / "other.py").write_text("x = 1\n") - with patch("crispen.engine.FullRepoManager", side_effect=RuntimeError("fail")): - result = _find_outside_callers(str(tmp_path), {"some.func"}, set()) - # Conservative: all target qnames are blocked. - assert result == {"some.func"} - - -# --------------------------------------------------------------------------- -# _find_outside_callers: wrapper.get_metadata_wrapper_for_path fails (154-155) -# --------------------------------------------------------------------------- - - -def test_find_outside_callers_wrapper_fails(tmp_path): - (tmp_path / "other.py").write_text("x = 1\n") - with patch("crispen.engine.FullRepoManager") as MockFRM: - MockFRM.return_value.get_metadata_wrapper_for_path.side_effect = RuntimeError( - "fail" - ) - result = _find_outside_callers(str(tmp_path), {"some.func"}, set()) - assert result == set() - - -# --------------------------------------------------------------------------- -# _apply_tuple_dataclass: parse error path (175-176) -# --------------------------------------------------------------------------- - - -def test_apply_tuple_dataclass_parse_error(): - bad_source = "def f(:\n pass\n" - source_out, msgs, td = _apply_tuple_dataclass( - "fake.py", [(1, 10)], bad_source, False, set() - ) - assert any("parse error" in m for m in msgs) - assert td is None - assert source_out == bad_source - - -# --------------------------------------------------------------------------- -# _apply_tuple_dataclass: CrispenAPIError propagates (188) -# --------------------------------------------------------------------------- - - -def test_apply_tuple_dataclass_crispen_api_error(): - with patch("crispen.engine.MetadataWrapper") as MockWrapper: - MockWrapper.return_value.visit.side_effect = CrispenAPIError("test api error") - with pytest.raises(CrispenAPIError): - _apply_tuple_dataclass("f.py", [(1, 1)], "x = 1\n", False, set()) - - -# --------------------------------------------------------------------------- -# Phase 2: file not under repo_root → ValueError caught (314-315, 317->406) -# --------------------------------------------------------------------------- - - -def test_cross_file_file_not_under_repo_root(tmp_path): - # repo_root is a separate directory; changed file is not under it. - repo_root = tmp_path / "repo" - repo_root.mkdir() - f = tmp_path / "code.py" - f.write_text("def public_func():\n return (1, 2, 3)\n", encoding="utf-8") - # _compute_qname raises ValueError → all_candidates stays empty → 317->406 branch. - msgs = list( - run_engine( - {str(f): [(1, 2)]}, - _repo_root=str(repo_root), - config=CrispenConfig(min_tuple_size=3), - ) - ) - assert not any("callers" in m for m in msgs) - - -# --------------------------------------------------------------------------- -# Phase 2: repo_root set but no public candidates (317->406) -# --------------------------------------------------------------------------- - - -def test_no_public_candidates_with_repo_root(tmp_path): - f = tmp_path / "code.py" - f.write_text("x = 1\n", encoding="utf-8") - msgs = list(run_engine({str(f): [(1, 1)]}, _repo_root=str(tmp_path))) - assert msgs == [] - - -# --------------------------------------------------------------------------- -# Phase 2: one approved, one blocked → alias loop hits non-approved (349->348) -# --------------------------------------------------------------------------- - - -def test_cross_file_one_approved_one_blocked(tmp_path): - pkg = _make_pkg(tmp_path, "mypkg") - - a = pkg / "a.py" - a.write_text("def approved_func():\n return (1, 2, 3)\n", encoding="utf-8") - - b = pkg / "b.py" - b.write_text("def blocked_func():\n return (1, 2, 3)\n", encoding="utf-8") - - # outside.py calls blocked_func and is NOT in the diff. - outside = pkg / "outside.py" - outside.write_text( - "from mypkg.b import blocked_func\nblocked_func()\n", encoding="utf-8" - ) - - changed = {str(a): [(1, 2)], str(b): [(1, 2)]} - msgs = list( - run_engine( - changed, _repo_root=str(tmp_path), config=CrispenConfig(min_tuple_size=3) - ) - ) - - # blocked_func is skipped; its identity entry in alias_map hits the 349->348 branch. - assert any( - "blocked_func" in m and "callers exist outside the diff" in m for m in msgs - ) - # approved_func is transformed. - assert any("TupleDataclass" in m for m in msgs) - - -# --------------------------------------------------------------------------- -# CallerUpdater pass: file not under repo_root → ValueError (369-370) -# --------------------------------------------------------------------------- - - -def test_cross_file_caller_updater_file_not_under_repo_root(tmp_path): - subdir = tmp_path / "repo" - subdir.mkdir() - (subdir / "__init__.py").write_text("") - - inside = subdir / "service.py" - inside.write_text("def approved():\n return (1, 2, 3)\n", encoding="utf-8") - - # This file is in the diff but outside repo_root (subdir). - outside_code = tmp_path / "outside_code.py" - outside_code.write_text("x = 1\n", encoding="utf-8") - - changed = {str(inside): [(1, 2)], str(outside_code): [(1, 1)]} - # No crash; outside_code.py's _file_to_module raises ValueError → continue. - list( - run_engine( - changed, _repo_root=str(subdir), config=CrispenConfig(min_tuple_size=3) - ) - ) - - -# --------------------------------------------------------------------------- -# CallerUpdater pass: parse error on state["source"] (374-375) -# --------------------------------------------------------------------------- - - -def test_cross_file_caller_updater_parse_error(tmp_path): - pkg = _make_pkg(tmp_path, "mypkg") - - service = pkg / "service.py" - service.write_text("def approved():\n return (1, 2, 3)\n", encoding="utf-8") - - changed = {str(service): [(1, 2)]} - - original_parse = cst.parse_module - - def patched_parse(source): - # After Phase 2 transforms the source, it will contain "@dataclass". - # Fail on that call to exercise the 374-375 parse-error branch. - if "@dataclass" in source: - raise cst.ParserSyntaxError( - "fake error", lines=("@dataclass",), raw_line=0, raw_column=0 - ) - return original_parse(source) - - with patch("crispen.engine.cst.parse_module", patched_parse): - # Should not crash; CallerUpdater pass silently continues. - list( - run_engine( - changed, - _repo_root=str(tmp_path), - config=CrispenConfig(min_tuple_size=3), - ) - ) - - -# --------------------------------------------------------------------------- -# CallerUpdater pass: CallerUpdater constructor raises (387-388) -# --------------------------------------------------------------------------- - - -def test_cross_file_caller_updater_raises(tmp_path): - pkg = _make_pkg(tmp_path, "mypkg") - - service = pkg / "service.py" - service.write_text("def approved():\n return (1, 2, 3)\n", encoding="utf-8") - - changed = {str(service): [(1, 2)]} - - with patch("crispen.engine.CallerUpdater", side_effect=RuntimeError("fail")): - # Should not crash; the exception is caught. - list( - run_engine( - changed, - _repo_root=str(tmp_path), - config=CrispenConfig(min_tuple_size=3), - ) - ) - - -# --------------------------------------------------------------------------- -# Cross-file: __init__.py alias is recognised -# --------------------------------------------------------------------------- - - -def test_cross_file_init_alias_detected_as_outside_caller(tmp_path): - pkg = _make_pkg(tmp_path, "mypkg") - - # Re-export get_user through __init__.py - (pkg / "__init__.py").write_text( - "from mypkg.service import get_user\n", encoding="utf-8" - ) - - service = pkg / "service.py" - service.write_text( - "def get_user():\n return (name, age, score)\n", encoding="utf-8" - ) - - # Outside file imports via the alias (pkg.get_user) - outside = tmp_path / "outside.py" - outside.write_text( - "from mypkg import get_user\na, b, c = get_user()\n", encoding="utf-8" - ) - - changed = {str(service): [(1, 2)]} - msgs = list( - run_engine( - changed, _repo_root=str(tmp_path), config=CrispenConfig(min_tuple_size=3) - ) - ) - - assert any("callers exist outside the diff" in m for m in msgs) - - -# --------------------------------------------------------------------------- -# _visit_with_timeout -# --------------------------------------------------------------------------- - - -def test_visit_with_timeout_completes(): - """Fast visit completes within timeout → returns True.""" - from unittest.mock import MagicMock - - wrapper = MagicMock() - finder = MagicMock() - assert _visit_with_timeout(wrapper, finder, 5.0) is True - wrapper.visit.assert_called_once_with(finder) - - -def test_visit_with_timeout_fires(): - """Slow visit that never completes → returns False after timeout.""" - block = threading.Event() - - class _HangWrapper: - def visit(self, finder): - block.wait() # blocks until released - - result = _visit_with_timeout(_HangWrapper(), object(), 0.01) - block.set() # unblock the daemon thread for cleanup - assert result is False - - -def test_find_outside_callers_scope_analysis_timeout(tmp_path): - """When _visit_with_timeout times out, all target qnames are blocked.""" - (tmp_path / "other.py").write_text("x = 1\n") - with patch("crispen.engine._visit_with_timeout", return_value=False): - result = _find_outside_callers(str(tmp_path), {"some.func"}, set()) - assert result == {"some.func"} - - -def test_find_outside_callers_deadline_expired(tmp_path): - """Total budget already exhausted before any file is visited: all blocked.""" - (tmp_path / "other.py").write_text("x = 1\n") - # A negative timeout makes the deadline fall in the past immediately. - with patch("crispen.engine._SCOPE_ANALYSIS_TIMEOUT", -1): - result = _find_outside_callers(str(tmp_path), {"some.func"}, set()) - assert result == {"some.func"} - - -# --------------------------------------------------------------------------- -# _find_outside_callers: excluded directory names are not scanned -# --------------------------------------------------------------------------- - - -def test_find_outside_callers_excludes_venv_dirs(tmp_path): - """Files inside excluded directories (.venv, __pycache__, etc.) are skipped.""" - for dirname in _EXCLUDED_DIR_NAMES: - excluded = tmp_path / dirname / "lib" - excluded.mkdir(parents=True, exist_ok=True) - (excluded / "pkg.py").write_text( - "from mypkg.service import get_user\nget_user()\n" - ) - # Even though each excluded dir has a caller, none should be counted. - result = _find_outside_callers(str(tmp_path), {"mypkg.service.get_user"}, set()) - assert "mypkg.service.get_user" not in result - - -# --------------------------------------------------------------------------- -# Phase 1 private-function caller updates -# --------------------------------------------------------------------------- - - -def _make_phase1_pkg(root): - """Helper: return a tmp_path containing a package for Phase 1 tests.""" - pkg = root / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - return pkg - - -def test_phase1_private_caller_updated(tmp_path): - """Private function callers in the same file are updated after Phase 1.""" - source = textwrap.dedent( - """\ - def _make_result(): - return (1, 2, 3) - - def use_it(): - a, b, c = _make_result() - """ - ) - f = tmp_path / "code.py" - f.write_text(source, encoding="utf-8") - msgs = _run({str(f): [(1, 100)]}) - result = f.read_text(encoding="utf-8") - assert "_ = _make_result()" in result - assert any("CallerUpdater" in m for m in msgs) - - -def test_phase1_private_no_callers_no_caller_updater_msg(tmp_path): - """Private transform with no callers produces no CallerUpdater message.""" - source = "def _make_result():\n return (1, 2, 3)\n" - f = tmp_path / "code.py" - f.write_text(source, encoding="utf-8") - msgs = _run({str(f): [(1, 100)]}) - assert any("TupleDataclass" in m for m in msgs) - assert not any("CallerUpdater" in m for m in msgs) - - -def test_phase1_private_caller_updater_exception_ignored(tmp_path): - """If CallerUpdater raises during Phase 1, the engine continues gracefully.""" - source = textwrap.dedent( - """\ - def _make_result(): - return (1, 2, 3) - - def use_it(): - a, b, c = _make_result() - """ - ) - f = tmp_path / "code.py" - f.write_text(source, encoding="utf-8") - with patch("crispen.engine.CallerUpdater", side_effect=RuntimeError("fail")): - msgs = _run({str(f): [(1, 100)]}) - # TupleDataclass still ran successfully - assert any("TupleDataclass" in m for m in msgs) - - -# --------------------------------------------------------------------------- -# _has_callers_outside_ranges -# --------------------------------------------------------------------------- - - -def test_has_callers_outside_ranges_found(): - source = "def f(): pass\nf()\n" # call on line 2, range is only line 1 - assert _has_callers_outside_ranges(source, "f", [(1, 1)]) is True - - -def test_has_callers_outside_ranges_not_found(): - source = "def f(): pass\nf()\n" # call on line 2, range covers line 2 - assert _has_callers_outside_ranges(source, "f", [(1, 2)]) is False - - -def test_has_callers_outside_ranges_syntax_error(): - assert _has_callers_outside_ranges("def f(:", "f", [(1, 1)]) is False - - -# --------------------------------------------------------------------------- -# _blocked_private_scopes -# --------------------------------------------------------------------------- - - -def test_blocked_private_scopes_finds_outside_callers(): - # _helper called at line 3, diff range only covers line 1 - source = "def _helper(): pass\n\n_helper()\n" - blocked = _blocked_private_scopes(source, [(1, 1)]) - assert "_helper" in blocked - - -def test_blocked_private_scopes_ignores_in_range_callers(): - # _helper called at line 3, diff range covers line 3 - source = "def _helper(): pass\n\n_helper()\n" - blocked = _blocked_private_scopes(source, [(1, 3)]) - assert "_helper" not in blocked - - -def test_blocked_private_scopes_syntax_error(): - blocked = _blocked_private_scopes("def f(:", [(1, 1)]) - assert blocked == set() - - -def test_blocked_private_scopes_ignores_public(): - # Public functions (no leading _) should not appear in blocked set - source = "def helper(): pass\n\nhelper()\n" - blocked = _blocked_private_scopes(source, [(1, 1)]) - assert "helper" not in blocked - - -# --------------------------------------------------------------------------- -# run_engine: config parameter -# --------------------------------------------------------------------------- - - -def test_run_engine_accepts_explicit_config(tmp_path): - """run_engine works when config is provided explicitly.""" - f = tmp_path / "code.py" - f.write_text("x = 1\n", encoding="utf-8") - config = CrispenConfig() - msgs = list(run_engine({str(f): [(1, 1)]}, config=config)) - assert msgs == [] - - -def test_run_engine_config_none_loads_default(tmp_path): - """run_engine with config=None (default) loads config from disk.""" - f = tmp_path / "code.py" - f.write_text("x = 1\n", encoding="utf-8") - # config=None triggers load_config() internally - msgs = list(run_engine({str(f): [(1, 1)]}, config=None)) - assert msgs == [] - - -# --------------------------------------------------------------------------- -# update_diff_file_callers=False: private function blocked by outside callers -# --------------------------------------------------------------------------- - - -def test_update_diff_file_callers_false_blocks_private_with_outside_caller(tmp_path): - """Private function with a caller outside diff ranges is NOT transformed.""" - source = textwrap.dedent( - """\ - def _make_result(): - return (a, b, c) - - def use_in_diff(): - x, y, z = _make_result() - - def use_outside_diff(): - p, q, r = _make_result() - """ - ) - f = tmp_path / "code.py" - f.write_text(source, encoding="utf-8") - config = CrispenConfig(min_tuple_size=3, update_diff_file_callers=False) - # Diff only covers the function definition and use_in_diff - msgs = list(run_engine({str(f): [(1, 5)]}, config=config)) - # Should NOT have been transformed (outside callers exist) - assert not any("TupleDataclass" in m for m in msgs) - assert "return (a, b, c)" in f.read_text(encoding="utf-8") - - -def test_update_diff_file_callers_false_allows_private_with_only_diff_callers( - tmp_path, -): - """Private function with all callers inside diff is transformed.""" - source = textwrap.dedent( - """\ - def _make_result(): - return (a, b, c) - - def use_in_diff(): - x, y, z = _make_result() - """ - ) - f = tmp_path / "code.py" - f.write_text(source, encoding="utf-8") - config = CrispenConfig(min_tuple_size=3, update_diff_file_callers=False) - msgs = list(run_engine({str(f): [(1, 5)]}, config=config)) - # Only diff caller exists → transformation should proceed - assert any("TupleDataclass" in m for m in msgs) - - -# --------------------------------------------------------------------------- -# update_diff_file_callers=False: public function blocked by diff-file outside callers -# --------------------------------------------------------------------------- - - -def test_update_diff_file_callers_false_blocks_public_with_diff_file_outside_caller( - tmp_path, -): - """Public function with callers outside diff in diff file is skipped.""" - pkg = _make_pkg(tmp_path, "mypkg") - - service = pkg / "service.py" - service.write_text( - "def get_user():\n return (name, age, score)\n", encoding="utf-8" - ) - - api = pkg / "api.py" - api.write_text( - "from mypkg.service import get_user\n" - "def in_diff():\n" - " a, b, c = get_user()\n" - "def not_in_diff():\n" - " x, y, z = get_user()\n", - encoding="utf-8", - ) - - # api.py diff only covers lines 1-3 (in_diff function) - changed = {str(service): [(1, 2)], str(api): [(1, 3)]} - config = CrispenConfig(min_tuple_size=3, update_diff_file_callers=False) - msgs = list(run_engine(changed, _repo_root=str(tmp_path), config=config)) - - # get_user has a caller outside the diff (not_in_diff at lines 4-5) - assert any("callers exist outside the diff" in m for m in msgs) - - -def test_update_diff_file_callers_false_allows_public_with_all_callers_in_diff( - tmp_path, -): - """Public function with all callers inside diff (no diff-file outside callers).""" - pkg = _make_pkg(tmp_path, "mypkg") - - service = pkg / "service.py" - service.write_text( - "def get_user():\n return (name, age, score)\n", encoding="utf-8" - ) - - api = pkg / "api.py" - api.write_text( - "from mypkg.service import get_user\n" - "def main():\n" - " a, b, c = get_user()\n", - encoding="utf-8", - ) - - changed = {str(service): [(1, 2)], str(api): [(1, 3)]} - config = CrispenConfig(min_tuple_size=3, update_diff_file_callers=False) - msgs = list(run_engine(changed, _repo_root=str(tmp_path), config=config)) - - # All callers within diff → transformation should proceed even with - # update_diff_file_callers=False (no callers outside diff ranges) - assert any("TupleDataclass" in m for m in msgs) - assert any("CallerUpdater" in m for m in msgs) - - -# --------------------------------------------------------------------------- -# _categorize_into_stats -# --------------------------------------------------------------------------- - - -def test_categorize_if_not_else(): - s = RunStats() - _categorize_into_stats(s, "IfNotElse: flipped if/else at line 3") - assert s.if_not_else == 1 - assert s.total_edits == 1 - - -def test_categorize_tuple_to_dataclass(): - s = RunStats() - _categorize_into_stats( - s, "TupleDataclass: replaced 3-tuple with FooResult at line 5" - ) - assert s.tuple_to_dataclass == 1 - - -def test_categorize_duplicate_matched(): - s = RunStats() - _categorize_into_stats(s, "DuplicateExtractor: replaced '_f' body with call to 'g'") - assert s.duplicate_matched == 1 - assert s.duplicate_extracted == 0 - - -def test_categorize_duplicate_extracted(): - s = RunStats() - _categorize_into_stats( - s, "DuplicateExtractor: extracted '_helper' from 2 duplicate blocks" - ) - assert s.duplicate_extracted == 1 - assert s.duplicate_matched == 0 - - -def test_categorize_function_split(): - s = RunStats() - _categorize_into_stats(s, "split 'big_func': extracted _step_two") - assert s.function_split == 1 - - -def test_categorize_other_message_ignored(): - s = RunStats() - _categorize_into_stats(s, "CallerUpdater: expanded FooResult unpacking at line 7") - assert s.total_edits == 0 - - -# --------------------------------------------------------------------------- -# run_engine: stats parameter is populated -# --------------------------------------------------------------------------- - - -def test_run_engine_stats_populated(tmp_path): - source = "if not x:\n a()\nelse:\n b()\n" - f = tmp_path / "code.py" - f.write_text(source, encoding="utf-8") - s = RunStats() - list(run_engine({str(f): [(1, 4)]}, config=CrispenConfig(), stats=s)) - assert s.if_not_else == 1 - assert s.files_edited == [str(f)] - assert s.lines_added + s.lines_deleted > 0 - - -def test_run_engine_stats_none_default(tmp_path): - """When stats is None (default), engine runs without error.""" - f = tmp_path / "code.py" - f.write_text("x = 1\n", encoding="utf-8") - msgs = list(run_engine({str(f): [(1, 1)]}, config=CrispenConfig())) - assert msgs == [] - - -# --------------------------------------------------------------------------- -# Phase 2 _apply_tuple_dataclass returning td=None (covers 579->567 branch) -# --------------------------------------------------------------------------- - - -def test_phase2_apply_tuple_dataclass_td_none(tmp_path): - """Phase 2 _apply_tuple_dataclass returning td=None is handled gracefully.""" - pkg = _make_pkg(tmp_path, "mypkg") - - service = pkg / "service.py" - service.write_text("def approved():\n return (1, 2, 3)\n", encoding="utf-8") - - orig_apply = _apply_tuple_dataclass - call_count = {"n": 0} - - def patched_apply(filepath, ranges, source, verbose, approved_public_funcs, **kw): - call_count["n"] += 1 - if call_count["n"] == 2: - # Phase 2 call: return td=None to exercise the td2 is None branch - return (source, [], None) - return orig_apply( - filepath, ranges, source, verbose, approved_public_funcs, **kw - ) - - with patch("crispen.engine._apply_tuple_dataclass", patched_apply): - msgs = list( - run_engine( - {str(service): [(1, 2)]}, - _repo_root=str(tmp_path), - config=CrispenConfig(min_tuple_size=3), - ) - ) - # Should not crash; Phase 2 gracefully skips categorization - assert isinstance(msgs, list) - - -# --------------------------------------------------------------------------- -# FileLimiter (Phase 3 of engine) -# --------------------------------------------------------------------------- - -_FL_PATCH = "crispen.engine.run_file_limiter" - - -def test_file_limiter_disabled_by_max_file_lines_zero(tmp_path): - """max_file_lines=0 disables FileLimiter entirely (branch: if > 0 is False).""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - with patch(_FL_PATCH) as mock_fl: - list(run_engine({str(f): [(1, 1)]}, config=CrispenConfig(max_file_lines=0))) - mock_fl.assert_not_called() - - -def test_file_limiter_skips_short_file(tmp_path): - """File under max_file_lines → FileLimiter is not called for that file.""" - f = tmp_path / "short.py" - f.write_text("x = 1\n", encoding="utf-8") - with patch(_FL_PATCH) as mock_fl: - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=100), - ) - ) - mock_fl.assert_not_called() - - -def test_file_limiter_abort_adds_skip_message(tmp_path): - """FileLimiter abort → SKIP message added; no new files written.""" - f = tmp_path / "big.py" - original = "".join(f"var_{i} = {i}\n" for i in range(10)) - f.write_text(original, encoding="utf-8") - abort_result = FileLimiterResult( - original_source=original, - new_files={}, - messages=[f"SKIP {f} (FileLimiter): file cannot be split"], - abort=True, - ) - with patch(_FL_PATCH, return_value=abort_result): - msgs = list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - assert any("SKIP" in m and "FileLimiter" in m for m in msgs) - assert not (tmp_path / "utils.py").exists() - - -def test_file_limiter_no_messages_no_new_files(tmp_path): - """FileLimiter returns empty messages + no new files → no output, no writes.""" - f = tmp_path / "big.py" - original = "".join(f"var_{i} = {i}\n" for i in range(10)) - f.write_text(original, encoding="utf-8") - no_op_result = FileLimiterResult( - original_source=original, - new_files={}, - messages=[], - abort=False, - ) - with patch(_FL_PATCH, return_value=no_op_result): - msgs = list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - assert not any("FileLimiter" in m for m in msgs) - - -def test_file_limiter_success_writes_new_file(tmp_path): - """FileLimiter success → new file written, original source updated.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - success_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"utils.py": "# new file\n"}, - messages=[f"{f}: FileLimiter: moved foo → utils.py"], - abort=False, - ) - with patch(_FL_PATCH, return_value=success_result): - msgs = list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - assert any("FileLimiter" in m for m in msgs) - new_file = tmp_path / "utils.py" - assert new_file.exists() - assert new_file.read_text(encoding="utf-8") == "# new file\n" - # Original file updated with reduced source. - assert f.read_text(encoding="utf-8") == "# reduced\n" - - -def test_file_limiter_creates_nested_directory(tmp_path): - """FileLimiter target in subdir → parent dirs and __init__.py created.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - success_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"helpers/utils.py": "# helpers\n"}, - messages=[f"{f}: FileLimiter: moved bar → helpers/utils.py"], - abort=False, - ) - with patch(_FL_PATCH, return_value=success_result): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - nested = tmp_path / "helpers" / "utils.py" - assert nested.exists() - assert nested.read_text(encoding="utf-8") == "# helpers\n" - # Subdirectory is initialised as a Python package. - assert (tmp_path / "helpers" / "__init__.py").exists() - - -def test_file_limiter_existing_init_not_overwritten(tmp_path): - """If the target subdir already has __init__.py, it is not overwritten.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - helpers = tmp_path / "helpers" - helpers.mkdir() - (helpers / "__init__.py").write_text("# existing\n", encoding="utf-8") - success_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"helpers/utils.py": "# utils\n"}, - messages=[], - abort=False, - ) - with patch(_FL_PATCH, return_value=success_result): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - assert (helpers / "__init__.py").read_text(encoding="utf-8") == "# existing\n" - - -def test_file_limiter_subdir_split_non_test_deletes_original(tmp_path): - """Non-test subdir split → original file deleted; __init__.py gets split content.""" - f = tmp_path / "service.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - success_result = FileLimiterResult( - original_source=f.read_text(encoding="utf-8"), # reset to original → no write - new_files={ - "service/__init__.py": "# init\n", - "service/utils.py": "# utils\n", - }, - messages=[f"{f}: FileLimiter: moved foo → service/utils.py"], - abort=False, - subdir_name="service", - ) - s = RunStats() - with patch(_FL_PATCH, return_value=success_result): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - stats=s, - ) - ) - # Original service.py must be deleted. - assert not f.exists() - # Package files must exist. - assert (tmp_path / "service" / "__init__.py").read_text( - encoding="utf-8" - ) == "# init\n" - assert (tmp_path / "service" / "utils.py").read_text( - encoding="utf-8" - ) == "# utils\n" - # All original lines must be counted as deleted so verified_lines ≤ lines_deleted. - assert s.lines_deleted == 10 - - -def test_file_limiter_subdir_split_test_keeps_original(tmp_path): - """Test subdir split → original test file kept (not deleted).""" - f = tmp_path / "test_service.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - re_export_src = "# re-exports\n" - success_result = FileLimiterResult( - original_source=re_export_src, - new_files={"service/test_utils.py": "# test utils\n"}, - messages=[], - abort=False, - subdir_name="service", - ) - with patch(_FL_PATCH, return_value=success_result): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - # Original test file must still exist (with re-export content written back). - assert f.exists() - assert f.read_text(encoding="utf-8") == re_export_src - - -def test_file_limiter_subdir_split_has_main_keeps_original(tmp_path): - """Non-test subdir split with has_main → original file kept and updated.""" - f = tmp_path / "service.py" - original_src = "".join(f"var_{i} = {i}\n" for i in range(10)) - f.write_text(original_src, encoding="utf-8") - re_export_src = ( - "from service_lib.utils import foo\n\nif __name__ == '__main__':\n foo()\n" - ) - success_result = FileLimiterResult( - original_source=re_export_src, - new_files={"service_lib/utils.py": "def foo():\n pass\n"}, - messages=[], - abort=False, - subdir_name="service_lib", - has_main=True, - ) - with patch(_FL_PATCH, return_value=success_result): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - # Original service.py must still exist (not deleted). - assert f.exists() - # It should be updated with the re-export stubs + __main__. - assert f.read_text(encoding="utf-8") == re_export_src - # New subdir file must exist. - assert (tmp_path / "service_lib" / "utils.py").read_text(encoding="utf-8") == ( - "def foo():\n pass\n" - ) - - -def test_file_limiter_api_error_propagates(tmp_path): - """CrispenAPIError from FileLimiter propagates out of run_engine.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - with patch(_FL_PATCH, side_effect=CrispenAPIError("rate limit")): - with pytest.raises(CrispenAPIError, match="rate limit"): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - - -def test_file_limiter_recursive_splits_new_file(tmp_path): - """When a new file from FileLimiter is over the limit, it is recursively split.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - # First call: original file → creates "chunk.py" which is still over the limit. - first_result = FileLimiterResult( - original_source="# reduced original\n", - new_files={"chunk.py": "".join(f"x_{i} = {i}\n" for i in range(10))}, - messages=[f"{f}: FileLimiter: moved vars → chunk.py"], - abort=False, - ) - # Second call (recursive): chunk.py → creates "chunk_a.py" and "chunk_b.py". - second_result = FileLimiterResult( - original_source="# reduced chunk\n", - new_files={"chunk_a.py": "# a\n", "chunk_b.py": "# b\n"}, - messages=[str(tmp_path / "chunk.py") + ": FileLimiter: moved → chunk_a/b"], - abort=False, - ) - - call_count = 0 - - def _fl_side_effect(**kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - return first_result - return second_result - - with patch(_FL_PATCH, side_effect=_fl_side_effect): - msgs = list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), - ) - ) - - assert call_count == 2 - # Messages from the recursive call are yielded. - assert any("chunk_a/b" in m for m in msgs) - # Recursive split wrote additional files. - assert (tmp_path / "chunk_a.py").exists() - assert (tmp_path / "chunk_b.py").exists() - # chunk.py was updated with the reduced source from the recursive split. - assert (tmp_path / "chunk.py").read_text(encoding="utf-8") == "# reduced chunk\n" - - -def test_file_limiter_recursive_disabled(tmp_path): - """file_limiter_recursive=False skips recursive processing of new files.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) - first_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"chunk.py": oversized}, - messages=[], - abort=False, - ) - - call_count = 0 - - def _fl_side_effect(**kwargs): - nonlocal call_count - call_count += 1 - return first_result - - with patch(_FL_PATCH, side_effect=_fl_side_effect): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=False), - ) - ) - - # Only one call: recursive processing was disabled. - assert call_count == 1 - - -def test_file_limiter_recursive_abort_stops_recursion(tmp_path): - """Recursive call that aborts does not enqueue further files.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) - first_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"chunk.py": oversized}, - messages=[], - abort=False, - ) - abort_result = FileLimiterResult( - original_source=oversized, - new_files={}, - messages=["SKIP chunk.py (FileLimiter): cannot be split"], - abort=True, - ) - - side_effects = [first_result, abort_result] - - with patch(_FL_PATCH, side_effect=side_effects): - msgs = list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), - ) - ) - - assert any("cannot be split" in m for m in msgs) - - -def test_file_limiter_recursive_api_error_propagates(tmp_path): - """CrispenAPIError during recursive FileLimiter call propagates out.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) - first_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"chunk.py": oversized}, - messages=[], - abort=False, - ) - - side_effects = [first_result, CrispenAPIError("rate limit")] - - with patch(_FL_PATCH, side_effect=side_effects): - with pytest.raises(CrispenAPIError, match="rate limit"): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), - ) - ) - - -def test_file_limiter_recursive_creates_nested_init(tmp_path): - """Recursive FileLimiter creating a file in a subdirectory creates __init__.py.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) - first_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"chunk.py": oversized}, - messages=[], - abort=False, - ) - # Recursive call creates a file in a subdirectory. - second_result = FileLimiterResult( - original_source="# reduced chunk\n", - new_files={"sub/part.py": "# part\n"}, - messages=[], - abort=False, - ) - - with patch(_FL_PATCH, side_effect=[first_result, second_result]): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), - ) - ) - - assert (tmp_path / "sub" / "part.py").exists() - assert (tmp_path / "sub" / "__init__.py").exists() - - -def test_file_limiter_recursive_chains(tmp_path): - """A file created by a recursive call that is still over the limit is re-queued.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) - first_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"chunk.py": oversized}, - messages=[], - abort=False, - ) - # chunk.py recursive call itself creates another oversized file. - second_result = FileLimiterResult( - original_source="# reduced chunk\n", - new_files={"chunk2.py": oversized}, - messages=[], - abort=False, - ) - third_result = FileLimiterResult( - original_source="# reduced chunk2\n", - new_files={}, - messages=[], - abort=True, - ) - - with patch(_FL_PATCH, side_effect=[first_result, second_result, third_result]): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), - ) - ) - - assert (tmp_path / "chunk.py").exists() - assert (tmp_path / "chunk2.py").exists() - - -def test_file_limiter_recursive_source_unchanged(tmp_path): - """Recursive result with same original_source does not rewrite the file.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) - first_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"chunk.py": oversized}, - messages=[], - abort=False, - ) - # Recursive call: original_source equals the input source → no rewrite. - second_result = FileLimiterResult( - original_source=oversized, # same as what was written - new_files={"part.py": "# part\n"}, - messages=[], - abort=False, - ) - - with patch(_FL_PATCH, side_effect=[first_result, second_result]): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), - ) - ) - - # chunk.py content is the oversized source (unchanged); the engine did not - # rewrite it because original_source == r_source. - assert (tmp_path / "chunk.py").read_text(encoding="utf-8") == oversized - - -def test_file_limiter_recursive_subdir_split_deletes_file(tmp_path): - """Recursive FileLimiter subdir split on a non-test file deletes the file.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) - first_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"chunk.py": oversized}, - messages=[], - abort=False, - ) - # Recursive call triggers subdir split: chunk.py → chunk/ package. - second_result = FileLimiterResult( - original_source=oversized, - new_files={"chunk/__init__.py": "# init\n", "chunk/utils.py": "# utils\n"}, - messages=[], - abort=False, - subdir_name="chunk", - ) - - with patch(_FL_PATCH, side_effect=[first_result, second_result]): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), - ) - ) - - # chunk.py was deleted because subdir_name is set and it's not a test file. - assert not (tmp_path / "chunk.py").exists() - assert (tmp_path / "chunk" / "__init__.py").exists() - - -# --------------------------------------------------------------------------- -# _should_run -# --------------------------------------------------------------------------- - - -def test_should_run_defaults_allow_all(): - cfg = CrispenConfig() - for name in ( - "if_not_else", - "duplicate_extractor", - "function_splitter", - "tuple_dataclass", - "file_limiter", - ): - assert _should_run(name, cfg) is True - - -def test_should_run_enabled_list_allows_listed(): - cfg = CrispenConfig(enabled_refactors=["if_not_else", "function_splitter"]) - assert _should_run("if_not_else", cfg) is True - assert _should_run("function_splitter", cfg) is True - - -def test_should_run_enabled_list_blocks_unlisted(): - cfg = CrispenConfig(enabled_refactors=["if_not_else"]) - assert _should_run("duplicate_extractor", cfg) is False - assert _should_run("tuple_dataclass", cfg) is False - assert _should_run("file_limiter", cfg) is False - - -def test_should_run_disabled_list_blocks_listed(): - cfg = CrispenConfig(disabled_refactors=["function_splitter", "file_limiter"]) - assert _should_run("function_splitter", cfg) is False - assert _should_run("file_limiter", cfg) is False - - -def test_should_run_disabled_list_allows_unlisted(): - cfg = CrispenConfig(disabled_refactors=["function_splitter"]) - assert _should_run("if_not_else", cfg) is True - assert _should_run("tuple_dataclass", cfg) is True - - -def test_should_run_enabled_takes_precedence_over_disabled(): - # enabled_refactors non-empty → disabled_refactors is ignored - cfg = CrispenConfig( - enabled_refactors=["if_not_else"], - disabled_refactors=["if_not_else"], - ) - assert _should_run("if_not_else", cfg) is True - - -# --------------------------------------------------------------------------- -# Engine integration — enabled_refactors / disabled_refactors -# --------------------------------------------------------------------------- - - -def test_engine_disabled_refactors_skips_if_not_else(tmp_path): - """With if_not_else disabled the pattern is left unchanged.""" - source = textwrap.dedent( - """\ - if not x: - a() - else: - b() - """ - ) - f = tmp_path / "code.py" - f.write_text(source, encoding="utf-8") - msgs = list( - run_engine( - {str(f): [(1, 4)]}, - config=CrispenConfig(disabled_refactors=["if_not_else"]), - ) - ) - assert not any("IfNotElse" in m for m in msgs) - assert f.read_text(encoding="utf-8") == source - - -def test_engine_enabled_refactors_runs_only_listed(tmp_path): - """enabled_refactors=["if_not_else"] — other refactors don't touch the file.""" - source = textwrap.dedent( - """\ - if not x: - a() - else: - b() - """ - ) - f = tmp_path / "code.py" - f.write_text(source, encoding="utf-8") - - called = [] - - class _Spy(Refactor): - @classmethod - def name(cls): - return "Spy" - - def get_changes(self): - called.append("Spy") - return [] - - with patch("crispen.engine._REFACTORS", [_Spy]): - with patch("crispen.engine._REFACTOR_KEY", {_Spy: "spy"}): - list( - run_engine( - {str(f): [(1, 4)]}, - config=CrispenConfig(enabled_refactors=["if_not_else"]), - ) - ) - - # _Spy is not in enabled_refactors, so it must not have been called. - assert called == [] - - -def test_engine_file_limiter_skipped_when_disabled(tmp_path): - """file_limiter in disabled_refactors prevents FileLimiter from running.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - success_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"utils.py": "# new\n"}, - messages=["FileLimiter: moved"], - abort=False, - ) - with patch(_FL_PATCH, return_value=success_result) as mock_fl: - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig( - max_file_lines=5, - disabled_refactors=["file_limiter"], - ), - ) - ) - mock_fl.assert_not_called() - - -def test_engine_match_function_disabled_passes_flag_to_duplicate_extractor(tmp_path): - """disabled_refactors=["match_function"] passes match_functions=False to DE.""" - f = tmp_path / "code.py" - f.write_text("x = 1\n", encoding="utf-8") - - constructed_with: dict = {} - - original_init = __import__( - "crispen.refactors.duplicate_extractor", fromlist=["DuplicateExtractor"] - ).DuplicateExtractor.__init__ - - def _spy_init(self, *args, **kwargs): - constructed_with.update(kwargs) - original_init(self, *args, **kwargs) - - with patch("crispen.engine.DuplicateExtractor.__init__", side_effect=_spy_init): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(disabled_refactors=["match_function"]), - ) - ) - - assert constructed_with.get("match_functions") is False - - -def test_engine_match_function_enabled_by_default(tmp_path): - """Without any filter, match_functions=True is passed to DuplicateExtractor.""" - f = tmp_path / "code.py" - f.write_text("x = 1\n", encoding="utf-8") - - constructed_with: dict = {} - - original_init = __import__( - "crispen.refactors.duplicate_extractor", fromlist=["DuplicateExtractor"] - ).DuplicateExtractor.__init__ - - def _spy_init(self, *args, **kwargs): - constructed_with.update(kwargs) - original_init(self, *args, **kwargs) - - with patch("crispen.engine.DuplicateExtractor.__init__", side_effect=_spy_init): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(), - ) - ) - - assert constructed_with.get("match_functions") is True - - -def test_file_limiter_empty_original_source_deletes_file(tmp_path): - """FileLimiter returns empty original_source → original file is deleted.""" - f = tmp_path / "big.py" - original = "".join(f"var_{i} = {i}\n" for i in range(10)) - f.write_text(original, encoding="utf-8") - # All content was moved out; original_source is empty (all entities migrated). - # new_files content is kept short (≤ max_file_lines) so it doesn't re-enter - # the recursive queue (file_limiter_recursive defaults to True). - moved_source = "# moved content\n" - drained_result = FileLimiterResult( - original_source="", - new_files={"utils.py": moved_source}, - messages=[f"{f}: FileLimiter: moved all → utils.py"], - abort=False, - ) - with patch(_FL_PATCH, return_value=drained_result): - msgs = list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - assert any("FileLimiter" in m for m in msgs) - # Original file must be deleted, not left as a blank file. - assert not f.exists() - # New file must exist with the moved content. - assert (tmp_path / "utils.py").exists() - - -def test_file_limiter_recursive_empty_original_source_deletes_file(tmp_path): - """Recursive FileLimiter with empty original_source deletes the recursive file.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) - # chunk_a content is short so it doesn't re-enter the recursive queue. - small = "# chunk_a content\n" - first_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"chunk.py": oversized}, - messages=[], - abort=False, - ) - # Recursive call drains chunk.py entirely; original_source is empty. - second_result = FileLimiterResult( - original_source="", - new_files={"chunk_a.py": small}, - messages=[], - abort=False, - ) - - with patch(_FL_PATCH, side_effect=[first_result, second_result]): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), - ) - ) - - # chunk.py was drained and must be deleted. - assert not (tmp_path / "chunk.py").exists() - # New file from the recursive split must exist. - assert (tmp_path / "chunk_a.py").exists() - - -def test_file_limiter_subdir_split_empty_source_file_already_deleted(tmp_path): - """Subdir split deletes the original file; empty original_source skips re-unlink.""" - f = tmp_path / "big.py" - original = "".join(f"var_{i} = {i}\n" for i in range(10)) - f.write_text(original, encoding="utf-8") - # subdir_name causes Phase 3 to delete the original file. original_source="" - # means the per_file loop sees an empty source for a file that no longer - # exists — exercising the elif-is-False branch (803→805). - # new_files content kept short (≤ max_file_lines) to avoid recursive queue. - subdir_result = FileLimiterResult( - original_source="", - new_files={"big/__init__.py": "# package\n"}, - messages=[f"{f}: FileLimiter: subdir split → big/"], - abort=False, - subdir_name="big", - ) - with patch(_FL_PATCH, return_value=subdir_result): - msgs = list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - assert any("FileLimiter" in m for m in msgs) - # Original file was deleted by the subdir split; must not exist. - assert not f.exists() - # Package init was created. - assert (tmp_path / "big" / "__init__.py").exists() - - -def test_file_limiter_empty_init_py_preserved(tmp_path): - """__init__.py is never deleted even when FileLimiter drains it to empty.""" - f = tmp_path / "__init__.py" - original = "".join(f"def func_{i}():\n pass\n\n" for i in range(10)) - f.write_text(original, encoding="utf-8") - drained = FileLimiterResult( - original_source="", - new_files={"utils.py": "# moved\n"}, - messages=[f"{f}: FileLimiter: moved all → utils.py"], - abort=False, - ) - with patch(_FL_PATCH, return_value=drained): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - ) - ) - # __init__.py must still exist (empty is fine; deletion would break the package). - assert f.exists() - assert f.read_text(encoding="utf-8") == "" - - -def test_file_limiter_recursive_empty_init_py_preserved(tmp_path): - """__init__.py created during recursive split is kept even when drained empty.""" - orig = tmp_path / "big.py" - orig.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - oversized = "".join(f"x_{i} = {i}\n" for i in range(10)) - first_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"pkg/__init__.py": oversized}, - messages=[], - abort=False, - ) - # Recursive pass drains pkg/__init__.py; original_source is empty. - second_result = FileLimiterResult( - original_source="", - new_files={"pkg/utils.py": "# utils\n"}, - messages=[], - abort=False, - ) - with patch(_FL_PATCH, side_effect=[first_result, second_result]): - list( - run_engine( - {str(orig): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), - ) - ) - # pkg/__init__.py must survive as an empty file, not be deleted. - assert (tmp_path / "pkg" / "__init__.py").exists() - assert (tmp_path / "pkg" / "__init__.py").read_text(encoding="utf-8") == "" - - -# --------------------------------------------------------------------------- -# _module_path_for_file -# --------------------------------------------------------------------------- - - -def test_module_path_for_file_returns_dotted_path(tmp_path): - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - sub = tmp_path / "tests" / "lua" - sub.mkdir(parents=True) - f = sub / "test_foo.py" - f.write_text("", encoding="utf-8") - assert _module_path_for_file(str(f)) == "tests.lua.test_foo" - - -def test_module_path_for_file_init_strips_init_segment(tmp_path): - """__init__.py resolves to the package name, not package.__init__.""" - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - pkg = tmp_path / "mypkg" / "subpkg" - pkg.mkdir(parents=True) - f = pkg / "__init__.py" - f.write_text("", encoding="utf-8") - assert _module_path_for_file(str(f)) == "mypkg.subpkg" - - -def test_module_path_for_file_no_markers_returns_none(tmp_path): - f = tmp_path / "test_foo.py" - f.write_text("", encoding="utf-8") - # No pyproject.toml / .git anywhere up the path — within tmp_path hierarchy. - # We can't guarantee no markers exist above tmp_path in the real filesystem, - # so only assert that the function returns a string or None without raising. - result = _module_path_for_file(str(f)) - assert result is None or isinstance(result, str) - - -# --------------------------------------------------------------------------- -# _redirect_inline_module_imports -# --------------------------------------------------------------------------- - - -def test_redirect_inline_module_imports_basic(): - source = "def run():\n from pkg.old import Foo, Bar\n Foo()\n" - result = _redirect_inline_module_imports( - source, "pkg.old", {"Foo": "pkg.new_foo", "Bar": "pkg.new_bar"} - ) - assert "from pkg.new_foo import Foo" in result - assert "from pkg.new_bar import Bar" in result - assert "from pkg.old import" not in result - - -def test_redirect_inline_module_imports_partial_redirect(): - # Only 'Foo' has a new location; 'Baz' is unknown and kept in old module. - source = "def run():\n from pkg.old import Foo, Baz\n Foo()\n" - result = _redirect_inline_module_imports(source, "pkg.old", {"Foo": "pkg.new_foo"}) - assert "from pkg.new_foo import Foo" in result - assert "from pkg.old import Baz" in result - - -def test_redirect_inline_module_imports_no_matching_import(): - source = "def run():\n from pkg.other import Foo\n Foo()\n" - result = _redirect_inline_module_imports(source, "pkg.old", {"Foo": "pkg.new"}) - assert result == source - - -def test_redirect_inline_module_imports_module_level(): - # Module-level import is also redirected. - source = "from pkg.old import Foo\n\ndef run():\n Foo()\n" - result = _redirect_inline_module_imports(source, "pkg.old", {"Foo": "pkg.new"}) - assert "from pkg.new import Foo" in result - assert "from pkg.old import" not in result - - -def test_redirect_inline_module_imports_syntax_error(): - source = "def (invalid" - result = _redirect_inline_module_imports(source, "pkg.old", {"Foo": "pkg.new"}) - assert result == source - - -def test_redirect_inline_module_imports_no_moved_names(): - # Import exists but none of the names are in the map — leave unchanged. - source = "def run():\n from pkg.old import Baz\n" - result = _redirect_inline_module_imports(source, "pkg.old", {"Foo": "pkg.new"}) - assert result == source - - -# --------------------------------------------------------------------------- -# _patch_inline_imports_after_test_deletion -# --------------------------------------------------------------------------- - - -def test_patch_inline_imports_after_test_deletion_updates_per_file(tmp_path): - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - sub = tmp_path / "pkg" - sub.mkdir() - # Simulate the deleted test file path and new files created by its split. - deleted_path = str(tmp_path / "pkg" / "test_old.py") - deleted_dir = tmp_path / "pkg" - new_files = { - "sub/test_new.py": "class TestFoo:\n pass\n", - } - (deleted_dir / "sub").mkdir() - (deleted_dir / "sub" / "test_new.py").write_text(new_files["sub/test_new.py"]) - - src = "def run():\n from pkg.test_old import TestFoo\n TestFoo()\n" - per_file = { - "parent.py": { - "source": src, - "original": src, - } - } - fl_new_file_final: dict = {} - - _patch_inline_imports_after_test_deletion( - deleted_path, deleted_dir, new_files, per_file, fl_new_file_final - ) - - updated = per_file["parent.py"]["source"] - assert "from pkg.sub.test_new import TestFoo" in updated - assert "from pkg.test_old import" not in updated - - -def test_patch_inline_imports_after_test_deletion_updates_new_files(tmp_path): - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - sub = tmp_path / "pkg" - sub.mkdir() - deleted_path = str(tmp_path / "pkg" / "test_old.py") - deleted_dir = tmp_path / "pkg" - - sibling_content = ( - "def helper():\n from pkg.test_old import TestFoo\n TestFoo()\n" - ) - sibling_path = str(tmp_path / "pkg" / "test_sibling.py") - (tmp_path / "pkg" / "test_sibling.py").write_text(sibling_content, encoding="utf-8") - - (deleted_dir / "sub").mkdir() - new_file_content = "class TestFoo:\n pass\n" - (deleted_dir / "sub" / "test_new.py").write_text(new_file_content) - - new_files = {"sub/test_new.py": new_file_content} - per_file: dict = {} - fl_new_file_final = {sibling_path: sibling_content} - - _patch_inline_imports_after_test_deletion( - deleted_path, deleted_dir, new_files, per_file, fl_new_file_final - ) - - updated = fl_new_file_final[sibling_path] - assert "from pkg.sub.test_new import TestFoo" in updated - assert "from pkg.test_old import" not in updated - # File was re-written to disk. - assert ( - "from pkg.sub.test_new import TestFoo" - in (tmp_path / "pkg" / "test_sibling.py").read_text() - ) - - -def test_patch_inline_imports_after_test_deletion_no_markers_skips(tmp_path): - # No pyproject.toml — module path unresolvable; function must not raise. - deleted_path = str(tmp_path / "pkg" / "test_old.py") - deleted_dir = tmp_path / "pkg" - deleted_dir.mkdir(parents=True) - per_file = { - "parent.py": { - "source": "def run():\n from pkg.test_old import TestFoo\n", - "original": "def run():\n from pkg.test_old import TestFoo\n", - } - } - # Should not raise; source is unchanged because old_mod is None. - _patch_inline_imports_after_test_deletion( - deleted_path, deleted_dir, {}, per_file, {} - ) - assert ( - per_file["parent.py"]["source"] - == "def run():\n from pkg.test_old import TestFoo\n" - ) - - -def test_patch_inline_imports_after_test_deletion_new_mod_none_skips(tmp_path): - # new_mod resolves to None for a path outside the project root → that entry - # is skipped and name_to_new_mod stays empty → function returns early. - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - sub = tmp_path / "pkg" - sub.mkdir() - deleted_path = str(tmp_path / "pkg" / "test_old.py") - deleted_dir = tmp_path / "pkg" - # Relative path that escapes the project root when resolved from deleted_dir. - outside_rel = "../../../outside/test_new.py" - per_file = { - "parent.py": { - "source": "def run():\n from pkg.test_old import TestFoo\n", - "original": "def run():\n from pkg.test_old import TestFoo\n", - } - } - _patch_inline_imports_after_test_deletion( - deleted_path, - deleted_dir, - {outside_rel: "class TestFoo:\n pass\n"}, - per_file, - {}, - ) - # Source unchanged because name_to_new_mod was empty. - assert ( - per_file["parent.py"]["source"] - == "def run():\n from pkg.test_old import TestFoo\n" - ) - - -def test_patch_inline_imports_after_test_deletion_syntax_error_in_new_file(tmp_path): - # SyntaxError in a new_file content → that entry is skipped. - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - sub = tmp_path / "pkg" / "sub" - sub.mkdir(parents=True) - (sub / "test_new.py").write_text("def (invalid", encoding="utf-8") - deleted_path = str(tmp_path / "pkg" / "test_old.py") - deleted_dir = tmp_path / "pkg" - per_file = { - "parent.py": { - "source": "def run():\n from pkg.test_old import TestFoo\n", - "original": "def run():\n from pkg.test_old import TestFoo\n", - } - } - _patch_inline_imports_after_test_deletion( - deleted_path, deleted_dir, {"sub/test_new.py": "def (invalid"}, per_file, {} - ) - # Source unchanged; SyntaxError in the new file caused it to be skipped. - assert ( - per_file["parent.py"]["source"] - == "def run():\n from pkg.test_old import TestFoo\n" - ) - - -def test_patch_inline_imports_after_test_deletion_no_class_or_func_in_new_file( - tmp_path, -): - # New file has no ClassDef/FunctionDef → name_to_new_mod stays empty → skip. - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - sub = tmp_path / "pkg" / "sub" - sub.mkdir(parents=True) - deleted_path = str(tmp_path / "pkg" / "test_old.py") - deleted_dir = tmp_path / "pkg" - per_file = { - "parent.py": { - "source": "def run():\n from pkg.test_old import TestFoo\n", - "original": "def run():\n from pkg.test_old import TestFoo\n", - } - } - _patch_inline_imports_after_test_deletion( - deleted_path, deleted_dir, {"sub/test_new.py": "X = 1\n"}, per_file, {} - ) - assert ( - per_file["parent.py"]["source"] - == "def run():\n from pkg.test_old import TestFoo\n" - ) - - -def test_patch_inline_imports_after_test_deletion_source_unchanged_no_import(tmp_path): - # per_file source has no import from old_mod → update is a no-op. - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - sub = tmp_path / "pkg" / "sub" - sub.mkdir(parents=True) - (sub / "test_new.py").write_text("class TestFoo:\n pass\n", encoding="utf-8") - deleted_path = str(tmp_path / "pkg" / "test_old.py") - deleted_dir = tmp_path / "pkg" - original_source = "def run():\n pass\n" - per_file = { - "parent.py": { - "source": original_source, - "original": original_source, - } - } - _patch_inline_imports_after_test_deletion( - deleted_path, - deleted_dir, - {"sub/test_new.py": "class TestFoo:\n pass\n"}, - per_file, - {}, - ) - # Source is identical to original (no-op branch taken). - assert per_file["parent.py"]["source"] == original_source - - -def test_patch_inline_imports_after_test_deletion_empty_fl_entry_skipped(tmp_path): - # fl_new_file_final entry with empty/None content is skipped without error. - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - sub = tmp_path / "pkg" / "sub" - sub.mkdir(parents=True) - (sub / "test_new.py").write_text("class TestFoo:\n pass\n", encoding="utf-8") - deleted_path = str(tmp_path / "pkg" / "test_old.py") - deleted_dir = tmp_path / "pkg" - fl_new_file_final = {str(tmp_path / "empty.py"): ""} - _patch_inline_imports_after_test_deletion( - deleted_path, - deleted_dir, - {"sub/test_new.py": "class TestFoo:\n pass\n"}, - {}, - fl_new_file_final, - ) - # Empty entry was skipped; dict unchanged. - assert fl_new_file_final[str(tmp_path / "empty.py")] == "" - - -# --------------------------------------------------------------------------- -# Engine integration: recursive test-file deletion patches parent imports -# --------------------------------------------------------------------------- - - -def test_file_limiter_recursive_test_deletion_patches_parent_inline_imports(tmp_path): - """When a recursive split deletes a test file, inline imports in the parent - file that point to the deleted module are updated to the new locations.""" - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - - # Parent test file with inline imports referencing a module that will be - # created by the first split and then deleted by the recursive split. - parent_src = ( - "def run_comprehensive_tests():\n" - " from tests.test_collections import TestA, TestB\n" - " TestA()\n" - " TestB()\n" - "\n" - "if __name__ == '__main__':\n" - " run_comprehensive_tests()\n" - ) - parent_file = tmp_path / "tests" / "test_suite.py" - parent_file.parent.mkdir(parents=True) - parent_file.write_text(parent_src, encoding="utf-8") - - # First pass: parent → creates test_collections.py (oversized). - # The original source has the inline import already present (as if - # _inject_inline_test_imports_original added it). - first_result = FileLimiterResult( - original_source=parent_src, # unchanged (inline import already injected) - new_files={ - "test_collections.py": ( - "class TestA:\n pass\n\nclass TestB:\n pass\n" - ) - }, - messages=[], - abort=False, - ) - - # Recursive pass: test_collections.py → split into sub/test_a.py + sub/test_b.py. - # All entities migrated; original_source is empty → file will be deleted. - recursive_result = FileLimiterResult( - original_source="", - new_files={ - "sub/test_a.py": "class TestA:\n pass\n", - "sub/test_b.py": "class TestB:\n pass\n", - }, - messages=[], - abort=False, - ) - - with patch(_FL_PATCH, side_effect=[first_result, recursive_result]): - list( - run_engine( - {str(parent_file): [(1, len(parent_src.splitlines()))]}, - config=CrispenConfig(max_file_lines=2, file_limiter_recursive=True), - ) - ) - - # test_collections.py was deleted. - assert not (parent_file.parent / "test_collections.py").exists() - - # Parent file now has updated inline imports pointing to the new locations. - updated = parent_file.read_text(encoding="utf-8") - assert "from tests.sub.test_a import TestA" in updated - assert "from tests.sub.test_b import TestB" in updated - assert "from tests.test_collections import" not in updated - - -def test_file_limiter_llm_timing_recorded_in_stats(tmp_path): - """When FileLimiterResult has llm_elapsed > 0, record_llm_call is invoked.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - timed_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"utils.py": "# new\n"}, - messages=[f"{f}: FileLimiter: moved → utils.py"], - abort=False, - llm_elapsed=1.5, - llm_input_tokens=100, - llm_output_tokens=50, - ) - stats = RunStats() - with patch(_FL_PATCH, return_value=timed_result): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5), - stats=stats, - ) - ) - assert "file_limiter" in stats.llm_elapsed_by_category - - -def test_file_limiter_recursive_llm_timing_recorded_in_stats(tmp_path): - """Recursive FileLimiterResult with llm_elapsed > 0 triggers record_llm_call.""" - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - first_result = FileLimiterResult( - original_source="# reduced original\n", - new_files={"chunk.py": "".join(f"x_{i} = {i}\n" for i in range(10))}, - messages=[f"{f}: moved vars → chunk.py"], - abort=False, - ) - second_result = FileLimiterResult( - original_source="# reduced chunk\n", - new_files={"chunk_a.py": "# a\n"}, - messages=[], - abort=False, - llm_elapsed=2.0, - llm_input_tokens=200, - llm_output_tokens=80, - ) - stats = RunStats() - call_count = 0 - - def _fl_side_effect(**kwargs): - nonlocal call_count - call_count += 1 - return first_result if call_count == 1 else second_result - - with patch(_FL_PATCH, side_effect=_fl_side_effect): - list( - run_engine( - {str(f): [(1, 1)]}, - config=CrispenConfig(max_file_lines=5, file_limiter_recursive=True), - stats=stats, - ) - ) - assert "file_limiter" in stats.llm_elapsed_by_category - - # --------------------------------------------------------------------------- # _collect_top_level_names # --------------------------------------------------------------------------- -def test_collect_top_level_names_various(): - """Covers functions, classes, assignments, aug/ann assigns, imports, from-imports, - non-Name aug-assign targets, and unrecognised statement types.""" - source = ( - "import os\n" - "import libcst as cst\n" - "from pathlib import Path\n" - "from typing import List as L\n" - "from os import *\n" # star import skipped - "_CONST = 42\n" - "x: int = 1\n" - "counter += 1\n" - "a, b = 1, 2\n" # tuple target → ast.Tuple, not ast.Name - "some_obj.attr += 1\n" # AugAssign with Attribute target → skipped - "if True: pass\n" # ast.If → matches no elif, skipped - "def my_func(): pass\n" - "class MyClass: pass\n" - "async def async_func(): pass\n" - ) - result = _collect_top_level_names(source) - assert "os" in result - assert "cst" in result - assert "Path" in result - assert "L" in result - assert "_CONST" in result - assert "x" in result - assert "counter" in result - assert "my_func" in result - assert "MyClass" in result - assert "async_func" in result - # Tuple-unpacking targets (a, b = …) are ast.Tuple, not ast.Name → skipped - assert "a" not in result - assert "b" not in result - # Attribute aug-assign (some_obj.attr += 1) → target is Attribute, skipped - assert "some_obj" not in result - - -def test_collect_top_level_names_syntax_error(): - """Invalid Python source → empty set.""" - assert _collect_top_level_names("def broken(:") == set() - - # _collect_imported_names # --------------------------------------------------------------------------- - - -def test_collect_imported_names_various(): - """Covers import, import-as, from-import, from-import-as, star (skip).""" - source = ( - "import os\n" - "import os.path\n" - "import json as json_mod\n" - "from pathlib import Path\n" - "from typing import List as L\n" - "from os import *\n" - ) - result = _collect_imported_names(source) - assert result == {"os", "path", "json_mod", "Path", "L"} - - -def test_collect_imported_names_syntax_error(): - """Invalid Python source → empty set.""" - assert _collect_imported_names("def broken(:") == set() - - -# --------------------------------------------------------------------------- -# _collect_assignment_names -# --------------------------------------------------------------------------- - - -def test_collect_assignment_names_basic(): - """Covers plain assignment, annotated assignment, augmented assignment.""" - source = ( - "_CONST = 42\n" - "x: int = 1\n" - "counter += 1\n" - "a, b = 1, 2\n" # tuple target → skipped - "obj.attr += 1\n" # attribute aug-assign → skipped - "def my_func(): pass\n" # function → skipped - "import os\n" # import → skipped - ) - result = _collect_assignment_names(source) - assert result == {"_CONST", "x", "counter"} - - -def test_collect_assignment_names_syntax_error(): - """Invalid Python source → empty set.""" - assert _collect_assignment_names("def broken(:") == set() - - -# --------------------------------------------------------------------------- -# _collect_code_referenced_names -# --------------------------------------------------------------------------- - - -def test_collect_code_referenced_names_finds_load_uses(): - """Names used in code expressions are returned.""" - src = "from .sub import MyFunc\nresult = MyFunc()\n" - assert "MyFunc" in _collect_code_referenced_names(src) - - -def test_collect_code_referenced_names_excludes_import_aliases(): - """Import alias names are not ast.Name nodes → not returned.""" - src = "from .sub import MyFunc\n" - assert "MyFunc" not in _collect_code_referenced_names(src) - - -def test_collect_code_referenced_names_excludes_funcdef_name(): - """Function definition names are not ast.Name Load nodes.""" - src = "def MyFunc(): pass\n" - assert "MyFunc" not in _collect_code_referenced_names(src) - - -def test_collect_code_referenced_names_syntax_error(): - """Returns empty set on unparseable source.""" - assert _collect_code_referenced_names("def (broken:") == set() - - -# --------------------------------------------------------------------------- -# _build_patch_map -# --------------------------------------------------------------------------- - - -def test_build_patch_map_empty_entity_to_target(tmp_path): - """No entity_to_target → empty map.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - f = tmp_path / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={}, - entity_to_target={}, - ) - result = _build_patch_map(str(f), fl_result, tmp_path) - assert result == {} - - -def test_build_patch_map_no_old_module(tmp_path): - """When _module_path_for_file returns None for filepath → empty map.""" - # No pyproject.toml anywhere → cannot find project root - f = tmp_path / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={"utils.py": "class MyClass: pass\n"}, - entity_to_target={"MyClass": "utils.py"}, - ) - result = _build_patch_map(str(f), fl_result, tmp_path) - assert result == {} - - -def test_build_patch_map_no_callers_uses_definer(tmp_path): - """Entity with no callers maps to its definition file.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={"utils.py": "class MyClass: pass\n"}, - entity_to_target={"MyClass": "utils.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg) - assert result == {"mypkg.module.MyClass": "mypkg.utils.MyClass"} - - -def test_build_patch_map_single_caller_uses_caller(tmp_path): - """Entity imported and used by exactly one new file → caller's module used.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "def MyFunc(): pass\n", - "caller.py": "from .sub import MyFunc\nMyFunc()\n", - }, - entity_to_target={"MyFunc": "sub.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg) - assert result == {"mypkg.module.MyFunc": "mypkg.caller.MyFunc"} - - -def test_build_patch_map_forking_entity_skipped(tmp_path): - """Entity used by multiple new files (forking) → skipped.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "def MyFunc(): pass\n", - "caller_a.py": "from .sub import MyFunc\nMyFunc()\n", - "caller_b.py": "from .sub import MyFunc\nMyFunc()\n", - }, - entity_to_target={"MyFunc": "sub.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg) - assert result == {} - - -def test_build_patch_map_empty_new_file_skipped(tmp_path): - """New file with empty source is skipped when building import index.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "class MyClass: pass\n", - "empty.py": "", - }, - entity_to_target={"MyClass": "sub.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg) - assert result == {"mypkg.module.MyClass": "mypkg.sub.MyClass"} - - -def test_build_patch_map_new_module_none(tmp_path): - """When target file's module path can't be resolved → entity is skipped.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - f = tmp_path / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={"utils.py": "class MyClass: pass\n"}, - entity_to_target={"MyClass": "utils.py"}, - ) - with patch( - "crispen.engine._module_path_for_file", side_effect=["mypkg.module", None] - ): - result = _build_patch_map(str(f), fl_result, tmp_path) - assert result == {} - - -def test_build_patch_map_import_alias_single_importer(tmp_path): - """Import alias from original used in exactly one new file → added.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - pre_split = "from external import Helper\ndef MyFunc(): pass\n" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "def MyFunc(): pass\n", - "utils.py": "from external import Helper\nHelper()\n", - }, - entity_to_target={"MyFunc": "sub.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg, pre_split) - assert result["mypkg.module.Helper"] == "mypkg.utils.Helper" - assert result["mypkg.module.MyFunc"] == "mypkg.sub.MyFunc" - - -def test_build_patch_map_import_alias_forking_skipped(tmp_path): - """Import alias used in zero or multiple new files is skipped.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - pre_split = ( - "from external import Forked\nfrom external import Nowhere\ndef F(): pass\n" - ) - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "from external import Forked\nForked()\ndef F(): pass\n", - "utils.py": "from external import Forked\nForked()\n", - }, - entity_to_target={"F": "sub.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg, pre_split) - # Forked used in 2 files → forking → skipped; Nowhere used in 0 files → skipped - assert "mypkg.module.Forked" not in result - assert "mypkg.module.Nowhere" not in result - - -def test_build_patch_map_import_alias_skips_entity_names(tmp_path): - """Import alias that is also an entity name is not double-processed.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - # "Helper" appears both as entity_to_target key and in pre_split imports - pre_split = "from external import Helper\n" - fl_result = FileLimiterResult( - original_source="", - new_files={"utils.py": "from external import Helper\n"}, - entity_to_target={"Helper": "utils.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg, pre_split) - # Entity loop handles Helper (definer=utils.py, no external callers → utils.py) - assert result == {"mypkg.module.Helper": "mypkg.utils.Helper"} - - -def test_build_patch_map_import_alias_module_none(tmp_path): - """Import alias target module can't be resolved → alias is skipped.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - pre_split = "from external import Helper\ndef MyFunc(): pass\n" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "def MyFunc(): pass\n", - "utils.py": "from external import Helper\nHelper()\n", - }, - entity_to_target={"MyFunc": "sub.py"}, - ) - # Third call (for alias importer utils.py) returns None - with patch( - "crispen.engine._module_path_for_file", - side_effect=["mypkg.module", "mypkg.sub", None], - ): - result = _build_patch_map(str(f), fl_result, pkg, pre_split) - # MyFunc was added (second call succeeded); Helper was skipped (third → None) - assert result == {"mypkg.module.MyFunc": "mypkg.sub.MyFunc"} - assert "mypkg.module.Helper" not in result - - -def test_build_patch_map_import_only_caller_falls_back_to_definer(tmp_path): - """Entity imported but not used by any new file → falls back to definer.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "def MyFunc(): pass\n", - "__init__.py": "from .sub import MyFunc\n", - }, - entity_to_target={"MyFunc": "sub.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg) - # __init__.py only re-exports (no Load usage) → 0 real callers → fall back to sub.py - assert result == {"mypkg.module.MyFunc": "mypkg.sub.MyFunc"} - - -def test_build_patch_map_reexport_ignored_real_caller_wins(tmp_path): - """Re-export stub ignored; the one file that actually calls the entity is used.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "def MyFunc(): pass\n", - "caller.py": "from .sub import MyFunc\nMyFunc()\n", - "__init__.py": "from .sub import MyFunc\n", - }, - entity_to_target={"MyFunc": "sub.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg) - # __init__.py has no Load usage; caller.py does → single real caller - assert result == {"mypkg.module.MyFunc": "mypkg.caller.MyFunc"} - - -def test_build_patch_map_init_real_usage_counted_as_caller(tmp_path): - """__init__.py that actually calls an entity is counted as a real caller. - - The module path strips .__init__ so the patch target is the public - package namespace (mypkg.MyFunc, not mypkg.__init__.MyFunc). - """ - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "def MyFunc(): pass\n", - "__init__.py": "from .sub import MyFunc\n_x = MyFunc()\n", - }, - entity_to_target={"MyFunc": "sub.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg) - # __init__.py has a Load reference → real caller; .__init__ stripped from path - assert result == {"mypkg.module.MyFunc": "mypkg.MyFunc"} - - -def test_build_patch_map_init_real_usage_plus_other_caller_forks(tmp_path): - """__init__.py calling entity + another caller → forking → skipped.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "def MyFunc(): pass\n", - "caller.py": "from .sub import MyFunc\nMyFunc()\n", - "__init__.py": "from .sub import MyFunc\nMyFunc()\n", - }, - entity_to_target={"MyFunc": "sub.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg) - # 2 real callers (caller.py + __init__.py) → forking → skipped - assert "mypkg.module.MyFunc" not in result - - -def test_build_patch_map_import_alias_reexport_stub_skipped(tmp_path): - """Import alias whose only importer is a re-export stub is skipped.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - f = pkg / "module.py" - pre_split = "from external import Helper\ndef F(): pass\n" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "sub.py": "def F(): pass\n", - "__init__.py": "from external import Helper\n", - }, - entity_to_target={"F": "sub.py"}, - ) - result = _build_patch_map(str(f), fl_result, pkg, pre_split) - # __init__.py imports Helper but has no Load usage → 0 real importers → skipped - assert "mypkg.module.Helper" not in result - - -# --------------------------------------------------------------------------- -# _build_patch_map — variable assignment section -# --------------------------------------------------------------------------- - - -def test_build_patch_map_assignment_no_callers(tmp_path): - """Module-level variable in original file, only used in its defining new file.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - pre_split = "_TIMEOUT = 30\ndef run(): pass\n" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "core.py": "_TIMEOUT = 30\ndef run(): pass\n", - }, - abort=False, - entity_to_target={"run": "core.py"}, - ) - result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) - # 0 callers → _TIMEOUT stays in its definer core.py - assert result["mypkg.big._TIMEOUT"] == "mypkg.core._TIMEOUT" - - -def test_build_patch_map_assignment_single_caller(tmp_path): - """Variable defined in one new file, imported and used by exactly one other.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - pre_split = "_TIMEOUT = 30\n" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "core.py": "_TIMEOUT = 30\n", - "runner.py": "from .core import _TIMEOUT\nif _TIMEOUT > 0: pass\n", - }, - abort=False, - entity_to_target={}, - ) - result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) - # runner.py imports and uses _TIMEOUT → single caller - assert result["mypkg.big._TIMEOUT"] == "mypkg.runner._TIMEOUT" - - -def test_build_patch_map_assignment_forking_skipped(tmp_path): - """Variable imported and used by two new files → forking → skipped.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - pre_split = "_TIMEOUT = 30\n" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "core.py": "_TIMEOUT = 30\n", - "a.py": "from .core import _TIMEOUT\nif _TIMEOUT: pass\n", - "b.py": "from .core import _TIMEOUT\nif _TIMEOUT: pass\n", - }, - abort=False, - entity_to_target={}, - ) - result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) - assert "mypkg.big._TIMEOUT" not in result - - -def test_build_patch_map_assignment_defined_in_multiple_files_skipped(tmp_path): - """Variable appearing in two new files' assignments → ambiguous → skipped.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - pre_split = "_TIMEOUT = 30\n" - fl_result = FileLimiterResult( - original_source="", - new_files={ - "core.py": "_TIMEOUT = 30\n", - "utils.py": "_TIMEOUT = 60\n", - }, - abort=False, - entity_to_target={}, - ) - result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) - assert "mypkg.big._TIMEOUT" not in result - - -def test_build_patch_map_assignment_not_in_original_skipped(tmp_path): - """Variable introduced by code generation (not in pre_split_source) → skipped.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - pre_split = "def run(): pass\n" # _TIMEOUT not in original - fl_result = FileLimiterResult( - original_source="", - new_files={"core.py": "_TIMEOUT = 30\ndef run(): pass\n"}, - abort=False, - entity_to_target={"run": "core.py"}, - ) - result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) - assert "mypkg.big._TIMEOUT" not in result - - -def test_build_patch_map_assignment_already_in_patch_map_skipped(tmp_path): - """Variable in patch_map from import-alias section → assignment section skips it.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - # pre_split both imports and assigns _TIMEOUT → import-alias section maps it first. - pre_split = "from ext import _TIMEOUT\n_TIMEOUT = 30\n" - fl_result = FileLimiterResult( - original_source="", - # core.py imports, assigns, and uses _TIMEOUT: alias + assignment. - new_files={ - "core.py": ( - "from ext import _TIMEOUT\n_TIMEOUT = 30\nif _TIMEOUT > 0: pass\n" - ) - }, - abort=False, - entity_to_target={}, - ) - result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) - # Import-alias section mapped it; assignment section hits old_path in patch_map. - assert result["mypkg.big._TIMEOUT"] == "mypkg.core._TIMEOUT" - # Verify mapped exactly once (assignment section did NOT add a duplicate). - assert list(result.values()).count("mypkg.core._TIMEOUT") == 1 - - -def test_build_patch_map_assignment_new_module_none_skipped(tmp_path): - """When _module_path_for_file returns None for the target, entry is skipped.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - pre_split = "_TIMEOUT = 30\n" - fl_result = FileLimiterResult( - original_source="", - # Target path cannot be resolved to a module (no pyproject.toml ancestor). - new_files={"/unresolvable/abs/path.py": "_TIMEOUT = 30\n"}, - abort=False, - entity_to_target={}, - ) - result = _build_patch_map(str(pkg / "big.py"), fl_result, pkg, pre_split) - assert "mypkg.big._TIMEOUT" not in result - - -# --------------------------------------------------------------------------- -# Phase 4 — @patch string update integration tests -# --------------------------------------------------------------------------- - - -def _make_fl_result_with_entities(source="# reduced\n"): - """Build a FileLimiterResult that moved MyClass → utils.py.""" - return FileLimiterResult( - original_source=source, - new_files={"utils.py": "class MyClass: pass\n"}, - messages=["big.py: FileLimiter: moved MyClass → utils.py"], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - - -def test_patch_update_ignore_mode(tmp_path): - """Default 'ignore' mode → @patch strings are never updated.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - other = tmp_path / "test_other.py" - other.write_text( - '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" - ) - - fl_result = _make_fl_result_with_entities() - with patch(_FL_PATCH, return_value=fl_result): - list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="ignore", - ), - _repo_root=str(tmp_path), - ) - ) - # test_other.py should be unchanged - assert ( - other.read_text(encoding="utf-8") - == '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n' - ) - - -def test_patch_update_no_combined_map(tmp_path): - """'update' mode but FL returned empty entity_to_target → no updates.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - other = tmp_path / "test_other.py" - other.write_text( - '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" - ) - - no_entity_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"utils.py": "class MyClass: pass\n"}, - messages=["big.py: FileLimiter: moved MyClass → utils.py"], - abort=False, - entity_to_target={}, # empty! - ) - with patch(_FL_PATCH, return_value=no_entity_result): - list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="basic", - ), - _repo_root=str(tmp_path), - ) - ) - assert ( - other.read_text(encoding="utf-8") - == '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n' - ) - - -def test_patch_update_updates_per_file_source(tmp_path): - """'update' mode, FL moved entities → per_file source with old path is updated.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - big_source = "".join(f"var_{i} = {i}\n" for i in range(10)) - f.write_text(big_source, encoding="utf-8") - # Another diff file with an old @patch string - other_diff = pkg / "test_big.py" - other_diff.write_text( - '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" - ) - - fl_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"utils.py": "class MyClass: pass\n"}, - messages=["big.py: FileLimiter: moved MyClass → utils.py"], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - with patch(_FL_PATCH, return_value=fl_result): - msgs = list( - run_engine( - {str(f): [(1, 10)], str(other_diff): [(1, 2)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="basic", - ), - _repo_root=str(tmp_path), - ) - ) - # The per_file source for other_diff should have the updated string - updated_text = other_diff.read_text(encoding="utf-8") - assert "mypkg.utils.MyClass" in updated_text - assert any("patch_update" in m for m in msgs) - - -def test_patch_update_updates_other_file(tmp_path): - """'update' mode, a separate file outside per_file gets updated on disk.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - big_source = "".join(f"var_{i} = {i}\n" for i in range(10)) - f.write_text(big_source, encoding="utf-8") - # A file NOT in the diff that has the old @patch string - other = tmp_path / "test_other.py" - other.write_text( - '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" - ) - - fl_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"utils.py": "class MyClass: pass\n"}, - messages=["big.py: FileLimiter: moved MyClass → utils.py"], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - with patch(_FL_PATCH, return_value=fl_result): - msgs = list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="basic", - ), - _repo_root=str(tmp_path), - ) - ) - updated_text = other.read_text(encoding="utf-8") - assert "mypkg.utils.MyClass" in updated_text - assert any("patch_update" in m for m in msgs) - - -def test_patch_update_skips_excluded_dir(tmp_path): - """Files under .venv/ are excluded from Phase 4 scanning.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - big_source = "".join(f"var_{i} = {i}\n" for i in range(10)) - f.write_text(big_source, encoding="utf-8") - venv_dir = tmp_path / ".venv" - venv_dir.mkdir() - venv_file = venv_dir / "test.py" - venv_content = '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n' - venv_file.write_text(venv_content, encoding="utf-8") - - fl_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"utils.py": "class MyClass: pass\n"}, - messages=["big.py: FileLimiter: moved MyClass → utils.py"], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - with patch(_FL_PATCH, return_value=fl_result): - list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="basic", - ), - _repo_root=str(tmp_path), - ) - ) - # .venv/test.py must not be modified - assert venv_file.read_text(encoding="utf-8") == venv_content - - -def test_patch_update_no_repo_root(tmp_path): - """When repo_root can't be found, Phase 4 skips entirely.""" - # No .git or pyproject.toml → _find_repo_root returns None - f = tmp_path / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - other = tmp_path / "test_other.py" - other.write_text( - '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" - ) - - fl_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"utils.py": "class MyClass: pass\n"}, - messages=[], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - with patch(_FL_PATCH, return_value=fl_result): - list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="basic", - ), - # No _repo_root passed, no .git in tmp_path → repo_root=None - ) - ) - # test_other.py should be unchanged - assert ( - other.read_text(encoding="utf-8") - == '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n' - ) - - -def test_patch_update_oserror_skipped(tmp_path): - """Phase 4 continues gracefully when read_text raises OSError.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - big_source = "".join(f"var_{i} = {i}\n" for i in range(10)) - f.write_text(big_source, encoding="utf-8") - # A file that will raise OSError when read - other = tmp_path / "test_other.py" - other.write_text( - '@patch("mypkg.big.MyClass")\ndef test_it(): pass\n', encoding="utf-8" - ) - - fl_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"utils.py": "class MyClass: pass\n"}, - messages=[], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - - other_abs = str(other.resolve()) - - original_pathlib_read = None - - def _patched_read_text(self, encoding="utf-8"): - if str(self.resolve()) == other_abs: - raise OSError("permission denied") - return original_pathlib_read(self, encoding=encoding) - - import pathlib - - original_pathlib_read = pathlib.Path.read_text - - with patch.object(pathlib.Path, "read_text", _patched_read_text): - with patch(_FL_PATCH, return_value=fl_result): - # Should not raise even though read_text raises OSError - msgs = list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="basic", - ), - _repo_root=str(tmp_path), - ) - ) - - # No patch_update message for other since it raised OSError - assert not any("test_other" in m and "patch_update" in m for m in msgs) - - -def test_patch_update_accumulates_from_recursive_fl(tmp_path): - """Entities from recursive FL results also contribute to the patch map.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - big_source = "".join(f"var_{i} = {i}\n" for i in range(10)) - f.write_text(big_source, encoding="utf-8") - - # Other file outside per_file with old @patch strings for both files - other = tmp_path / "test_other.py" - other.write_text( - '@patch("mypkg.big.MyClass")\n' - '@patch("mypkg.utils.HelperClass")\n' - "def test_it(): pass\n", - encoding="utf-8", - ) - - # First FL result: big.py → utils.py (MyClass moved there) - first_result = FileLimiterResult( - original_source="# reduced\n", - new_files={ - "utils.py": "class MyClass: pass\n" * 10 - }, # over limit for recursion - messages=[], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - - # utils.py written by first result; set up that file - utils_path = pkg / "utils.py" - - # Second FL result: utils.py → helpers.py (HelperClass moved there) - second_result = FileLimiterResult( - original_source="# utils reduced\n", - new_files={"helpers.py": "class HelperClass: pass\n"}, - messages=[], - abort=False, - entity_to_target={"HelperClass": "helpers.py"}, - ) - - call_count = 0 - - def _fl_side_effect(**kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - # Write utils.py so the recursive call can find it - utils_path.write_text("class MyClass: pass\n" * 10, encoding="utf-8") - return first_result - return second_result - - with patch(_FL_PATCH, side_effect=_fl_side_effect): - msgs = list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_recursive=True, - file_limiter_patch_update="basic", - ), - _repo_root=str(tmp_path), - ) - ) - - # Verify combined_patch_map was non-empty by checking at least one - # patch_update message was generated (from the other file or per_file). - updated_text = other.read_text(encoding="utf-8") - # At minimum, MyClass should be updated (from first pass) - assert "mypkg.utils.MyClass" in updated_text or any( - "patch_update" in m for m in msgs - ) - - -def test_patch_update_chain_flattening(tmp_path): - """Transitive chains in combined_patch_map are flattened before apply. - - When a first split produces A→B and a recursive split produces B→C, - apply_patch_strings must map A directly to C, not to the intermediate B. - """ - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - # big.py imports get_api_key and uses it; also defines func_0 (moved entity) - big_source = "from llm import get_api_key\n" + "".join( - f"def func_{i}(): get_api_key()\n" for i in range(10) - ) - f.write_text(big_source, encoding="utf-8") - - # Other file has @patch pointing at the imported alias in big.py - other = tmp_path / "test_other.py" - other.write_text( - '@patch("mypkg.big.get_api_key")\ndef test_it(): pass\n', - encoding="utf-8", - ) - - # First FL result: big.py → utils.py. - # utils.py is over max_file_lines so it will be queued for recursive split. - # It imports and uses get_api_key so the alias ends up in utils's map entry. - utils_source = "from llm import get_api_key\n" + "".join( - f"def helper_{i}(): get_api_key()\n" for i in range(10) - ) - first_result = FileLimiterResult( - original_source="# big reduced\n", - new_files={"utils.py": utils_source}, - messages=[], - abort=False, - entity_to_target={ - "func_0": "utils.py" - }, # non-empty to trigger _build_patch_map - ) - - # Second FL result (recursive split of utils.py) → helpers.py - helpers_source = "from llm import get_api_key\n" "def helper_0(): get_api_key()\n" - second_result = FileLimiterResult( - original_source="# utils reduced\n", - new_files={"helpers.py": helpers_source}, - messages=[], - abort=False, - entity_to_target={"helper_0": "helpers.py"}, # non-empty - ) - - call_count = 0 - - def _fl_side_effect(**kwargs): - nonlocal call_count - call_count += 1 - return first_result if call_count == 1 else second_result - - with patch(_FL_PATCH, side_effect=_fl_side_effect): - list( - run_engine( - {str(f): [(1, len(big_source.splitlines()))]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_recursive=True, - file_limiter_patch_update="basic", - ), - _repo_root=str(tmp_path), - ) - ) - - # Round 1 map: mypkg.big.get_api_key → mypkg.utils.get_api_key - # Round 2 map: mypkg.utils.get_api_key → mypkg.helpers.get_api_key - # After flattening: mypkg.big.get_api_key → mypkg.helpers.get_api_key - # Without flattening the test file would still hold the intermediate path. - updated = other.read_text(encoding="utf-8") - assert ( - "mypkg.helpers.get_api_key" in updated - ), f"Expected chain-flattened path but got: {updated!r}" - - -# --------------------------------------------------------------------------- -# _add_fl_context (engine helper for "rewrite" patch mode) -# --------------------------------------------------------------------------- - - -def test_add_fl_context_no_module_path(): - """When module path cannot be determined, _add_fl_context does nothing.""" - fl_list = [] - fl_result = FileLimiterResult( - original_source="", - new_files={}, - abort=False, - entity_to_target={"X": "a.py"}, - ) - # A path with no ancestor containing pyproject.toml / .git → returns None. - _add_fl_context(fl_list, "/no/project/root/here/file.py", "", fl_result, {}) - assert fl_list == [] - - -def test_add_fl_context_no_forking(tmp_path): - """When all entities are already in combined_patch_map, nothing is appended.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - (tmp_path / "mypkg").mkdir() - fl_list = [] - fl_result = FileLimiterResult( - original_source="", - new_files={}, - abort=False, - entity_to_target={"X": "a.py"}, - ) - filepath = str(tmp_path / "mypkg" / "big.py") - # Entity already covered by combined_patch_map → forking_old_paths is empty. - # No _block_N entities → nothing appended. - _add_fl_context(fl_list, filepath, "", fl_result, {"mypkg.big.X": "mypkg.a.X"}) - assert fl_list == [] - - -def test_add_fl_context_block_entity_uses_specific_names(tmp_path): - """When a _block_N entity was moved and all named entities are mapped, - the block-internal names (vars, imports) from the target file are used as - specific scan keys — NOT the broad module path — so already-updated strings - like ``old_module.sub.run_engine`` are not re-sent to the LLM.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - (tmp_path / "mypkg").mkdir() - fl_list = [] - fl_result = FileLimiterResult( - original_source="modified\n", - new_files={ - "core.py": "_REFACTORS = []\nimport libcst as cst\n\ndef X(): pass\n" - }, - abort=False, - entity_to_target={"_block_1": "core.py", "X": "core.py"}, - ) - filepath = str(tmp_path / "mypkg" / "big.py") - # Both entities are in the patch map → forking_old_paths would be empty. - # _block_1 is a TOP_LEVEL block → scan core.py for block-internal names. - # X is in entity_to_target so it's excluded; _REFACTORS and cst are not. - combined = { - "mypkg.big._block_1": "mypkg.core._block_1", - "mypkg.big.X": "mypkg.core.X", - } - _add_fl_context(fl_list, filepath, "original\n", fl_result, combined) - assert len(fl_list) == 1 - # Only _REFACTORS and cst are block-internal; X is excluded (named entity). - assert fl_list[0].forking_old_paths == {"mypkg.big._REFACTORS", "mypkg.big.cst"} - assert fl_list[0].old_module == "mypkg.big" - - -def test_add_fl_context_block_entity_no_new_names(tmp_path): - """When a _block_N entity was moved but the target file contains no names - beyond those already in entity_to_target, nothing is appended.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - (tmp_path / "mypkg").mkdir() - fl_list = [] - fl_result = FileLimiterResult( - original_source="modified\n", - # core.py only defines X, which is already in entity_to_target. - new_files={"core.py": "def X(): pass\n"}, - abort=False, - entity_to_target={"_block_1": "core.py", "X": "core.py"}, - ) - filepath = str(tmp_path / "mypkg" / "big.py") - combined = { - "mypkg.big._block_1": "mypkg.core._block_1", - "mypkg.big.X": "mypkg.core.X", - } - _add_fl_context(fl_list, filepath, "original\n", fl_result, combined) - assert fl_list == [] - - -def test_add_fl_context_forking_and_block_combined(tmp_path): - """Forking entities AND block-internal names are both added to forking_old_paths.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - (tmp_path / "mypkg").mkdir() - fl_list = [] - fl_result = FileLimiterResult( - original_source="modified\n", - new_files={"core.py": "_TIMEOUT = 30\ndef Y(): pass\n"}, - abort=False, - # Y is forking (not in combined_patch_map); _block_1 moved with _TIMEOUT inside. - entity_to_target={"_block_1": "core.py", "Y": "core.py"}, - ) - filepath = str(tmp_path / "mypkg" / "big.py") - # Only _block_1 is in combined_patch_map; Y is not (forking). - combined = {"mypkg.big._block_1": "mypkg.core._block_1"} - _add_fl_context(fl_list, filepath, "original\n", fl_result, combined) - assert len(fl_list) == 1 - # Y is a forking entity; _TIMEOUT is block-internal; Y in new file is excluded - # (it's in all_entity_names). - assert fl_list[0].forking_old_paths == {"mypkg.big.Y", "mypkg.big._TIMEOUT"} - - -def test_add_fl_context_normal(tmp_path): - """Forking entity not in combined_patch_map → appended to fl_all_contexts.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - (tmp_path / "mypkg").mkdir() - fl_list = [] - fl_result = FileLimiterResult( - original_source="modified\n", - new_files={"utils.py": "class X: pass\n"}, - abort=False, - entity_to_target={"X": "utils.py"}, - ) - filepath = str(tmp_path / "mypkg" / "big.py") - _add_fl_context(fl_list, filepath, "original\n", fl_result, {}) - assert len(fl_list) == 1 - assert fl_list[0].forking_old_paths == {"mypkg.big.X"} - assert fl_list[0].old_module == "mypkg.big" - assert fl_list[0].original_source == "original\n" - assert fl_list[0].modified_source == "modified\n" - - -def test_add_fl_context_forked_import_alias_added(tmp_path): - """Import aliases forked across multiple new files are added to forking_old_paths. - - When the original file imports ``call_with_tool`` and multiple new sub-files - also import it, basic mode skips it (forking). _add_fl_context must still - add ``old_module.call_with_tool`` to forking_old_paths so the LLM rewrite - step can detect and update @patch decorators that reference it. - """ - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - (tmp_path / "mypkg").mkdir() - fl_list = [] - # Original file imports call_with_tool; both new sub-files also import it - # (forking) so basic mode left it out of combined_patch_map. - pre_split = "from external import call_with_tool\ndef F(): pass\n" - fl_result = FileLimiterResult( - original_source="modified\n", - new_files={ - "a.py": ( - "from external import call_with_tool\ncall_with_tool()\ndef F(): pass\n" - ), - "b.py": "from external import call_with_tool\ncall_with_tool()\n", - }, - abort=False, - entity_to_target={"F": "a.py"}, - ) - filepath = str(tmp_path / "mypkg" / "big.py") - # F is already in combined_patch_map (non-forking entity); call_with_tool - # is NOT in combined_patch_map (forked, skipped by basic mode). - combined = {"mypkg.big.F": "mypkg.a.F"} - _add_fl_context(fl_list, filepath, pre_split, fl_result, combined) - assert len(fl_list) == 1 - # call_with_tool must be in forking_old_paths despite F being already mapped. - assert "mypkg.big.call_with_tool" in fl_list[0].forking_old_paths - - -def test_add_fl_context_forked_import_alias_entity_name_skipped(tmp_path): - """Import alias that is also an entity name is skipped by the alias loop's continue. - - Helper is in entity_to_target (not in combined_patch_map) so the entity - section already adds it to forking_old_paths. The alias loop hits the - ``continue`` branch and does not process it again. ``other`` (import alias - only, not an entity) is picked up by the alias loop instead. - """ - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - (tmp_path / "mypkg").mkdir() - fl_list = [] - pre_split = "from ext import Helper, other\ndef F(): pass\n" - fl_result = FileLimiterResult( - original_source="modified\n", - new_files={"a.py": "from ext import other\nother()\ndef F(): pass\n"}, - abort=False, - # Helper is both an imported alias and a named entity (forking entity). - entity_to_target={"Helper": "a.py", "F": "a.py"}, - ) - filepath = str(tmp_path / "mypkg" / "big.py") - # Neither entity is in combined_patch_map → both are forking. - _add_fl_context(fl_list, filepath, pre_split, fl_result, {}) - assert len(fl_list) == 1 - # Helper was added by the entity section; other was added by the alias loop. - assert "mypkg.big.Helper" in fl_list[0].forking_old_paths - assert "mypkg.big.other" in fl_list[0].forking_old_paths - - -def test_add_fl_context_forked_import_alias_already_mapped_skipped(tmp_path): - """Import alias already in combined_patch_map is not re-added.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - (tmp_path / "mypkg").mkdir() - fl_list = [] - # pre_split imports Helper (already mapped by basic) and call_with_tool (forked). - pre_split = "from ext import Helper, call_with_tool\ndef F(): pass\n" - fl_result = FileLimiterResult( - original_source="modified\n", - new_files={ - "a.py": "from ext import call_with_tool\ncall_with_tool()\ndef F(): pass\n", - "b.py": "from ext import call_with_tool\ncall_with_tool()\n", - }, - abort=False, - entity_to_target={"F": "a.py"}, - ) - filepath = str(tmp_path / "mypkg" / "big.py") - # Helper is already in combined_patch_map (basic mapped it); call_with_tool is not. - combined = { - "mypkg.big.F": "mypkg.a.F", - "mypkg.big.Helper": "mypkg.a.Helper", - } - _add_fl_context(fl_list, filepath, pre_split, fl_result, combined) - assert len(fl_list) == 1 - # Helper is already mapped → not added again; call_with_tool is forked → added. - assert "mypkg.big.Helper" not in fl_list[0].forking_old_paths - assert "mypkg.big.call_with_tool" in fl_list[0].forking_old_paths - - -def test_add_fl_context_subdir_split_uses_init_as_modified_source(tmp_path): - """Non-test subdir split: modified_source comes from new_files[subdir/__init__.py]. - - runner.py restores fl_result.original_source to the pre-split source for - non-test, non-has_main subdir splits and places the post-split __init__.py - content in new_files. _add_fl_context must use that __init__.py content as - modified_source so _build_rename_guard_sets and the BFS terminal builder see - the correct set of names still present in the module after the split. - """ - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - (tmp_path / "mypkg").mkdir() - fl_list = [] - pre_split = "from llm import call_with_tool\ndef F(): pass\ndef G(): pass\n" - # Post-split __init__.py re-exports F but call_with_tool is NOT re-exported. - init_src = "from .sub import F\ndef advise(): pass\n" - fl_result = FileLimiterResult( - # runner.py restored original_source to pre-split for non-test subdir. - original_source=pre_split, - new_files={ - "advisor/__init__.py": init_src, - "advisor/sub.py": "from llm import call_with_tool\ndef F(): pass\n", - }, - abort=False, - subdir_name="advisor", - entity_to_target={"F": "advisor/sub.py"}, - ) - filepath = str(tmp_path / "mypkg" / "big.py") - _add_fl_context(fl_list, filepath, pre_split, fl_result, {}) - assert len(fl_list) == 1 - # modified_source must be the __init__.py content, not original_source. - assert fl_list[0].modified_source == init_src - assert fl_list[0].original_source == pre_split - - -def test_add_fl_context_subdir_split_no_init_falls_back_to_original_source(tmp_path): - """Test/has_main subdir split: no __init__.py → falls back to original_source. - - For test files and has_main files with subdir_name set, runner.py does NOT - add a subdir/__init__.py to new_files. The modified_source should therefore - fall back to fl_result.original_source (the post-split original file). - """ - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - (tmp_path / "mypkg").mkdir() - fl_list = [] - pre_split = "from llm import call_with_tool\ndef F(): pass\n" - # No __init__.py in new_files; original_source is the post-split state. - post_split_original = "from llm import call_with_tool\ndef F(): pass\n# stubs\n" - fl_result = FileLimiterResult( - original_source=post_split_original, - new_files={"advisor/sub.py": "def G(): pass\n"}, - abort=False, - subdir_name="advisor", - entity_to_target={"G": "advisor/sub.py"}, - ) - filepath = str(tmp_path / "mypkg" / "big.py") - _add_fl_context(fl_list, filepath, pre_split, fl_result, {}) - assert len(fl_list) == 1 - # Falls back to fl_result.original_source since no __init__.py in new_files. - assert fl_list[0].modified_source == post_split_original - - -# --------------------------------------------------------------------------- -# "rewrite" patch mode in Phase 4 -# --------------------------------------------------------------------------- - - -_REWRITE_PATCH = "crispen.engine.apply_patch_rewrite" - - -def test_patch_update_rewrite_mode_calls_apply_patch_rewrite(tmp_path): - """'rewrite' mode with forking entities calls apply_patch_rewrite in Phase 4.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - # Entity appears as a caller in two new files → forking → skipped by basic. - fl_result = FileLimiterResult( - original_source="# reduced\n", - new_files={ - "utils.py": "class MyClass: pass\n", - "caller_a.py": "from .big import MyClass\nMyClass()\n", - "caller_b.py": "from .big import MyClass\nMyClass()\n", - }, - messages=[], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - with ( - patch(_FL_PATCH, return_value=fl_result), - patch(_REWRITE_PATCH, return_value=iter([])) as mock_rewrite, - ): - list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="rewrite", - ), - _repo_root=str(tmp_path), - ) - ) - mock_rewrite.assert_called_once() - contexts = mock_rewrite.call_args[0][0] - assert len(contexts) == 1 - assert "mypkg.big.MyClass" in contexts[0].forking_old_paths - - -def test_patch_update_rewrite_mode_records_llm_stats(tmp_path): - """Rewrite accumulator with non-zero elapsed/tokens triggers record_llm_call.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - fl_result = FileLimiterResult( - original_source="# reduced\n", - new_files={ - "utils.py": "class MyClass: pass\n", - "caller_a.py": "from .big import MyClass\nMyClass()\n", - "caller_b.py": "from .big import MyClass\nMyClass()\n", - }, - messages=[], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - - def _rewrite_with_acc( - fl_contexts, per_file, repo_root, config, verbose=False, _acc=None, **_kwargs - ): - if _acc is not None: - _acc.calls = 2 - _acc.elapsed = 1.5 - _acc.input_tokens = 100 - _acc.output_tokens = 20 - _acc.files_updated = 1 - return iter([]) - - stats = RunStats() - with ( - patch(_FL_PATCH, return_value=fl_result), - patch(_REWRITE_PATCH, side_effect=_rewrite_with_acc), - ): - list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="rewrite", - ), - _repo_root=str(tmp_path), - stats=stats, - ) - ) - assert stats.patch_rewrite_llm_calls == 2 - assert stats.patch_update_edits == 1 - assert stats.llm_elapsed == 1.5 - assert stats.llm_input_tokens == 100 - assert "patch_rewriter" in stats.llm_elapsed_by_refactor - - -def test_patch_update_rewrite_mode_no_fl_contexts_skips_apply(tmp_path): - """'rewrite' mode but no forking entities → apply_patch_rewrite not called.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - # Entity has only ONE caller → non-forking → goes into combined_patch_map. - fl_result = FileLimiterResult( - original_source="# reduced\n", - new_files={"utils.py": "class MyClass: pass\n"}, - messages=[], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - with ( - patch(_FL_PATCH, return_value=fl_result), - patch(_REWRITE_PATCH, return_value=iter([])) as mock_rewrite, - ): - list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="rewrite", - ), - _repo_root=str(tmp_path), - ) - ) - mock_rewrite.assert_not_called() - - -def test_patch_update_rewrite_mode_recursive_fl_context_added(tmp_path): - """'rewrite' mode: forking entity from recursive FL pass is collected.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - # Main FL result: produces medium.py with 6 lines (> max_file_lines=5), - # which triggers the recursive pass. No entity_to_target here so the - # main-loop rewrite branch is not entered. - medium_src = "".join(f"med_{i} = {i}\n" for i in range(6)) - main_fl_result = FileLimiterResult( - original_source="# big_reduced\n", - new_files={"medium.py": medium_src}, - messages=[], - abort=False, - entity_to_target={}, - ) - - # Recursive FL result: MyClass appears in two callers → forking → skipped - # by _build_patch_map → not in combined_patch_map → triggers _add_fl_context. - recursive_fl_result = FileLimiterResult( - original_source="# medium_reduced\n", - new_files={ - "small.py": "class MyClass: pass\n", - "caller_a.py": "from .medium import MyClass\nMyClass()\n", - "caller_b.py": "from .medium import MyClass\nMyClass()\n", - }, - messages=[], - abort=False, - entity_to_target={"MyClass": "small.py"}, - ) - - with ( - patch(_FL_PATCH, side_effect=[main_fl_result, recursive_fl_result]), - patch(_REWRITE_PATCH, return_value=iter([])) as mock_rewrite, - ): - list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_recursive=True, - file_limiter_patch_update="rewrite", - ), - _repo_root=str(tmp_path), - ) - ) - - mock_rewrite.assert_called_once() - contexts = mock_rewrite.call_args[0][0] - assert any("mypkg.medium.MyClass" in ctx.forking_old_paths for ctx in contexts) - - -_CG_PATCH = "crispen.engine.apply_patch_callgraph" - - -def test_patch_update_callgraph_yields_message(tmp_path): - """apply_patch_callgraph message increments patch_update_edits and is yielded.""" - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - # Entity appears in two callers → forking → _fl_all_contexts is populated - fl_result = FileLimiterResult( - original_source="# reduced\n", - new_files={ - "utils.py": "class MyClass: pass\n", - "caller_a.py": "from .big import MyClass\nMyClass()\n", - "caller_b.py": "from .big import MyClass\nMyClass()\n", - }, - messages=[], - abort=False, - entity_to_target={"MyClass": "utils.py"}, - ) - - cg_msg = "test_other.py: patch_callgraph: resolved MyClass" - - stats = RunStats() - with ( - patch(_FL_PATCH, return_value=fl_result), - patch(_CG_PATCH, return_value=iter([cg_msg])) as mock_cg, - ): - msgs = list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_patch_update="basic", - ), - _repo_root=str(tmp_path), - stats=stats, - ) - ) - - mock_cg.assert_called_once() - assert cg_msg in msgs - assert stats.patch_update_edits >= 1 - - -def test_patch_update_ignore_mode_recursive_fl_entity_to_target(tmp_path): - """'ignore' mode: recursive FL result with entity_to_target skips _add_fl_context.""" # noqa: E501 - (tmp_path / "pyproject.toml").write_text("[tool.crispen]\n", encoding="utf-8") - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - f = pkg / "big.py" - f.write_text("".join(f"var_{i} = {i}\n" for i in range(10)), encoding="utf-8") - - medium_src = "".join(f"med_{i} = {i}\n" for i in range(6)) - main_fl_result = FileLimiterResult( - original_source="# big_reduced\n", - new_files={"medium.py": medium_src}, - messages=[], - abort=False, - entity_to_target={}, # empty — no _add_fl_context for main result - ) - - # Recursive FL result has non-empty entity_to_target; with "ignore" mode the - # branch at engine.py line 1278 is False → _add_fl_context is not called. - recursive_fl_result = FileLimiterResult( - original_source="# medium_reduced\n", - new_files={"small.py": "class MyClass: pass\n"}, - messages=[], - abort=False, - entity_to_target={"MyClass": "small.py"}, - ) - - medium_path = pkg / "medium.py" - call_count = 0 - - def _fl_side_effect(**kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - medium_path.write_text(medium_src, encoding="utf-8") - return main_fl_result - return recursive_fl_result - - with patch(_FL_PATCH, side_effect=_fl_side_effect): - msgs = list( - run_engine( - {str(f): [(1, 10)]}, - config=CrispenConfig( - max_file_lines=5, - file_limiter_recursive=True, - file_limiter_patch_update="ignore", - ), - _repo_root=str(tmp_path), - ) - ) - - assert call_count == 2 # main pass + one recursive pass - assert not any("callgraph" in m for m in msgs) diff --git a/tests/test_examples.py b/tests/test_examples.py index bc021b7..04d8c63 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -305,7 +305,7 @@ def test_match_existing_function_no_arg_helper(monkeypatch): with ( patch("crispen.llm_client.anthropic.Anthropic"), patch( - "crispen.refactors.duplicate_extractor._run_with_timeout", + "crispen.refactors.duplicate_extractor.extractor._run_with_timeout", return_value=(True, "identical logging setup", ""), ), ): diff --git a/tests/test_function_splitter.py b/tests/test_function_splitter.py index 41272ab..1df4897 100644 --- a/tests/test_function_splitter.py +++ b/tests/test_function_splitter.py @@ -1,1751 +1,3 @@ """Tests for function_splitter: 100% branch coverage.""" from __future__ import annotations - -import textwrap -from unittest.mock import MagicMock, patch - -import libcst as cst -import pytest -from libcst.metadata import MetadataWrapper, PositionProvider - -from crispen.refactors.function_splitter import ( - _ApiTimeout, - _FuncInfo, - _FunctionCollector, - _SplitTask, - _choose_best_split, - _count_body_lines, - _extract_func_source, - _find_free_vars, - _find_valid_splits, - _func_in_changed_range, - _generate_call, - _generate_helper_source, - _has_nested_funcdef, - _has_new_undefined_names, - _has_yield, - _head_effective_lines, - _is_docstring_stmt, - _llm_name_helpers, - _module_global_names, - _run_with_timeout, - _stmts_source, - FunctionSplitter, -) - - -# --------------------------------------------------------------------------- -# Test helpers -# --------------------------------------------------------------------------- - - -def _parse_func(source: str): - """Return (body_stmts, positions, source_lines) for the first function. - - Uses a CSTVisitor to capture body_stmts from the wrapper's internal copy, - ensuring they match the keys in the positions dict. - """ - tree = cst.parse_module(source) - wrapper = MetadataWrapper(tree) - positions = wrapper.resolve(PositionProvider) - - class _Getter(cst.CSTVisitor): - METADATA_DEPENDENCIES = (PositionProvider,) - - def __init__(self): - self.stmts: list = [] - - def visit_FunctionDef(self, node: cst.FunctionDef) -> None: - if not self.stmts: # first function only - self.stmts = list(node.body.body) - - getter = _Getter() - wrapper.visit(getter) - source_lines = source.splitlines(keepends=True) - return getter.stmts, positions, source_lines - - -def _make_mock_response(names_list): - """Build a mock Anthropic message response for the name_helper_functions tool.""" - mock_block = MagicMock() - mock_block.type = "tool_use" - mock_block.name = "name_helper_functions" - mock_block.input = { - "names": [{"id": str(i), "name": n} for i, n in enumerate(names_list)] - } - mock_response = MagicMock() - mock_response.content = [mock_block] - return mock_response - - -# --------------------------------------------------------------------------- -# _is_docstring_stmt -# --------------------------------------------------------------------------- - - -def _parse_stmt(src: str) -> cst.BaseStatement: - return cst.parse_module(src).body[0] - - -def test_is_docstring_triple_quoted(): - stmt = _parse_stmt('def f():\n """doc"""\n').body.body[0] - assert _is_docstring_stmt(stmt) is True - - -def test_is_docstring_single_quoted(): - stmt = _parse_stmt("def f():\n 'doc'\n").body.body[0] - assert _is_docstring_stmt(stmt) is True - - -def test_is_docstring_concatenated(): - stmt = _parse_stmt('def f():\n "foo" "bar"\n').body.body[0] - assert _is_docstring_stmt(stmt) is True - - -def test_is_docstring_non_docstring_expr(): - # A numeric literal is not a docstring - stmt = _parse_stmt("def f():\n 42\n").body.body[0] - assert _is_docstring_stmt(stmt) is False - - -def test_is_docstring_import(): - stmt = _parse_stmt("import os\n") - assert _is_docstring_stmt(stmt) is False - - -def test_is_docstring_assignment(): - stmt = _parse_stmt("x = 1\n") - assert _is_docstring_stmt(stmt) is False - - -def test_is_docstring_two_stmts_on_line(): - # Two statements on one line — len(body) != 1 - stmt = _parse_stmt("x = 1; y = 2\n") - assert _is_docstring_stmt(stmt) is False - - -def test_is_docstring_compound_stmt(): - # A compound statement (If) is not a SimpleStatementLine - src = "def f():\n if True:\n pass\n" - stmt = cst.parse_module(src).body[0].body.body[0] - assert _is_docstring_stmt(stmt) is False - - -# --------------------------------------------------------------------------- -# _count_body_lines -# --------------------------------------------------------------------------- - - -def test_count_body_lines_no_docstring(): - src = "def foo():\n x = 1\n y = 2\n z = 3\n" - assert _count_body_lines(src) == 3 - - -def test_count_body_lines_with_docstring(): - src = 'def foo():\n """doc"""\n x = 1\n y = 2\n' - # docstring skipped; body is lines 2 (x=1) and 3 (y=2) - assert _count_body_lines(src) == 2 - - -def test_count_body_lines_multiline_docstring(): - src = 'def foo():\n """line1\n line2\n """\n x = 1\n' - # docstring spans lines 2-4; body starts at x=1 (line 5) - result = _count_body_lines(src) - assert result == 1 - - -def test_count_body_lines_only_docstring(): - # Body has only a docstring → effectively empty - src = 'def foo():\n """doc"""\n' - assert _count_body_lines(src) == 0 - - -def test_count_body_lines_parse_error(): - assert _count_body_lines("def f(\n !!invalid") == 0 - - -def test_count_body_lines_no_funcdef(): - # Module-level code, no function - assert _count_body_lines("x = 1\n") == 0 - - -# --------------------------------------------------------------------------- -# _find_free_vars -# --------------------------------------------------------------------------- - - -def test_find_free_vars_all_local(): - src = "x = 1\ny = x + 1\n" - assert _find_free_vars(src) == [] - - -def test_find_free_vars_one_free(): - src = "y = external_var + 1\n" - result = _find_free_vars(src) - assert "external_var" in result - assert "y" not in result - - -def test_find_free_vars_builtins_excluded(): - src = "print(len([1, 2, 3]))\n" - result = _find_free_vars(src) - assert "print" not in result - assert "len" not in result - - -def test_find_free_vars_nested_function_not_recursed(): - src = "def inner():\n return outer_var\n" - # outer_var is used inside nested function — not recursed into - assert _find_free_vars(src) == [] - - -def test_find_free_vars_nested_class_not_recursed(): - src = "class Inner:\n x = class_var\n" - # class_var inside nested class — not recursed - assert _find_free_vars(src) == [] - - -def test_find_free_vars_for_target_not_free(): - src = "for item in some_list:\n pass\n" - result = _find_free_vars(src) - # item is a store, some_list is a load - assert "item" not in result - assert "some_list" in result - - -def test_find_free_vars_import_not_free(): - src = "import os\npath = os.getcwd()\n" - result = _find_free_vars(src) - # os is imported (stored), path is stored - assert "os" not in result - assert "path" not in result - - -def test_find_free_vars_import_from_not_free(): - src = "from os import path\nresult = path.join('a', 'b')\n" - result = _find_free_vars(src) - assert "path" not in result - - -def test_find_free_vars_parse_error(): - assert _find_free_vars("def f(\n !!") == [] - - -def test_find_free_vars_del_is_store(): - src = "del some_name\n" - # some_name has Del context (not Load) — not treated as free - result = _find_free_vars(src) - assert "some_name" not in result - - -def test_find_free_vars_augassign_free(): - # weight += 1 reads weight before writing — weight must come from outside - src = "weight += 1\n" - result = _find_free_vars(src) - assert "weight" in result - - -def test_find_free_vars_augassign_already_defined(): - # weight is unconditionally assigned first, so AugAssign doesn't need it free - src = "weight = 0\nweight += 1\n" - result = _find_free_vars(src) - assert "weight" not in result - - -def test_find_free_vars_augassign_subscript(): - # data[0] += 1: target is a subscript, data is loaded - src = "data[0] += 1\n" - result = _find_free_vars(src) - assert "data" in result - - -def test_find_free_vars_for_orelse(): - # for-else: orelse runs when loop completes normally - src = "for item in data:\n pass\nelse:\n fallback()\n" - result = _find_free_vars(src) - assert "item" not in result # for target is locally scoped - assert "data" in result - assert "fallback" in result # used in orelse, not locally defined - - -def test_find_free_vars_with_target(): - # with-statement target is locally scoped inside the body - src = "with open(filename) as fp:\n content = fp.read()\n" - result = _find_free_vars(src) - assert "fp" not in result # with target, locally scoped - assert "filename" in result # context_expr is free - - -def test_find_free_vars_with_no_target(): - # with-statement without 'as' clause - src = "with ctx_mgr():\n do_work()\n" - result = _find_free_vars(src) - assert "ctx_mgr" in result - assert "do_work" in result - - -def test_find_free_vars_except_handler_name(): - # except-handler name is locally bound for the handler body - src = "try:\n risky()\nexcept ValueError as exc:\n handle(exc)\n" - result = _find_free_vars(src) - assert "exc" not in result # locally bound by except clause - assert "risky" in result - assert "handle" in result - - -def test_find_free_vars_except_no_name(): - # bare except without 'as' binding - src = "try:\n risky()\nexcept ValueError:\n pass\n" - result = _find_free_vars(src) - assert "risky" in result - - -def test_find_free_vars_listcomp(): - # list comprehension: loop var is locally scoped - src = "result = [x * 2 for x in data]\n" - result = _find_free_vars(src) - assert "x" not in result # comprehension target, locally scoped - assert "data" in result - - -def test_find_free_vars_listcomp_with_filter(): - # comprehension with 'if' guard: threshold must come from outside - src = "result = [x for x in data if x > threshold]\n" - result = _find_free_vars(src) - assert "x" not in result - assert "data" in result - assert "threshold" in result - - -def test_find_free_vars_dictcomp(): - # dict comprehension: both key and value expressions are walked - src = "result = {k: v for k, v in pairs}\n" - result = _find_free_vars(src) - assert "k" not in result # tuple target of comprehension - assert "v" not in result - assert "pairs" in result - - -def test_find_free_vars_tuple_for_target(): - # tuple-unpacking for target: both names locally scoped - src = "for a, b in pairs:\n use(a, b)\n" - result = _find_free_vars(src) - assert "a" not in result - assert "b" not in result - assert "pairs" in result - - -def test_find_free_vars_subscript_assign_target(): - # subscript assignment target (e.g. data[0] = 1): _target_names returns {} - # so nothing is added to definitely_defined, but data is loaded - src = "data[0] = 1\n" - result = _find_free_vars(src) - assert "data" in result # data is loaded as the subscript base - - -def test_find_free_vars_annassign_with_value(): - # annotated assignment with value: name is definitely defined afterwards - src = "x: int = 5\ny = x + 1\n" - result = _find_free_vars(src) - assert "x" not in result - assert "y" not in result - - -def test_find_free_vars_annassign_no_value(): - # annotation without assignment: x is NOT definitely defined - src = "x: int\ny = x + 1\n" - result = _find_free_vars(src) - assert "x" in result # not assigned, so it is free - - -def test_find_free_vars_annassign_non_name_target(): - # annotated assignment where target is not a plain Name - src = "obj.attr: int = 5\n" - result = _find_free_vars(src) - assert "obj" in result # obj is loaded to set the attribute - - -def test_find_free_vars_conditional_store_is_free(): - # variables only assigned inside a conditional block remain free - src = "for i in xs:\n result = f(i)\nprint(result)\n" - result = _find_free_vars(src) - assert "result" in result # conditionally assigned → still free after loop - - -def test_find_free_vars_for_body_sequential(): - # a variable assigned then used in the same for-body iteration is not free - src = "for alias in names:\n name = alias.asname\n result.add(name)\n" - result = _find_free_vars(src) - assert "name" not in result # assigned before used in same loop body - assert "names" in result - assert "result" in result - - -def test_find_free_vars_if_branch(): - # if-body assignments do not propagate to after the if block - src = "if cond:\n x = 1\nelse:\n y = 2\nz = x + y\n" - result = _find_free_vars(src) - assert "cond" in result - assert "x" in result # only conditionally defined in if body - assert "y" in result # only conditionally defined in else body - - -def test_find_free_vars_while_loop(): - # while condition is free; while-else is walked - src = "while running:\n do_work()\nelse:\n finalize()\n" - result = _find_free_vars(src) - assert "running" in result - assert "do_work" in result - assert "finalize" in result - - -def test_find_free_vars_try_propagates(): - # variables assigned in a try body propagate to code after the try block - src = textwrap.dedent( - """\ - try: - lineno = compute() - except ValueError: - return - use(lineno) - """ - ) - result = _find_free_vars(src) - assert "lineno" not in result # defined in try body, propagated outward - assert "compute" in result - assert "use" in result - - -def test_find_free_vars_try_orelse(): - # try-else clause is walked with the try-body scope (x is defined there) - src = textwrap.dedent( - """\ - try: - x = compute() - except ValueError: - return - else: - use(x) - """ - ) - result = _find_free_vars(src) - assert "x" not in result # defined in try body, visible in else clause - assert "use" in result - assert "compute" in result - - -def test_find_free_vars_try_finally(): - # try with finally and no handlers: handlers loop is empty - src = "try:\n x = compute()\nfinally:\n cleanup()\n" - result = _find_free_vars(src) - assert "compute" in result - assert "cleanup" in result - assert "x" not in result # defined in try body, propagated - - -def test_find_free_vars_bare_except(): - # bare 'except:' has node.type = None (covers the None branch) - src = "try:\n risky()\nexcept:\n pass\n" - result = _find_free_vars(src) - assert "risky" in result - - -def test_find_free_vars_lambda_param_not_free(): - # lambda parameter must not appear as a free variable - src = "result = sorted(tasks, key=lambda t: t.name)\n" - result = _find_free_vars(src) - assert "t" not in result - assert "tasks" in result - - -def test_find_free_vars_lambda_vararg_not_free(): - # *args in lambda body — args is the vararg, not free - src = "f = lambda *args: list(args)\n" - result = _find_free_vars(src) - assert "args" not in result - - -def test_find_free_vars_lambda_kwarg_not_free(): - # **kw in lambda body — kw is the kwarg, not free - src = "f = lambda **kw: kw\n" - result = _find_free_vars(src) - assert "kw" not in result - - -def test_find_free_vars_lambda_default_outer_scope(): - # Default values are evaluated in the enclosing scope, not the lambda scope. - src = "f = lambda x=outer_val: x\n" - result = _find_free_vars(src) - assert "outer_val" in result # evaluated in outer scope → free - assert "x" not in result # lambda param → not free - - -def test_find_free_vars_lambda_kw_default_none_entry(): - # keyword-only param without a default: kw_defaults has a None entry - # lambda *, x, y=outer_val: x+y → kw_defaults=[None, outer_val_node] - src = "f = lambda *, x, y=outer_val: x + y\n" - result = _find_free_vars(src) - assert "x" not in result # kwonly param → not free - assert "y" not in result # kwonly param → not free - assert "outer_val" in result # kw_default evaluated in outer scope → free - - -# --------------------------------------------------------------------------- -# _stmts_source -# --------------------------------------------------------------------------- - - -def test_stmts_source_basic(): - src = "def foo():\n x = 1\n y = 2\n z = 3\n" - stmts, positions, lines = _parse_func(src) - result = _stmts_source(stmts[:2], lines, positions) - assert "x = 1" in result - assert "y = 2" in result - assert "z = 3" not in result - - -def test_stmts_source_empty(): - src = "def foo():\n x = 1\n" - _, positions, lines = _parse_func(src) - assert _stmts_source([], lines, positions) == "" - - -def test_stmts_source_dedented(): - src = "def foo():\n x = 1\n y = 2\n" - stmts, positions, lines = _parse_func(src) - result = _stmts_source(stmts, lines, positions) - # Should be dedented (no leading 4-space indent) - assert result.startswith("x = 1") or result.startswith("x = 1\n") - - -# --------------------------------------------------------------------------- -# _head_effective_lines -# --------------------------------------------------------------------------- - - -def test_head_effective_lines_no_docstring(): - src = "def foo():\n x = 1\n y = 2\n z = 3\n" - stmts, positions, lines = _parse_func(src) - # split_idx=2: head=[x,y], last=y at line 3, first=x at line 2 → 3-2+2=3 - result = _head_effective_lines(stmts, 2, positions, False) - assert result == 3 - - -def test_head_effective_lines_with_docstring_normal(): - src = 'def foo():\n """doc"""\n x = 1\n y = 2\n z = 3\n' - stmts, positions, lines = _parse_func(src) - # split_idx=3: head=[doc, x, y], first_non_doc=x at line 3, last=y at line 4 - # 4-3+2=3 - result = _head_effective_lines(stmts, 3, positions, True) - assert result == 3 - - -def test_head_effective_lines_only_docstring_in_head(): - # split_idx=1 with docstring: first_non_doc_idx=1 >= split_idx=1 → returns 1 - src = 'def foo():\n """doc"""\n x = 1\n y = 2\n' - stmts, positions, lines = _parse_func(src) - result = _head_effective_lines(stmts, 1, positions, True) - assert result == 1 - - -# --------------------------------------------------------------------------- -# _find_valid_splits -# --------------------------------------------------------------------------- - - -def test_find_valid_splits_all_valid(): - src = "def foo():\n a = 1\n b = 2\n c = 3\n d = 4\n" - stmts, positions, lines = _parse_func(src) - # With a very loose limit, all splits should be valid - result = _find_valid_splits(stmts, positions, max_lines=1000) - assert len(result) > 0 - # Ordered latest first - assert result == sorted(result, reverse=True) - - -def test_find_valid_splits_none_valid(): - # max_lines=1 means even a 1-stmt head (+ return call = 2 lines) is invalid - src = "def foo():\n a = 1\n b = 2\n c = 3\n" - stmts, positions, lines = _parse_func(src) - result = _find_valid_splits(stmts, positions, max_lines=1) - assert result == [] - - -def test_find_valid_splits_stops_at_max_candidates(): - # 7 statements → iterates from 6 down, stops after 5 valid candidates - src = "def foo():\n" + "".join(f" a{i} = {i}\n" for i in range(7)) - stmts, positions, lines = _parse_func(src) - result = _find_valid_splits(stmts, positions, max_lines=1000) - assert len(result) == 5 - - -def test_find_valid_splits_fewer_than_max(): - # 4 statements → at most 3 valid splits (indices 3, 2, 1) - src = "def foo():\n a = 1\n b = 2\n c = 3\n d = 4\n" - stmts, positions, lines = _parse_func(src) - result = _find_valid_splits(stmts, positions, max_lines=1000) - assert 1 <= len(result) <= 3 - - -def test_find_valid_splits_empty_body(): - # Should not crash with an empty list (though normally not called) - result = _find_valid_splits([], {}, max_lines=1000) - assert result == [] - - -def test_find_valid_splits_nested_funcdef_restricts_upper(): - # First nested funcdef at index 2 → valid splits only at indices ≤ 2. - src = textwrap.dedent( - """\ - def outer(): - a = 1 - b = 2 - def inner(): - pass - c = 3 - d = 4 - """ - ) - stmts, positions, lines = _parse_func(src) - # body_stmts: [a=1, b=2, def inner, c=3, d=4] - # First nested funcdef at index 2 → upper=2 → range(2, 0, -1) = [2, 1] - result = _find_valid_splits(stmts, positions, max_lines=1000) - assert all(i <= 2 for i in result) - assert 3 not in result - assert 4 not in result - - -# --------------------------------------------------------------------------- -# _choose_best_split -# --------------------------------------------------------------------------- - - -def test_choose_best_split_fewest_params(): - # Two splits: one has free vars, one doesn't - src = textwrap.dedent( - """\ - def foo(external): - a = 1 - b = external + 1 - """ - ) - stmts, positions, lines = _parse_func(src) - # split_idx=1: tail=[b=external+1] → free vars: [external] - # split_idx=2: tail=[] → but we need at least 1 stmt in tail, - # so valid splits are [1] only for 2-stmt function - # Let's use 3 stmts with different free var counts - src2 = textwrap.dedent( - """\ - def foo(ext): - a = 1 - b = ext + 1 - c = a + b - """ - ) - stmts2, positions2, lines2 = _parse_func(src2) - # split_idx=1: tail=[b=ext+1, c=a+b] → free vars: [a, ext] (a from head) - # Actually 'a' is assigned in head (split_idx=1 → head=[a=1]) and used in tail - # So tail [b=ext+1, c=a+b] has free vars: [a, ext] - # split_idx=2: tail=[c=a+b] → free vars: [a, b] (assigned in head) - # Wait no, head=[a=1, b=ext+1] so tail=[c=a+b] has free vars: [a, b] - # split_idx=3: not valid (needs at least 1 in tail) - # So split_idx=1 has 2 free vars [a, ext], split_idx=2 has 2 free vars [a, b] - # Tie → choose earliest in list = latest split = 2 - valid_splits = [2, 1] # latest first - split_idx, params, _ = _choose_best_split( - stmts2, valid_splits, lines2, positions2, ["ext"] - ) - # Both have 2 free vars, tie broken by latest (first in list) = 2 - assert split_idx == 2 - - -def test_choose_best_split_fewer_params_wins(): - # Use a source where one split clearly has fewer params - src = textwrap.dedent( - """\ - def foo(): - a = 1 - b = 2 - c = a + b - """ - ) - stmts, positions, lines = _parse_func(src) - # split_idx=1: tail=[b=2, c=a+b] → free vars: [a] (1 free var) - # split_idx=2: tail=[c=a+b] → free vars: [a, b] (2 free vars) - valid_splits = [2, 1] - split_idx, params, _ = _choose_best_split(stmts, valid_splits, lines, positions, []) - # split_idx=1 has 1 free var (a) vs split_idx=2 has 2 free vars (a, b) - assert split_idx == 1 - assert params == ["a"] - - -def test_choose_best_split_single_candidate(): - src = "def foo():\n x = 1\n y = 2\n" - stmts, positions, lines = _parse_func(src) - split_idx, params, _ = _choose_best_split(stmts, [1], lines, positions, []) - assert split_idx == 1 - - -def test_choose_best_split_self_in_tail_returns_instance_method(): - # Tail requires self → extracted as instance method, not static - src = textwrap.dedent( - """\ - class Foo: - def method(self, x): - a = 1 - b = self.value + a - """ - ) - stmts, positions, lines = _parse_func(src) - # split_idx=1: tail=[b = self.value + a] → free: [a, self] → instance method - result = _choose_best_split(stmts, [1], lines, positions, ["self", "x"]) - assert result is not None - split_idx, params, is_instance_method = result - assert split_idx == 1 - assert is_instance_method is True - assert "self" not in params # self is implicit, not in params list - assert "a" in params # a is still a real param - - -def test_choose_best_split_empty_splits_returns_none(): - # No valid split candidates → None returned - src = "def foo():\n x = 1\n y = 2\n" - stmts, positions, lines = _parse_func(src) - result = _choose_best_split(stmts, [], lines, positions, []) - assert result is None - - -def test_choose_best_split_filters_module_globals(): - # Tail references a module-level import; it must not appear in params. - src = textwrap.dedent( - """\ - def foo(): - x = 1 - y = os.path.join("a", "b") - """ - ) - stmts, positions, lines = _parse_func(src) - # Without filtering: "os" would be a free var of the tail. - # With module_globals={"os"}: "os" is filtered out → params = [] - result = _choose_best_split(stmts, [1], lines, positions, [], module_globals={"os"}) - assert result is not None - _, params, _ = result - assert "os" not in params - - -# --------------------------------------------------------------------------- -# _module_global_names -# --------------------------------------------------------------------------- - - -def test_module_global_names_imports(): - source = "import ast\nfrom pathlib import Path\nimport libcst as cst\n" - result = _module_global_names(source) - assert "ast" in result - assert "Path" in result - assert "cst" in result - - -def test_module_global_names_functions_and_classes(): - source = "def foo():\n pass\n\nclass Bar:\n pass\n" - result = _module_global_names(source) - assert "foo" in result - assert "Bar" in result - - -def test_module_global_names_assignments(): - source = "_CONST = frozenset()\nVALUE: int = 42\n" - result = _module_global_names(source) - assert "_CONST" in result - assert "VALUE" in result - - -def test_module_global_names_syntax_error(): - result = _module_global_names("def foo(") - assert result == set() - - -def test_module_global_names_tuple_assign_target_not_collected(): - # Tuple-unpacking: Assign target is a Tuple node, not a Name → skipped - source = "a, b = 1, 2\n" - result = _module_global_names(source) - assert "a" not in result - assert "b" not in result - - -def test_module_global_names_ann_assign_non_name_target_skipped(): - # AnnAssign where target is an Attribute, not a Name → skipped - source = "Foo.x: int\n" - result = _module_global_names(source) - assert "x" not in result - - -# --------------------------------------------------------------------------- -# _generate_helper_source -# --------------------------------------------------------------------------- - - -def test_generate_helper_source_with_staticmethod(): - result = _generate_helper_source( - name="process", - params=["x", "y"], - tail_source="return x + y\n", - func_indent=" ", - is_static=True, - add_docstring=False, - ) - assert "@staticmethod" in result - assert "def _process(x, y):" in result - assert "return x + y" in result - assert result.startswith(" @staticmethod") - - -def test_generate_helper_source_without_staticmethod(): - result = _generate_helper_source( - name="process", - params=["x"], - tail_source="return x * 2\n", - func_indent="", - is_static=False, - add_docstring=False, - ) - assert "@staticmethod" not in result - assert "def _process(x):" in result - assert "return x * 2" in result - - -def test_generate_helper_source_with_docstring(): - result = _generate_helper_source( - name="process", - params=[], - tail_source="return 42\n", - func_indent="", - is_static=False, - add_docstring=True, - ) - assert '"""' in result - assert "return 42" in result - - -def test_generate_helper_source_instance_method(): - result = _generate_helper_source( - name="process", - params=["a"], - tail_source="return self.x + a\n", - func_indent=" ", - is_static=False, - add_docstring=False, - is_instance_method=True, - ) - assert "@staticmethod" not in result - assert "def _process(self, a):" in result - assert "return self.x + a" in result - - -def test_generate_helper_source_indentation_correct(): - result = _generate_helper_source( - name="helper", - params=[], - tail_source="x = 1\ny = 2\n", - func_indent=" ", - is_static=False, - add_docstring=False, - ) - # Body should be indented by 8 spaces (func_indent=4 + body_indent=4) - assert " x = 1" in result - assert " y = 2" in result - - -# --------------------------------------------------------------------------- -# _generate_call -# --------------------------------------------------------------------------- - - -def test_generate_call_with_class(): - result = _generate_call("helper", ["x", "y"], "MyClass", " ") - assert result == " return MyClass._helper(x, y)" - - -def test_generate_call_module_level(): - result = _generate_call("helper", ["a"], None, " ") - assert result == " return _helper(a)" - - -def test_generate_call_no_params(): - result = _generate_call("do_work", [], None, " ") - assert result == " return _do_work()" - - -def test_generate_call_class_no_params(): - result = _generate_call("do_work", [], "Foo", " ") - assert result == " return Foo._do_work()" - - -def test_generate_call_instance_method(): - result = _generate_call("process", ["a", "b"], "MyClass", " ", True) - assert result == " return self._process(a, b)" - - -def test_generate_call_instance_method_no_params(): - result = _generate_call("process", [], "MyClass", " ", True) - assert result == " return self._process()" - - -# --------------------------------------------------------------------------- -# _has_yield -# --------------------------------------------------------------------------- - - -def test_has_yield_simple(): - src = "def gen():\n yield 1\n" - func = cst.parse_module(src).body[0] - assert _has_yield(func) is True - - -def test_has_yield_from(): - src = "def gen():\n yield from [1, 2]\n" - func = cst.parse_module(src).body[0] - assert _has_yield(func) is True - - -def test_has_yield_none(): - src = "def foo():\n return 1\n" - func = cst.parse_module(src).body[0] - assert _has_yield(func) is False - - -def test_has_yield_nested_not_counted(): - src = textwrap.dedent( - """\ - def foo(): - def inner(): - yield 1 - return inner - """ - ) - func = cst.parse_module(src).body[0] - # yield is inside nested function, should not count - assert _has_yield(func) is False - - -# --------------------------------------------------------------------------- -# _has_nested_funcdef -# --------------------------------------------------------------------------- - - -def test_has_nested_funcdef_with_nested(): - src = textwrap.dedent( - """\ - def outer(): - x = 1 - def inner(): - return x - return inner - """ - ) - func = cst.parse_module(src).body[0] - assert _has_nested_funcdef(func) is True - - -def test_has_nested_funcdef_without_nested(): - src = "def foo():\n x = 1\n return x\n" - func = cst.parse_module(src).body[0] - assert _has_nested_funcdef(func) is False - - -def test_has_nested_funcdef_first_stmt(): - # Nested funcdef is the very first statement in the body - src = textwrap.dedent( - """\ - def outer(): - def inner(): - pass - return inner() - """ - ) - func = cst.parse_module(src).body[0] - assert _has_nested_funcdef(func) is True - - -# --------------------------------------------------------------------------- -# _run_with_timeout -# --------------------------------------------------------------------------- - - -def test_run_with_timeout_success(): - result = _run_with_timeout(lambda x: x * 2, 5, 21) - assert result == 42 - - -def test_run_with_timeout_exceeds(): - import time - - with pytest.raises(_ApiTimeout): - _run_with_timeout(lambda: time.sleep(10), timeout=0.05) - - -def test_run_with_timeout_propagates_exception(): - def _raise(): - raise ValueError("test error") - - with pytest.raises(ValueError, match="test error"): - _run_with_timeout(_raise, 5) - - -# --------------------------------------------------------------------------- -# _func_in_changed_range / _extract_func_source -# --------------------------------------------------------------------------- - - -def _make_func_info(start, end): - """Create a minimal _FuncInfo for range tests.""" - mock_node = MagicMock() - return _FuncInfo( - node=mock_node, - start_line=start, - end_line=end, - class_name=None, - indent="", - original_params=[], - ) - - -def test_func_in_changed_range_overlaps(): - fi = _make_func_info(5, 15) - assert _func_in_changed_range(fi, [(1, 10)]) is True - - -def test_func_in_changed_range_no_overlap(): - fi = _make_func_info(5, 10) - assert _func_in_changed_range(fi, [(20, 30)]) is False - - -def test_func_in_changed_range_adjacent(): - fi = _make_func_info(5, 10) - assert _func_in_changed_range(fi, [(10, 20)]) is True - - -def test_extract_func_source(): - lines = ["line1\n", "line2\n", "line3\n", "line4\n"] - fi = _make_func_info(2, 3) - result = _extract_func_source(fi, lines) - assert result == "line2\nline3\n" - - -# --------------------------------------------------------------------------- -# _FunctionCollector -# --------------------------------------------------------------------------- - - -def test_function_collector_module_level(): - src = "def foo():\n x = 1\n" - tree = cst.parse_module(src) - wrapper = MetadataWrapper(tree) - collector = _FunctionCollector() - wrapper.visit(collector) - assert len(collector.functions) == 1 - assert collector.functions[0].node.name.value == "foo" - assert collector.functions[0].class_name is None - assert collector.functions[0].indent == "" - - -def test_function_collector_class_method(): - src = textwrap.dedent( - """\ - class Foo: - def bar(self): - pass - """ - ) - tree = cst.parse_module(src) - wrapper = MetadataWrapper(tree) - collector = _FunctionCollector() - wrapper.visit(collector) - assert len(collector.functions) == 1 - assert collector.functions[0].class_name == "Foo" - assert collector.functions[0].indent == " " - - -def test_function_collector_skips_async(): - src = "async def foo():\n pass\n" - tree = cst.parse_module(src) - wrapper = MetadataWrapper(tree) - collector = _FunctionCollector() - wrapper.visit(collector) - assert len(collector.functions) == 0 - - -def test_function_collector_skips_generator(): - src = "def gen():\n yield 1\n" - tree = cst.parse_module(src) - wrapper = MetadataWrapper(tree) - collector = _FunctionCollector() - wrapper.visit(collector) - assert len(collector.functions) == 0 - - -def test_function_collector_skips_nested_functions(): - # Functions with nested funcdefs are skipped entirely; inner functions - # (inside a function scope) are also skipped by the scope-kind guard. - src = textwrap.dedent( - """\ - def outer(): - def inner(): - pass - return inner - """ - ) - tree = cst.parse_module(src) - wrapper = MetadataWrapper(tree) - collector = _FunctionCollector() - wrapper.visit(collector) - # outer has a nested funcdef → skipped; inner is in a function scope → skipped - assert len(collector.functions) == 0 - - -def test_function_collector_captures_params(): - src = "def foo(a, b, c):\n pass\n" - tree = cst.parse_module(src) - wrapper = MetadataWrapper(tree) - collector = _FunctionCollector() - wrapper.visit(collector) - assert collector.functions[0].original_params == ["a", "b", "c"] - - -# --------------------------------------------------------------------------- -# _llm_name_helpers -# --------------------------------------------------------------------------- - - -def _make_task(func_name, params=None, tail_source="return 0\n"): - """Create a minimal _SplitTask for testing _llm_name_helpers.""" - mock_node = MagicMock() - mock_node.name.value = func_name - fi = _FuncInfo( - node=mock_node, - start_line=1, - end_line=5, - class_name=None, - indent="", - original_params=[], - ) - return _SplitTask(fi, 1, params or [], tail_source=tail_source) - - -@patch("crispen.llm_client.anthropic") -def test_llm_name_helpers_success(mock_anthropic): - mock_response = _make_mock_response(["process_tail"]) - mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response - - tasks = [_make_task("my_func")] - client = mock_anthropic.Anthropic.return_value - result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) - assert result == ["process_tail"] - - -@patch("crispen.llm_client.anthropic") -def test_llm_name_helpers_result_none(mock_anthropic): - # LLM returns no tool use block - mock_response = MagicMock() - mock_response.content = [] - mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response - - tasks = [_make_task("my_func")] - client = mock_anthropic.Anthropic.return_value - result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) - # Falls back to "my_func_helper" - assert result == ["my_func_helper"] - - -@patch("crispen.llm_client.anthropic") -def test_llm_name_helpers_no_names_key(mock_anthropic): - # LLM returns tool use but without "names" key - mock_block = MagicMock() - mock_block.type = "tool_use" - mock_block.name = "name_helper_functions" - mock_block.input = {"something_else": []} - mock_response = MagicMock() - mock_response.content = [mock_block] - mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response - - tasks = [_make_task("my_func")] - client = mock_anthropic.Anthropic.return_value - result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) - assert result == ["my_func_helper"] - - -@patch("crispen.llm_client.anthropic") -def test_llm_name_helpers_strips_leading_underscore(mock_anthropic): - mock_response = _make_mock_response(["__private_name"]) - mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response - - tasks = [_make_task("foo")] - client = mock_anthropic.Anthropic.return_value - result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) - assert result == ["private_name"] - - -@patch("crispen.llm_client.anthropic") -def test_llm_name_helpers_all_underscores_uses_helper(mock_anthropic): - mock_response = _make_mock_response(["___"]) - mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response - - tasks = [_make_task("foo")] - client = mock_anthropic.Anthropic.return_value - result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) - assert result == ["helper"] - - -@patch("crispen.llm_client.anthropic") -def test_llm_name_helpers_bad_item_skipped(mock_anthropic): - # One item has a TypeError (e.g. name is not a string) - mock_block = MagicMock() - mock_block.type = "tool_use" - mock_block.name = "name_helper_functions" - mock_block.input = { - "names": [{"id": "0", "name": None}] # None.lstrip() raises AttributeError - } - mock_response = MagicMock() - mock_response.content = [mock_block] - mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response - - tasks = [_make_task("foo")] - client = mock_anthropic.Anthropic.return_value - result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", tasks) - # Falls back to "foo_helper" because item had AttributeError - assert result == ["foo_helper"] - - -@patch("crispen.llm_client.anthropic") -def test_llm_name_helpers_with_class_name(mock_anthropic): - mock_response = _make_mock_response(["process"]) - mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response - - mock_node = MagicMock() - mock_node.name.value = "method" - fi = _FuncInfo( - node=mock_node, - start_line=1, - end_line=5, - class_name="MyClass", - indent=" ", - original_params=[], - ) - task = _SplitTask(fi, 1, [], tail_source="return 0\n") - client = mock_anthropic.Anthropic.return_value - result = _llm_name_helpers(client, "claude-sonnet-4-6", "anthropic", [task]) - assert result == ["process"] - - -# --------------------------------------------------------------------------- -# FunctionSplitter — integration tests -# --------------------------------------------------------------------------- - - -def _make_long_func(n_stmts: int, func_name: str = "long_func") -> str: - """Build a function with n_stmts independent assignments.""" - lines = [f"def {func_name}():\n"] - for i in range(n_stmts): - lines.append(f" a{i} = {i}\n") - lines.append(" return 0\n") - return "".join(lines) - - -def test_function_splitter_under_limits_no_op(): - # A small function should not be split - src = "def small():\n x = 1\n return x\n" - splitter = FunctionSplitter([(1, 10)], source=src, verbose=False) - assert splitter.get_rewritten_source() is None - - -def test_function_splitter_parse_error_no_crash(): - # Invalid source should not crash - splitter = FunctionSplitter([(1, 10)], source="def f(\n !!invalid", verbose=False) - assert splitter.get_rewritten_source() is None - - -def test_function_splitter_out_of_range_no_op(): - # Function exists but is outside changed ranges - src = _make_long_func(80) - splitter = FunctionSplitter([(200, 300)], source=src, verbose=False, max_lines=10) - assert splitter.get_rewritten_source() is None - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_over_line_limit(mock_anthropic): - mock_anthropic.Anthropic.return_value.messages.create.return_value = ( - _make_mock_response(["process_tail"]) - ) - src = _make_long_func(80) - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], - source=src, - verbose=False, - max_lines=50, - ) - - result = splitter.get_rewritten_source() - assert result is not None - compile(result, "", "exec") - assert "_process_tail" in result - assert "return _process_tail(" in result - assert len(splitter.changes_made) >= 1 - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_nested_funcdef_not_split(mock_anthropic): - # A long function containing a nested funcdef should never be split, - # even if it far exceeds the line limit. Splitting across a closure - # boundary produces cascading re-splits and semantically fragile helpers. - lines = ["def func_with_closure():\n"] - for i in range(80): - lines.append(f" a{i} = {i}\n") - lines.append(" def inner():\n") - lines.append(" return 0\n") - lines.append(" return inner()\n") - src = "".join(lines) - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], source=src, verbose=False, max_lines=10 - ) - - assert splitter.get_rewritten_source() is None - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_async_skipped(mock_anthropic): - # Async functions should not be split - src = ( - "async def foo():\n" - + "".join(f" a{i} = {i}\n" for i in range(80)) - + " return 0\n" - ) - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], source=src, verbose=False, max_lines=10 - ) - - assert splitter.get_rewritten_source() is None - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_generator_skipped(mock_anthropic): - # Generator functions should not be split - src = ( - "def gen():\n" - + "".join(f" a{i} = {i}\n" for i in range(80)) - + " yield 0\n" - ) - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], source=src, verbose=False, max_lines=10 - ) - - assert splitter.get_rewritten_source() is None - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_llm_fallback_on_api_error(mock_anthropic): - # API key not set → get_api_key raises CrispenAPIError → fallback names used - mock_anthropic.Anthropic.return_value.messages.create.return_value = ( - _make_mock_response(["tail"]) - ) - src = _make_long_func(60, "my_func") - - # No ANTHROPIC_API_KEY → get_api_key raises → fallback to "my_func_helper" - with patch.dict("os.environ", {}, clear=True): - # Remove any existing API key - import os - - os.environ.pop("ANTHROPIC_API_KEY", None) - splitter = FunctionSplitter( - [(1, 1000)], source=src, verbose=False, max_lines=30 - ) - - result = splitter.get_rewritten_source() - assert result is not None - compile(result, "", "exec") - # Fallback name used: "my_func_helper" - assert "_my_func_helper" in result - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_recursive_split(mock_anthropic): - # With small max_lines and broad changed_ranges, triggers multiple iterations - # First call names helper for first function, second call for helper - mock_anthropic.Anthropic.return_value.messages.create.side_effect = [ - _make_mock_response(["part1"]), - _make_mock_response(["part2"]), - _make_mock_response(["part3"]), - ] - - # 13 body statements → with max_lines=5, needs multiple splits - src = _make_long_func(13, "func") - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], # broad range covers all helpers too - source=src, - verbose=False, - max_lines=5, - ) - - result = splitter.get_rewritten_source() - assert result is not None - compile(result, "", "exec") - # Multiple splits occurred - assert len(splitter.changes_made) >= 2 - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_syntax_error_in_output_is_skipped(mock_anthropic): - # If the assembled edit is invalid Python, the change is not applied - # We simulate this by making _generate_call return something invalid - # Instead, test the path via a function with 1-stmt body (no valid split) - src = "def foo():\n x = 1\n" # only 1 stmt → can't split - splitter = FunctionSplitter([(1, 10)], source=src, verbose=False, max_lines=0) - # body lines=1 > 0=max_lines → tries to split but len(body_stmts)=1 < 2 → skip - assert splitter.get_rewritten_source() is None - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_no_valid_split_skipped(mock_anthropic): - # max_lines=1 → even a head with 1 stmt (+return call=2) > max_lines=1 - # So no valid splits → no change - src = _make_long_func(5, "foo") - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter([(1, 1000)], source=src, verbose=False, max_lines=1) - - assert splitter.get_rewritten_source() is None - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_with_helper_docstrings(mock_anthropic): - mock_anthropic.Anthropic.return_value.messages.create.return_value = ( - _make_mock_response(["process"]) - ) - src = _make_long_func(80) - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], - source=src, - verbose=False, - max_lines=50, - helper_docstrings=True, - ) - - result = splitter.get_rewritten_source() - assert result is not None - assert '"""' in result - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_class_method(mock_anthropic): - mock_anthropic.Anthropic.return_value.messages.create.return_value = ( - _make_mock_response(["tail_work"]) - ) - lines = ["class Foo:\n", " def method(self):\n"] - for i in range(80): - lines.append(f" a{i} = {i}\n") - lines.append(" return 0\n") - src = "".join(lines) - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], - source=src, - verbose=False, - max_lines=50, - ) - - result = splitter.get_rewritten_source() - assert result is not None - compile(result, "", "exec") - # Class methods use staticmethod and ClassName._ call - assert "@staticmethod" in result - assert "Foo._tail_work(" in result - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_llm_timeout_fallback(mock_anthropic): - # LLM call times out → fallback names - from crispen.refactors.function_splitter import _ApiTimeout - - mock_anthropic.Anthropic.return_value.messages.create.side_effect = _ApiTimeout( - "timed out" - ) - src = _make_long_func(60, "slow_func") - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], - source=src, - verbose=False, - max_lines=30, - ) - - result = splitter.get_rewritten_source() - assert result is not None - compile(result, "", "exec") - # Fallback name "slow_func_helper" used - assert "_slow_func_helper" in result - - -# --------------------------------------------------------------------------- -# FunctionSplitter — additional branch coverage tests -# --------------------------------------------------------------------------- - - -def test_function_splitter_empty_source(): - """FunctionSplitter created with no source does nothing.""" - splitter = FunctionSplitter([(1, 10)]) - assert splitter.get_rewritten_source() is None - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_max_iterations_loop_exhausted(mock_anthropic): - """Loop runs to completion (no break) when max iterations reached.""" - mock_anthropic.Anthropic.return_value.messages.create.return_value = ( - _make_mock_response(["helper"]) - ) - src = _make_long_func(80, "foo") - - # Patch _MAX_SPLIT_ITERATIONS to 1 → loop runs exactly once without breaking - # (break only occurs at START of next iteration when tasks=[]) - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - with patch("crispen.refactors.function_splitter._MAX_SPLIT_ITERATIONS", 1): - splitter = FunctionSplitter( - [(1, 1000)], source=src, verbose=False, max_lines=50 - ) - - result = splitter.get_rewritten_source() - assert result is not None - assert len(splitter.changes_made) == 1 - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_syntax_error_in_generated_output(mock_anthropic): - """If assembled output fails compile(), the change is not applied.""" - mock_anthropic.Anthropic.return_value.messages.create.return_value = ( - _make_mock_response(["helper"]) - ) - src = _make_long_func(80, "foo") - - import builtins as _builtins - - orig_compile = _builtins.compile - - def _selective_compile(source, filename, mode, *args, **kwargs): - if filename == "": - raise SyntaxError("mocked error for test") - return orig_compile(source, filename, mode, *args, **kwargs) - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - with patch("builtins.compile", side_effect=_selective_compile): - splitter = FunctionSplitter( - [(1, 1000)], source=src, verbose=False, max_lines=50 - ) - - assert splitter.get_rewritten_source() is None - - -def test_find_free_vars_del_context(): - """del statement adds name to stores (else branch for non-Load contexts).""" - src = "del my_var\n" - result = _find_free_vars(src) - assert "my_var" not in result - - -# --------------------------------------------------------------------------- -# _has_new_undefined_names -# --------------------------------------------------------------------------- - - -def test_has_new_undefined_names_no_new(): - """No new undefined names → returns False.""" - before = "x = 1\ny = x + 1\n" - after = "x = 1\ny = x + 1\nz = y + 1\n" - assert _has_new_undefined_names(before, after) is False - - -def test_has_new_undefined_names_introduced(): - """After introduces an undefined name that before didn't have → returns True.""" - before = "x = 1\n" - after = "x = undefined_var\n" - assert _has_new_undefined_names(before, after) is True - - -def test_has_new_undefined_names_non_undefined_warning(): - """Non-UndefinedName pyflakes warning (e.g. UnusedImport) → returns False.""" - # An unused import produces an UnusedImport warning, not UndefinedName. - # This exercises the isinstance() False branch inside _Collector.flake. - before = "" - after = "import os\n" - assert _has_new_undefined_names(before, after) is False - - -def test_has_new_undefined_names_exception(): - """If pyflakes raises an unexpected exception, returns False (safe default).""" - with patch("pyflakes.api.check", side_effect=RuntimeError("boom")): - assert _has_new_undefined_names("x = 1\n", "y = 1\n") is False - - -@patch( - "crispen.refactors.function_splitter._has_new_undefined_names", return_value=True -) -@patch("crispen.llm_client.anthropic") -def test_function_splitter_pyflakes_rejects_output(mock_anthropic, mock_has_undef): - """If pyflakes detects new undefined names in output, the split is not applied.""" - mock_anthropic.Anthropic.return_value.messages.create.return_value = ( - _make_mock_response(["helper"]) - ) - src = _make_long_func(80, "foo") - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], source=src, verbose=False, max_lines=50 - ) - - # Pyflakes check returned True → split not applied - assert splitter.get_rewritten_source() is None - - -# --------------------------------------------------------------------------- -# Engine integration: FunctionSplitter branch is exercised -# --------------------------------------------------------------------------- - - -def test_engine_includes_function_splitter_no_op(tmp_path): - """FunctionSplitter is in _REFACTORS and runs without error for simple files.""" - from crispen.engine import run_engine - from crispen.config import CrispenConfig - - py_file = tmp_path / "sample.py" - py_file.write_text("def foo():\n return 1\n") - config = CrispenConfig(max_function_length=75) - msgs = list(run_engine({str(py_file): [(1, 2)]}, verbose=False, config=config)) - # No split needed — no messages expected (or just no errors) - assert all("FunctionSplitter" not in m for m in msgs) - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_method_self_needed_uses_instance_method(mock_anthropic): - """When every tail needs self, split into a regular instance method helper.""" - mock_anthropic.Anthropic.return_value.messages.create.return_value = ( - _make_mock_response(["tail_work"]) - ) - lines = ["class Foo:\n", " def method(self):\n"] - for i in range(40): - lines.append(f" a{i} = self.val + {i}\n") - lines.append(" return 0\n") - src = "".join(lines) - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], source=src, verbose=False, max_lines=20 - ) - - result = splitter.get_rewritten_source() - assert result is not None - compile(result, "", "exec") - assert "@staticmethod" not in result - assert "return self._tail_work(" in result - assert "def _tail_work(self" in result - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_skips_name_collision(mock_anthropic): - """Helper name colliding with an existing function causes the task to be dropped.""" - mock_anthropic.Anthropic.return_value.messages.create.return_value = ( - _make_mock_response(["helper"]) # would produce _helper - ) - # _helper already exists; the LLM would name the extracted helper "helper" - existing = "def _helper():\n pass\n\n\n" - src = existing + _make_long_func(80) - - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], source=src, verbose=False, max_lines=50 - ) - - # collision detected → task dropped → no rewrite - assert splitter.get_rewritten_source() is None - - -# --------------------------------------------------------------------------- -# _llm_name_helpers with _timing_out -# --------------------------------------------------------------------------- - - -@patch("crispen.llm_client.anthropic") -def test_llm_name_helpers_with_timing_out(mock_anthropic): - """_llm_name_helpers appends result to _timing_out when provided.""" - mock_response = _make_mock_response(["process_tail"]) - mock_anthropic.Anthropic.return_value.messages.create.return_value = mock_response - mock_anthropic.APIError = Exception - - tasks = [_make_task("my_func")] - client = mock_anthropic.Anthropic.return_value - timing: list = [] - result = _llm_name_helpers( - client, "claude-sonnet-4-6", "anthropic", tasks, _timing_out=timing - ) - assert result == ["process_tail"] - assert len(timing) == 1 - - -# --------------------------------------------------------------------------- -# FunctionSplitter-level timing branch (if timing:) -# --------------------------------------------------------------------------- - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_timing_recorded(mock_anthropic): - """FunctionSplitter records LLM timing after a successful split.""" - mock_anthropic.Anthropic.return_value.messages.create.return_value = ( - _make_mock_response(["process_tail"]) - ) - mock_anthropic.APIError = Exception - - src = _make_long_func(80) - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - splitter = FunctionSplitter( - [(1, 1000)], source=src, verbose=False, max_lines=50 - ) - - # The timing branch was hit; record_llm_call ran for the edit call. - assert splitter.stats.llm_edit_calls >= 1 - # The elapsed time dict was populated. - assert "edit" in splitter.stats.llm_elapsed_by_category - - -# --------------------------------------------------------------------------- -# FunctionSplitter-level timing == "detailed" verbose print -# --------------------------------------------------------------------------- - - -@patch("crispen.llm_client.anthropic") -def test_function_splitter_detailed_timing_print(mock_anthropic, capsys): - """FunctionSplitter prints per-call timing in verbose + detailed mode.""" - mock_client = mock_anthropic.Anthropic.return_value - mock_client.messages.create.return_value = _make_mock_response(["process_tail"]) - mock_anthropic.APIError = Exception - - src = _make_long_func(80) - with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "test-key"}): - # Construct without source so _analyze is not called yet. - splitter = FunctionSplitter([(1, 1000)], source="", verbose=True, max_lines=50) - splitter.timing = "detailed" - # Now trigger _analyze with detailed timing in place. - splitter._analyze(src) - - err = capsys.readouterr().err - assert "→ naming [" in err diff --git a/tests/test_patch_rewriter.py b/tests/test_patch_rewriter.py index 4a02437..cc7832e 100644 --- a/tests/test_patch_rewriter.py +++ b/tests/test_patch_rewriter.py @@ -1,7737 +1,3 @@ """Tests for patch_rewriter — 100% branch coverage.""" from __future__ import annotations - -from unittest.mock import MagicMock, patch as mock_patch - -import libcst as cst - -from crispen.config import CrispenConfig -from crispen.llm_client import LLMCallResult -from crispen.patch_rewriter import ( - _ConstRef, - _FLContext, - RewriteAccumulator, - _CgIndex, - _CG_CANDIDATES_LLM_THRESHOLD, - _CG_MAX_DEPTH, - _CG_MAX_MODULES, - _apply_cross_file_const_updates, - _expand_module_terminals, - _build_attr_const_map, - _build_classify_prompt, - _build_const_map, - _build_context_message, - _build_func_verify_prompt, - _build_local_const_map, - _build_no_change_verify_prompt, - _build_rewrite_func_prompt, - _build_rewrite_verify_prompt, - _callgraph_update_file, - _candidates_check, - _patch_strings_in_text, - _rewrite_candidates_check, - _cg_build_index, - _cg_collect_called_names, - _cg_collect_defined_names, - _cg_collect_func_body_calls, - _cg_file_to_module_and_package, - _cg_parse_imports, - _cg_resolve_call_to_import, - _compiles, - _extract_migration_reminder, - _extract_patch_lookup, - _build_rename_guard_sets, - _extract_still_imported_names, - _is_bad_rename, - _find_test_functions_to_update, - _find_with_patch_paths_in_body, - _import_header, - _is_patch_call, - _get_external_import_names, - _name_reference_map, - _matches_any, - _process_file_source, - _resolve_forking_path_candidates, - _resolve_forking_path_via_callgraph, - _resolve_import_to_file, - _splice_function, - _restore_const_refs, - _get_const_votes_from_rewrite, - _substitute_consts_in_func_text, - apply_patch_callgraph, - apply_patch_rewrite, -) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _ok(tool_input=None) -> LLMCallResult: - return LLMCallResult( - tool_input=tool_input, elapsed=0.0, input_tokens=0, output_tokens=0 - ) - - -def _truncated_ok() -> LLMCallResult: - """Simulate a truncated verify response (tool_input=None, truncated=True).""" - return LLMCallResult( - tool_input=None, elapsed=0.0, input_tokens=0, output_tokens=0, truncated=True - ) - - -def _make_fl_ctx(**kwargs) -> _FLContext: - defaults = dict( - filepath="/proj/pkg/big.py", - old_module="pkg.big", - original_source="class A: pass\nclass B: pass\n", - modified_source="from .sub_a import A\nfrom .sub_b import B\n", - new_files={"sub_a.py": "class A: pass\n", "sub_b.py": "class B: pass\n"}, - new_module_paths={"sub_a.py": "pkg.sub_a", "sub_b.py": "pkg.sub_b"}, - entity_to_target={"A": "sub_a.py", "B": "sub_b.py"}, - forking_old_paths={"pkg.big.A", "pkg.big.B"}, - ) - defaults.update(kwargs) - return _FLContext(**defaults) - - -_CFG = CrispenConfig(patch_update_retries=1) -_CFG_NO_LLM_VERIFY = CrispenConfig(patch_update_retries=1, llm_verify_retries=0) -_FORKING_PATHS = {"crispen.before.X"} -_SRC_WITH_PATCH = '@patch("crispen.before.X")\ndef test_f(mock_x):\n pass\n' - -_PATCH_GET_KEY = "crispen.patch_rewriter.get_api_key" -_PATCH_MAKE_CLIENT = "crispen.patch_rewriter.make_client" -_PATCH_CALL_TOOL = "crispen.patch_rewriter.call_with_tool" - -# Shorthand classify tool_inputs. -_CLASSIFY_RENAME = { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.after.X"}, -} -_CLASSIFY_NO_CHANGE = {"needs_rewrite": False, "patch_renames": {}} -_CLASSIFY_REWRITE = {"needs_rewrite": True} -_VERIFY_OK = {"correct": True, "issue": ""} -_VERIFY_REJECT = {"correct": False, "issue": "wrong path"} -_VERIFY_REJECT_WITH_CORRECTIONS = { - "correct": False, - "issue": "wrong path", - "corrections": {"crispen.before.X": "crispen.after.X"}, -} -_REWRITE_VERIFY_OK = {"correct": True, "issue": ""} -_REWRITE_VERIFY_REJECT = {"correct": False, "issue": "wrong mock setup"} - - -# --------------------------------------------------------------------------- -# _is_patch_call -# --------------------------------------------------------------------------- - - -def test_is_patch_call_name_match(): - call_node = cst.parse_expression('patch("foo")') - assert _is_patch_call(call_node) is True - - -def test_is_patch_call_attribute_match(): - call_node = cst.parse_expression('mock.patch("foo")') - assert _is_patch_call(call_node) is True - - -def test_is_patch_call_other_name(): - call_node = cst.parse_expression('other("foo")') - assert _is_patch_call(call_node) is False - - -# --------------------------------------------------------------------------- -# _matches_any -# --------------------------------------------------------------------------- - - -def test_matches_any_exact(): - assert _matches_any("a.b.C", {"a.b.C"}) is True - - -def test_matches_any_prefix(): - assert _matches_any("a.b.C.method", {"a.b.C"}) is True - - -def test_matches_any_near_miss(): - # "a.b.CExtra" should NOT match "a.b.C" - assert _matches_any("a.b.CExtra", {"a.b.C"}) is False - - -def test_matches_any_no_match(): - assert _matches_any("x.y.Z", {"a.b.C"}) is False - - -# --------------------------------------------------------------------------- -# _compiles -# --------------------------------------------------------------------------- - - -def test_compiles_valid(): - assert _compiles("x = 1\n") is True - - -def test_compiles_invalid(): - assert _compiles("def f(:\n pass\n") is False - - -# --------------------------------------------------------------------------- -# _find_test_functions_to_update -# --------------------------------------------------------------------------- - - -def test_find_empty_old_paths(): - src = '@patch("crispen.before.X")\ndef test_f(): pass\n' - assert _find_test_functions_to_update(src, set()) == [] - - -def test_find_parse_error(): - assert _find_test_functions_to_update("def f(:\n", {"crispen.before.X"}) == [] - - -def test_find_no_match(): - src = '@patch("other.mod.Y")\ndef test_f(): pass\n' - assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] - - -def test_find_match_exact(): - src = '@patch("crispen.before.X")\ndef test_f(): pass\n' - result = _find_test_functions_to_update(src, {"crispen.before.X"}) - assert len(result) == 1 - assert result[0].function_name == "test_f" - assert "crispen.before.X" in result[0].old_patch_paths - - -def test_find_match_prefix(): - src = '@patch("crispen.before.X.method")\ndef test_f(): pass\n' - result = _find_test_functions_to_update(src, {"crispen.before.X"}) - assert len(result) == 1 - assert "crispen.before.X.method" in result[0].old_patch_paths - - -def test_find_not_a_call_decorator(): - # @patch used as a bare name (no parentheses), not a Call node. - src = "@patch\ndef test_f(): pass\n" - assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] - - -def test_find_no_args(): - src = "@patch()\ndef test_f(): pass\n" - assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] - - -def test_find_arg_not_simple_string(): - # @patch(some_variable) — first arg is a Name, not a SimpleString. - src = "@patch(some_var)\ndef test_f(): pass\n" - assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] - - -def test_find_prefixed_string(): - # b"..." — raw[0] is 'b', not a quote character. - src = '@patch(b"crispen.before.X")\ndef test_f(): pass\n' - assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] - - -def test_find_triple_quoted(): - src = '@patch("""crispen.before.X""")\ndef test_f(): pass\n' - assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] - - -def test_find_not_patch_name(): - # @decorate("crispen.before.X") — attribute name is not "patch". - src = '@decorate("crispen.before.X")\ndef test_f(): pass\n' - assert _find_test_functions_to_update(src, {"crispen.before.X"}) == [] - - -def test_find_attribute_patch(): - # @mock.patch("crispen.before.X") — Attribute form. - src = '@mock.patch("crispen.before.X")\ndef test_f(): pass\n' - result = _find_test_functions_to_update(src, {"crispen.before.X"}) - assert len(result) == 1 - assert result[0].function_name == "test_f" - - -def test_find_multiple_functions(): - src = ( - '@patch("crispen.before.X")\ndef test_a(): pass\n\n' - '@patch("crispen.before.Y")\ndef test_b(): pass\n' - ) - result = _find_test_functions_to_update( - src, {"crispen.before.X", "crispen.before.Y"} - ) - assert {f.function_name for f in result} == {"test_a", "test_b"} - - -def test_find_full_text_includes_decorator(): - src = '@patch("crispen.before.X")\ndef test_f():\n pass\n' - result = _find_test_functions_to_update(src, {"crispen.before.X"}) - assert '@patch("crispen.before.X")' in result[0].full_text - assert "def test_f" in result[0].full_text - - -def test_find_start_end_lines(): - # line 1: # header, line 2: @patch..., line 3: def test_f, line 4: pass - src = "# header\n" '@patch("crispen.before.X")\n' "def test_f():\n" " pass\n" - result = _find_test_functions_to_update(src, {"crispen.before.X"}) - assert result[0].start_line == 2 # @patch line (first decorator) - assert result[0].end_line == 4 # last line of body - - -def test_find_body_with_patch_no_decorator(): - # Function has no @patch decorator but uses ``with patch(...)`` in the body. - src = ( - "def test_f():\n" ' with patch("crispen.before.X") as m:\n' " pass\n" - ) - result = _find_test_functions_to_update(src, {"crispen.before.X"}) - assert len(result) == 1 - assert result[0].function_name == "test_f" - assert "crispen.before.X" in result[0].old_patch_paths - # start_line should be the ``def`` line (no decorators). - assert result[0].start_line == 1 - - -def test_find_body_with_patch_combined_with_decorator(): - # Function has both an @patch decorator and a body-level with patch(...). - src = ( - '@patch("crispen.before.Y")\n' - "def test_f(mock_y):\n" - ' with patch("crispen.before.X") as m:\n' - " pass\n" - ) - result = _find_test_functions_to_update( - src, {"crispen.before.X", "crispen.before.Y"} - ) - assert len(result) == 1 - paths = result[0].old_patch_paths - assert "crispen.before.X" in paths - assert "crispen.before.Y" in paths - - -# --------------------------------------------------------------------------- -# _find_with_patch_paths_in_body -# --------------------------------------------------------------------------- - - -def test_body_scan_syntax_error(): - assert _find_with_patch_paths_in_body("def f(:\n", {"old.X"}, {}, {}) == [] - - -def test_body_scan_no_funcdef(): - # Parsed text has no FunctionDef at the top level. - assert _find_with_patch_paths_in_body("x = 1\n", {"old.X"}, {}, {}) == [] - - -def test_body_scan_simple_match(): - src = 'def test_f():\n with patch("old.X") as m:\n pass\n' - result = _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) - assert result == ["old.X"] - - -def test_body_scan_no_match(): - src = 'def test_f():\n with patch("other.Y") as m:\n pass\n' - assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] - - -def test_body_scan_attribute_patch(): - # ``with mock.patch(...)`` form. - src = 'def test_f():\n with mock.patch("old.X") as m:\n pass\n' - result = _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) - assert result == ["old.X"] - - -def test_body_scan_not_patch_call(): - src = 'def test_f():\n with other("old.X") as m:\n pass\n' - assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] - - -def test_body_scan_no_args(): - src = "def test_f():\n with patch() as m:\n pass\n" - assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] - - -def test_body_scan_non_call_context_manager(): - # Context manager is a plain Name, not a Call. - src = "def test_f():\n with ctx as m:\n pass\n" - assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] - - -def test_body_scan_non_string_arg(): - # First arg is a Call expression (not string/Name/Attribute). - src = "def test_f():\n with patch(get_target()) as m:\n pass\n" - assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] - - -def test_body_scan_name_const_match(): - const_map = {"MY_TARGET": ("old.X", "/file.py")} - src = "def test_f():\n with patch(MY_TARGET) as m:\n pass\n" - result = _find_with_patch_paths_in_body(src, {"old.X"}, const_map, {}) - assert result == ["old.X"] - - -def test_body_scan_name_const_no_match(): - # Constant value doesn't match old_paths. - const_map = {"MY_TARGET": ("other.Y", "/file.py")} - src = "def test_f():\n with patch(MY_TARGET) as m:\n pass\n" - assert _find_with_patch_paths_in_body(src, {"old.X"}, const_map, {}) == [] - - -def test_body_scan_name_not_in_const_map(): - src = "def test_f():\n with patch(unknown_var) as m:\n pass\n" - assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] - - -def test_body_scan_attr_const_match(): - attr_const_map = {"consts": {"TARGET": ("old.X", "/consts.py")}} - src = "def test_f():\n with patch(consts.TARGET) as m:\n pass\n" - result = _find_with_patch_paths_in_body(src, {"old.X"}, {}, attr_const_map) - assert result == ["old.X"] - - -def test_body_scan_attr_const_module_not_in_map(): - src = "def test_f():\n with patch(unknown_mod.X) as m:\n pass\n" - assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] - - -def test_body_scan_attr_const_attr_not_in_map(): - attr_const_map = {"consts": {"OTHER": ("old.X", "/consts.py")}} - src = "def test_f():\n with patch(consts.MISSING) as m:\n pass\n" - assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, attr_const_map) == [] - - -def test_body_scan_attr_const_no_match(): - # Attribute constant value doesn't match old_paths. - attr_const_map = {"consts": {"TARGET": ("other.Y", "/consts.py")}} - src = "def test_f():\n with patch(consts.TARGET) as m:\n pass\n" - assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, attr_const_map) == [] - - -def test_body_scan_nested_funcdef_excluded(): - # ``with patch(...)`` inside a nested function should NOT trigger inclusion of - # the outer function — the nested function is its own unit. - src = ( - "def test_outer():\n" - " def inner():\n" - ' with patch("old.X") as m:\n' - " pass\n" - ) - assert _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) == [] - - -def test_body_scan_multiple_with_items(): - # ``with patch("a") as m, patch("b") as n:`` — both items should be found. - src = ( - "def test_f():\n" - ' with patch("old.X") as m, patch("old.Y") as n:\n' - " pass\n" - ) - result = _find_with_patch_paths_in_body(src, {"old.X", "old.Y"}, {}, {}) - assert set(result) == {"old.X", "old.Y"} - - -def test_body_scan_async_with(): - src = 'async def test_f():\n async with patch("old.X") as m:\n pass\n' - result = _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) - assert result == ["old.X"] - - -def test_body_scan_nested_in_if(): - # ``with patch(...)`` inside an ``if`` block should still be found. - src = ( - "def test_f():\n" - " if True:\n" - ' with patch("old.X") as m:\n' - " pass\n" - ) - result = _find_with_patch_paths_in_body(src, {"old.X"}, {}, {}) - assert result == ["old.X"] - - -# --------------------------------------------------------------------------- -# _build_context_message -# --------------------------------------------------------------------------- - - -def test_build_context_no_diff(): - # Diff is no longer included — only imports header and entity migration. - ctx = _make_fl_ctx() - msg = _build_context_message([ctx]) - assert "```diff" not in msg - - -def test_build_context_new_file_imports_and_refs(): - # New-file section shows imports header and name-reference map; no bodies. - src = "import os\n\ndef my_func():\n os.path.join('a', 'b')\n" - ctx = _make_fl_ctx(new_files={"sub_a.py": src, "sub_b.py": "class B: pass\n"}) - msg = _build_context_message([ctx]) - assert "**Imports:**" in msg - assert "import os" in msg - assert "**Name references**" in msg - assert "`os`: `my_func`" in msg - assert "def my_func" not in msg # body not included - - -def test_build_context_entity_migration_present(): - ctx = _make_fl_ctx() - msg = _build_context_message([ctx]) - assert "sub_a.py" in msg - assert "pkg.sub_a" in msg - - -def test_build_context_empty_new_files_and_entities(): - # Covers the zero-iteration branches of the two for-loops. - ctx = _make_fl_ctx(new_files={}, new_module_paths={}, entity_to_target={}) - msg = _build_context_message([ctx]) - assert "Split module" in msg - assert "Entity migration" in msg - - -def test_build_context_multiple_contexts(): - ctx1 = _make_fl_ctx(old_module="pkg.big", filepath="/p/pkg/big.py") - ctx2 = _make_fl_ctx(old_module="pkg.large", filepath="/p/pkg/large.py") - msg = _build_context_message([ctx1, ctx2]) - assert "pkg.big" in msg - assert "pkg.large" in msg - - -# --------------------------------------------------------------------------- -# _import_header -# --------------------------------------------------------------------------- - - -def test_import_header_stops_before_def(): - src = "import os\nfrom x import y\n\ndef foo():\n pass\n" - assert _import_header(src) == "import os\nfrom x import y\n" - - -def test_import_header_stops_before_class(): - src = "import os\n\nclass Foo:\n pass\n" - assert _import_header(src) == "import os\n" - - -def test_import_header_stops_before_async_def(): - src = "import os\nasync def foo(): pass\n" - assert _import_header(src) == "import os\n" - - -def test_import_header_no_defs_returns_all(): - src = "import os\nfrom x import y\n" - assert _import_header(src) == "import os\nfrom x import y\n" - - -def test_import_header_empty_source(): - assert _import_header("") == "" - - -def test_import_header_strips_trailing_blanks(): - src = "import os\n\n\ndef foo(): pass\n" - assert _import_header(src) == "import os\n" - - -# --------------------------------------------------------------------------- -# _name_reference_map -# --------------------------------------------------------------------------- - - -def test_name_reference_map_basic(): - src = ( - "import os\n" - "from x import Foo\n" - "\n" - "def alpha():\n" - " os.getcwd()\n" - " Foo()\n" - "\n" - "def beta():\n" - " os.path.join('a', 'b')\n" - ) - refs = _name_reference_map(src) - assert refs["os"] == ["alpha", "beta"] - assert refs["Foo"] == ["alpha"] - - -def test_name_reference_map_alias(): - src = "import libcst as cst\n\ndef run():\n cst.parse_module('x')\n" - refs = _name_reference_map(src) - assert refs["cst"] == ["run"] - - -def test_name_reference_map_unused_import(): - # Imported but never referenced in a function body → absent from map. - src = "import os\n\ndef alpha():\n pass\n" - refs = _name_reference_map(src) - assert "os" not in refs - - -def test_name_reference_map_no_imports(): - src = "def alpha():\n x = 1\n" - assert _name_reference_map(src) == {} - - -def test_name_reference_map_star_import_ignored(): - # ``from x import *`` should not add anything (alias.name == "*" branch). - src = "from x import *\n\ndef alpha():\n foo()\n" - refs = _name_reference_map(src) - assert refs == {} - - -def test_name_reference_map_syntax_error(): - assert _name_reference_map("def (broken:") == {} - - -def test_name_reference_map_class(): - src = ( - "from x import Dep\n" - "\n" - "class MyClass:\n" - " def method(self):\n" - " return Dep()\n" - ) - refs = _name_reference_map(src) - assert refs["Dep"] == ["MyClass"] - - -# --------------------------------------------------------------------------- -# _splice_function -# --------------------------------------------------------------------------- - - -def test_splice_function_basic(): - source = "line1\nline2\nline3\nline4\n" - result = _splice_function(source, 2, 3, "new2\nnew3\n") - assert result == "line1\nnew2\nnew3\nline4\n" - - -def test_splice_function_single_line(): - source = "line1\nline2\nline3\n" - result = _splice_function(source, 2, 2, "replacement\n") - assert result == "line1\nreplacement\nline3\n" - - -def test_splice_function_size_change(): - # Replace 1 line with 3 lines. - source = "a\nb\nc\n" - result = _splice_function(source, 2, 2, "x\ny\nz\n") - assert result == "a\nx\ny\nz\nc\n" - - -def test_splice_function_no_trailing_newline(): - # new_func_text without trailing newline gets one added. - source = "a\nb\nc\n" - result = _splice_function(source, 2, 2, "replacement") - assert result == "a\nreplacement\nc\n" - - -def test_splice_function_empty_new_text(): - # Empty string: no trailing newline added (falsy check), splitlines gives []. - source = "a\nb\nc\n" - result = _splice_function(source, 2, 2, "") - assert result == "a\nc\n" - - -# --------------------------------------------------------------------------- -# _extract_migration_reminder -# --------------------------------------------------------------------------- - - -def test_extract_migration_reminder_basic(): - ctx_msg = _build_context_message([_make_fl_ctx()]) - reminder = _extract_migration_reminder(ctx_msg) - assert "Entity migration (quick reference)" in reminder - assert "pkg.sub_a" in reminder - assert "pkg.sub_b" in reminder - - -def test_extract_migration_reminder_empty_context(): - reminder = _extract_migration_reminder("no migration here") - assert reminder == "" - - -def test_extract_migration_reminder_no_entities(): - ctx = _make_fl_ctx(entity_to_target={}, new_module_paths={}) - ctx_msg = _build_context_message([ctx]) - # Empty entity_to_target → no bullets → reminder is empty string - reminder = _extract_migration_reminder(ctx_msg) - assert reminder == "" - - -def test_extract_migration_reminder_heading_stops_capture(): - # When a second fl_context follows the first, a new ## heading appears after - # the entity migration section — the extractor must stop capturing there. - ctx1 = _make_fl_ctx(old_module="pkg.big", filepath="/p/pkg/big.py") - ctx2 = _make_fl_ctx(old_module="pkg.large", filepath="/p/pkg/large.py") - ctx_msg = _build_context_message([ctx1, ctx2]) - reminder = _extract_migration_reminder(ctx_msg) - # The reminder should contain migration bullets from both contexts but - # not any heading markers. - assert "### Entity migration:" not in reminder - assert "## Split module:" not in reminder - assert "pkg.sub_a" in reminder - - -# --------------------------------------------------------------------------- -# _get_external_import_names -# --------------------------------------------------------------------------- - - -def test_get_external_import_names_absolute(): - src = "from pkg import Foo\nimport os\n" - names = _get_external_import_names(src) - assert "Foo" in names - assert "os" in names - - -def test_get_external_import_names_level1_skipped(): - src = "from .sub import Bar\nfrom . import Baz\n" - names = _get_external_import_names(src) - assert names == set() - - -def test_get_external_import_names_level2_included(): - src = "from ..pkg import Foo\nfrom ...llm_client import call_with_tool\n" - names = _get_external_import_names(src) - assert "Foo" in names - assert "call_with_tool" in names - - -def test_get_external_import_names_star_import_skipped(): - src = "from pkg import *\n" - names = _get_external_import_names(src) - assert names == set() - - -def test_get_external_import_names_asname(): - src = "import libcst as cst\nfrom pkg import Foo as F\n" - names = _get_external_import_names(src) - assert "cst" in names - assert "F" in names - assert "libcst" not in names - assert "Foo" not in names - - -def test_get_external_import_names_syntax_error(): - assert _get_external_import_names("def (broken:") == set() - - -# --------------------------------------------------------------------------- -# _extract_patch_lookup -# --------------------------------------------------------------------------- - - -def _make_ctx_with_ext_imports() -> _FLContext: - """Context where original_source has real external imports that moved.""" - orig = "from ...llm_client import call_with_tool\ndef foo(): pass\n" - mod = "from .llm_planning import call_with_tool\n" - new_files = { - "llm_planning.py": ( - "from ...llm_client import call_with_tool\ndef advise(): call_with_tool()\n" - ) - } - return _make_fl_ctx( - original_source=orig, - modified_source=mod, - new_files=new_files, - new_module_paths={"llm_planning.py": "pkg.llm_planning"}, - entity_to_target={"advise": "llm_planning.py"}, - ) - - -def test_extract_patch_lookup_basic(): - ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) - lookup = _extract_patch_lookup(ctx_msg) - assert "Patch target lookup" in lookup - assert "call_with_tool" in lookup - assert "pkg.llm_planning" in lookup - - -def test_extract_patch_lookup_no_section(): - # Default fixture has no external imports → no lookup section generated. - ctx_msg = _build_context_message([_make_fl_ctx()]) - assert _extract_patch_lookup(ctx_msg) == "" - - -def test_extract_patch_lookup_multiple_contexts(): - ctx1 = _make_ctx_with_ext_imports() - orig2 = "from ...config import CrispenConfig\ndef bar(): pass\n" - mod2 = "from .cfg import CrispenConfig\n" - new2 = {"cfg.py": "from ...config import CrispenConfig\ndef run(): pass\n"} - ctx2 = _make_fl_ctx( - old_module="pkg.other", - filepath="/proj/pkg/other.py", - original_source=orig2, - modified_source=mod2, - new_files=new2, - new_module_paths={"cfg.py": "pkg.cfg"}, - entity_to_target={"run": "cfg.py"}, - ) - ctx_msg = _build_context_message([ctx1, ctx2]) - lookup = _extract_patch_lookup(ctx_msg) - assert "call_with_tool" in lookup - assert "CrispenConfig" in lookup - - -def test_extract_patch_lookup_still_in_section(): - # Name in both original and modified → appears under "still imported". - orig = "from ...llm_client import call_with_tool, make_client\ndef foo(): pass\n" - mod = ( - "from ...llm_client import make_client\n" - "from .llm_planning import call_with_tool\n" - ) - new_files = { - "llm_planning.py": ( - "from ...llm_client import call_with_tool\ndef advise(): pass\n" - ) - } - ctx = _make_fl_ctx( - original_source=orig, - modified_source=mod, - new_files=new_files, - new_module_paths={"llm_planning.py": "pkg.llm_planning"}, - entity_to_target={"advise": "llm_planning.py"}, - ) - ctx_msg = _build_context_message([ctx]) - lookup = _extract_patch_lookup(ctx_msg) - assert "call_with_tool" in lookup - assert "make_client" in lookup - assert "still" in lookup - - -def test_extract_patch_lookup_name_not_in_new_files(): - # Name moved out but not found in any new file → "(not found in new files)". - orig = "from ...llm_client import call_with_tool\ndef foo(): pass\n" - mod = "" # name removed - new_files = {"sub.py": "class X: pass\n"} # no imports - ctx = _make_fl_ctx( - original_source=orig, - modified_source=mod, - new_files=new_files, - new_module_paths={"sub.py": "pkg.sub"}, - entity_to_target={}, - ) - ctx_msg = _build_context_message([ctx]) - lookup = _extract_patch_lookup(ctx_msg) - assert "not found in new files" in lookup - - -# --------------------------------------------------------------------------- -# _extract_still_imported_names -# --------------------------------------------------------------------------- - - -def test_extract_still_imported_names_basic(): - ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) - # _make_ctx_with_ext_imports has call_with_tool moved out — not still imported. - names = _extract_still_imported_names(ctx_msg) - assert "call_with_tool" not in names - - -def test_extract_still_imported_names_finds_retained(): - # Build a context where a name is retained in the modified original. - orig = "from ...llm_client import call_with_tool, make_client\ndef foo(): pass\n" - mod = ( - "from ...llm_client import make_client\n" - "from .llm_planning import call_with_tool\n" - ) - new_files = { - "llm_planning.py": ( - "from ...llm_client import call_with_tool\ndef advise(): pass\n" - ) - } - ctx = _make_fl_ctx( - original_source=orig, - modified_source=mod, - new_files=new_files, - new_module_paths={"llm_planning.py": "pkg.llm_planning"}, - entity_to_target={"advise": "llm_planning.py"}, - ) - ctx_msg = _build_context_message([ctx]) - names = _extract_still_imported_names(ctx_msg) - assert "make_client" in names - assert "call_with_tool" not in names - - -def test_extract_still_imported_names_no_section(): - # No lookup section in context → empty set. - names = _extract_still_imported_names("no relevant section here") - assert names == set() - - -def test_extract_still_imported_names_section_ends_at_non_bullet(): - # Section capture stops when a non-bullet line is encountered. - ctx_msg = ( - "Names still externally imported in the modified original (check):\n" - "- `alpha`\n" - "- `beta`\n" - "\n" # blank line — not a bullet, stops capture - "- `gamma`\n" # not captured - ) - names = _extract_still_imported_names(ctx_msg) - assert "alpha" in names - assert "beta" in names - assert "gamma" not in names - - -def test_extract_still_imported_names_malformed_bullet_ignored(): - # A bullet that starts with "- `" but has no closing backtick is silently skipped. - ctx_msg = ( - "Names still externally imported in the modified original (check):\n" - "- `valid`\n" - "- `\n" # malformed — no closing backtick → end <= 3 branch - ) - names = _extract_still_imported_names(ctx_msg) - assert "valid" in names - assert len(names) == 1 - - -# --------------------------------------------------------------------------- -# _build_rename_guard_sets -# --------------------------------------------------------------------------- - - -def test_build_rename_guard_sets_moved_out(): - # call_with_tool is in original_source but removed from modified_source. - ctx = _make_fl_ctx( - original_source="from ...llm_client import call_with_tool\ndef f(): pass\n", - modified_source="from .sub import call_with_tool\n", - new_files={"sub.py": "from ...llm_client import call_with_tool\n"}, - ) - moved_out, still_in, orig_users, new_mod_imports = _build_rename_guard_sets([ctx]) - assert "call_with_tool" in moved_out - assert "call_with_tool" not in still_in - - -def test_build_rename_guard_sets_still_imported(): - # make_client stays in modified_source as an external import. - ctx = _make_fl_ctx( - original_source=( - "from ...llm_client import make_client, call_with_tool\n" - "def advise(): make_client()\n" - ), - modified_source=( - "from ...llm_client import make_client\ndef advise(): make_client()\n" - ), - new_files={"sub.py": "from ...llm_client import call_with_tool\n"}, - ) - moved_out, still_in, orig_users, new_mod_imports = _build_rename_guard_sets([ctx]) - assert "make_client" in still_in - assert "call_with_tool" in moved_out - assert "make_client" not in moved_out - - -def test_build_rename_guard_sets_orig_users_map(): - # make_client is still imported and used by advise in modified_source. - ctx = _make_fl_ctx( - original_source="from ...llm_client import make_client\ndef advise(): pass\n", - modified_source=( - "from ...llm_client import make_client\ndef advise(): make_client()\n" - ), - new_files={}, - ) - _, _, orig_users, *_ = _build_rename_guard_sets([ctx]) - assert orig_users.get("make_client") == ["advise"] - - -def test_build_rename_guard_sets_no_users_not_in_map(): - # make_client is still imported but not referenced by any top-level def. - ctx = _make_fl_ctx( - original_source="from ...llm_client import make_client\ndef advise(): pass\n", - modified_source="from ...llm_client import make_client\ndef advise(): pass\n", - new_files={}, - ) - _, _, orig_users, *_ = _build_rename_guard_sets([ctx]) - assert "make_client" not in orig_users - - -def test_build_rename_guard_sets_empty_contexts(): - moved_out, still_in, orig_users, new_mod_imports = _build_rename_guard_sets([]) - assert moved_out == set() - assert still_in == set() - assert orig_users == {} - assert new_mod_imports == {} - - -def test_build_rename_guard_sets_merges_multiple_contexts(): - # Two contexts each contributing one still-in name with users. - ctx1 = _make_fl_ctx( - original_source="from ...a import foo\ndef f1(): foo()\n", - modified_source="from ...a import foo\ndef f1(): foo()\n", - new_files={}, - ) - ctx2 = _make_fl_ctx( - original_source="from ...b import bar\ndef f2(): bar()\n", - modified_source="from ...b import bar\ndef f2(): bar()\n", - new_files={}, - ) - _, still_in, orig_users, *_ = _build_rename_guard_sets([ctx1, ctx2]) - assert "foo" in still_in - assert "bar" in still_in - assert orig_users["foo"] == ["f1"] - assert orig_users["bar"] == ["f2"] - - -def test_build_rename_guard_sets_deduplicates_merged_users(): - # Same name+user in two contexts → appears once in orig_users_map. - ctx1 = _make_fl_ctx( - original_source="from ...a import foo\ndef f1(): foo()\n", - modified_source="from ...a import foo\ndef f1(): foo()\n", - new_files={}, - ) - ctx2 = _make_fl_ctx( - original_source="from ...a import foo\ndef f1(): foo()\n", - modified_source="from ...a import foo\ndef f1(): foo()\n", - new_files={}, - ) - _, _, orig_users, *_ = _build_rename_guard_sets([ctx1, ctx2]) - assert orig_users["foo"].count("f1") == 1 - - -# --------------------------------------------------------------------------- -# _is_bad_rename -# --------------------------------------------------------------------------- - - -def test_is_bad_rename_pattern_a_shallowing_moved_out(): - # advisor.placement.call_with_tool → advisor.call_with_tool - # call_with_tool is moved out; new_depth < old_depth → bad - assert _is_bad_rename( - "crispen.advisor.placement.call_with_tool", - "crispen.advisor.call_with_tool", - moved_out_names={"call_with_tool"}, - still_imported=set(), - orig_users_map={}, - test_text="", - ) - - -def test_is_bad_rename_pattern_a_deepening_moved_out_ok(): - # Deepening a moved-out name is fine (not shallowing). - assert not _is_bad_rename( - "crispen.advisor.call_with_tool", - "crispen.advisor.placement.call_with_tool", - moved_out_names={"call_with_tool"}, - still_imported=set(), - orig_users_map={}, - test_text="", - ) - - -def test_is_bad_rename_pattern_b_deepening_still_in_with_orig_user_in_test(): - # advisor.make_client → advisor.placement.make_client - # make_client is still_imported, orig user advise_file_limiter is in test body → bad - assert _is_bad_rename( - "crispen.advisor.make_client", - "crispen.advisor.placement.make_client", - moved_out_names=set(), - still_imported={"make_client"}, - orig_users_map={"make_client": ["advise_file_limiter"]}, - test_text="def test_foo():\n advise_file_limiter(src)\n", - ) - - -def test_is_bad_rename_pattern_b_deepening_still_in_no_orig_user_in_test(): - # Same deepening but test body doesn't contain advise_file_limiter → ok - assert not _is_bad_rename( - "crispen.advisor.make_client", - "crispen.advisor.placement.make_client", - moved_out_names=set(), - still_imported={"make_client"}, - orig_users_map={"make_client": ["advise_file_limiter"]}, - test_text="def test_foo():\n _propose_files_step(src)\n", - ) - - -def test_is_bad_rename_pattern_b_deepening_no_orig_users_map(): - # Name is still_imported but not in orig_users_map → not blocked - assert not _is_bad_rename( - "crispen.advisor.make_client", - "crispen.advisor.placement.make_client", - moved_out_names=set(), - still_imported={"make_client"}, - orig_users_map={}, - test_text="def test_foo():\n advise_file_limiter(src)\n", - ) - - -def test_is_bad_rename_not_bad_when_no_relevant_sets(): - assert not _is_bad_rename( - "a.b.foo", - "a.b.c.foo", - moved_out_names=set(), - still_imported=set(), - orig_users_map={}, - test_text="", - ) - - -def test_is_bad_rename_pattern_c_target_module_missing_name(): - # Target module "pkg.advisor.placement" exists in new_module_imports - # but doesn't import call_with_tool; name is in moved_out_names → bad rename. - assert _is_bad_rename( - "pkg.advisor.call_with_tool", - "pkg.advisor.placement.call_with_tool", - moved_out_names={"call_with_tool"}, - still_imported=set(), - orig_users_map={}, - test_text="", - new_module_imports={"pkg.advisor.placement": {"make_client"}}, - ) - - -def test_is_bad_rename_pattern_c_target_module_has_name(): - # Target module imports the name → not blocked by Pattern C. - assert not _is_bad_rename( - "pkg.advisor.call_with_tool", - "pkg.advisor.placement.call_with_tool", - moved_out_names={"call_with_tool"}, - still_imported=set(), - orig_users_map={}, - test_text="", - new_module_imports={"pkg.advisor.placement": {"call_with_tool"}}, - ) - - -def test_is_bad_rename_pattern_c_target_module_unknown(): - # Target module not in new_module_imports (unknown module) → not blocked. - assert not _is_bad_rename( - "pkg.advisor.call_with_tool", - "pkg.advisor.placement.call_with_tool", - moved_out_names={"call_with_tool"}, - still_imported=set(), - orig_users_map={}, - test_text="", - new_module_imports={"pkg.advisor.schemas": {"call_with_tool"}}, - ) - - -def test_is_bad_rename_pattern_c_name_not_tracked(): - # Name is not in moved_out_names or still_imported → Pattern C skipped - # even if the target module doesn't import it (locally-defined symbols). - assert not _is_bad_rename( - "pkg.big.A", - "pkg.sub_a.A", - moved_out_names=set(), - still_imported=set(), - orig_users_map={}, - test_text="", - new_module_imports={"pkg.sub_a": set()}, - ) - - -def test_is_bad_rename_pattern_c_none_new_module_imports(): - # new_module_imports=None (not passed) → Pattern C skipped entirely. - assert not _is_bad_rename( - "pkg.advisor.call_with_tool", - "pkg.advisor.placement.call_with_tool", - moved_out_names={"call_with_tool"}, - still_imported=set(), - orig_users_map={}, - test_text="", - new_module_imports=None, - ) - - -def test_build_rename_guard_sets_new_module_imports(): - # new_files with known module paths populate new_module_imports correctly. - ctx = _make_fl_ctx( - original_source="from ...llm_client import call_with_tool, make_client\n", - modified_source="from .placement import call_with_tool\n", - new_files={ - "placement.py": "from ...llm_client import call_with_tool\n", - "schemas.py": "from ...llm_client import make_client\n", - }, - new_module_paths={ - "placement.py": "pkg.advisor.placement", - "schemas.py": "pkg.advisor.schemas", - }, - ) - _, _, _, new_mod_imports = _build_rename_guard_sets([ctx]) - assert new_mod_imports["pkg.advisor.placement"] == {"call_with_tool"} - assert new_mod_imports["pkg.advisor.schemas"] == {"make_client"} - - -# --------------------------------------------------------------------------- -# _build_context_message: patch target lookup section -# --------------------------------------------------------------------------- - - -def test_build_context_lookup_present_when_names_moved(): - ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) - assert "Patch target lookup" in ctx_msg - assert "call_with_tool" in ctx_msg - - -def test_build_context_lookup_annotates_using_entities(): - # When a moved-out name is used by a top-level entity in a new file, the - # lookup entry should include "used by: " so the LLM can pick the - # right sub-module when the name appears in multiple new files. - orig = "from ...llm_client import call_with_tool\ndef foo(): pass\n" - mod = "from .sub import call_with_tool\n" - new_files = { - "sub.py": ( - "from ...llm_client import call_with_tool\n" - "def _do_work(): call_with_tool()\n" - ) - } - ctx = _make_fl_ctx( - original_source=orig, - modified_source=mod, - new_files=new_files, - new_module_paths={"sub.py": "pkg.sub"}, - entity_to_target={"_do_work": "sub.py"}, - ) - ctx_msg = _build_context_message([ctx]) - assert "used by" in ctx_msg - assert "_do_work" in ctx_msg - - -def test_build_context_lookup_no_using_entities_when_name_unused(): - # If a moved-out name is imported but not referenced by any top-level entity, - # the entry should not include a "used by" annotation. - orig = "from ...llm_client import call_with_tool\ndef foo(): pass\n" - mod = "from .sub import call_with_tool\n" - new_files = {"sub.py": "from ...llm_client import call_with_tool\n"} - ctx = _make_fl_ctx( - original_source=orig, - modified_source=mod, - new_files=new_files, - new_module_paths={"sub.py": "pkg.sub"}, - entity_to_target={}, - ) - ctx_msg = _build_context_message([ctx]) - assert "used by" not in ctx_msg - - -def test_build_context_lookup_absent_when_no_ext_imports(): - # Default fixture has class defs only — no external imports. - ctx_msg = _build_context_message([_make_fl_ctx()]) - assert "Patch target lookup" not in ctx_msg - - -def test_build_context_lookup_only_still_in(): - # All external imports preserved in modified original → only "still imported" - # section, no "moved" section. Covers the if moved_out: False branch. - # sub.py does NOT import make_client → "NOT imported in any new submodule". - orig = "from ...llm_client import make_client\ndef foo(): pass\n" - mod = "from ...llm_client import make_client\nfrom .sub import helper\n" - ctx = _make_fl_ctx( - original_source=orig, - modified_source=mod, - new_files={"sub.py": "def helper(): pass\n"}, - new_module_paths={"sub.py": "pkg.sub"}, - entity_to_target={"helper": "sub.py"}, - ) - ctx_msg = _build_context_message([ctx]) - assert "Patch target lookup" in ctx_msg - assert "still" in ctx_msg - assert "moved" not in ctx_msg - assert "NOT imported in any new submodule" in ctx_msg - - -def test_build_context_lookup_still_in_also_in_new_submodule_with_users(): - # A still-in name imported by a new submodule whose entity USES it → - # annotation shows "used by" and the migration-based guidance. - orig = "from ...llm_client import make_client\ndef foo(): pass\n" - mod = "from ...llm_client import make_client\nfrom .sub import helper\n" - ctx = _make_fl_ctx( - original_source=orig, - modified_source=mod, - new_files={ - "sub.py": ( - "from ...llm_client import make_client\n" - "def helper(): make_client()\n" - ) - }, - new_module_paths={"sub.py": "pkg.sub"}, - entity_to_target={"helper": "sub.py"}, - ) - ctx_msg = _build_context_message([ctx]) - assert "also externally imported in" in ctx_msg - assert "pkg.sub" in ctx_msg - assert "used by" in ctx_msg - assert "helper" in ctx_msg - assert "migrated to that submodule" in ctx_msg - assert "Name references" in ctx_msg - - -def test_build_context_lookup_still_in_also_in_new_submodule_no_users(): - # A still-in name imported by a new submodule but NOT referenced by any - # top-level entity → annotation shows the submodule without "used by". - orig = "from ...llm_client import make_client\ndef foo(): pass\n" - mod = "from ...llm_client import make_client\nfrom .sub import helper\n" - ctx = _make_fl_ctx( - original_source=orig, - modified_source=mod, - new_files={ - "sub.py": "from ...llm_client import make_client\ndef helper(): pass\n" - }, - new_module_paths={"sub.py": "pkg.sub"}, - entity_to_target={"helper": "sub.py"}, - ) - ctx_msg = _build_context_message([ctx]) - assert "also externally imported in" in ctx_msg - assert "pkg.sub" in ctx_msg - # No entity in sub.py uses make_client → no "(used by: ...)" parenthetical. - assert "(used by:" not in ctx_msg - - -# --------------------------------------------------------------------------- -# _build_classify_prompt -# --------------------------------------------------------------------------- - - -def _ctx_msg() -> str: - return _build_context_message([_make_fl_ctx()]) - - -def test_build_classify_prompt_no_prev(): - prompt = _build_classify_prompt( - _ctx_msg(), "def test_f(): pass", ["crispen.before.X"] - ) - assert "crispen.before.X" in prompt - assert "Previous attempt was rejected" not in prompt - assert "patch_renames" in prompt - assert "Entity migration (quick reference)" in prompt - - -def test_build_classify_prompt_with_prev(): - prompt = _build_classify_prompt( - _ctx_msg(), - "def test_f(): pass", - ["crispen.before.X"], - prev_issue="wrong module", - prev_proposed="{'crispen.before.X': 'bad.mod.X'}", - ) - assert "Previous attempt was rejected" in prompt - assert "wrong module" in prompt - assert "bad.mod.X" in prompt - - -def test_build_classify_prompt_multiple_paths(): - prompt = _build_classify_prompt( - _ctx_msg(), "def test_f(): pass", ["crispen.before.X", "crispen.before.Y"] - ) - assert "crispen.before.X" in prompt - assert "crispen.before.Y" in prompt - - -def test_build_classify_prompt_with_lookup(): - # When the context has a patch target lookup, it appears in the classify prompt - # and the simplified lookup-based algorithm is used. - ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) - prompt = _build_classify_prompt( - ctx_msg, "def test_f(): pass", ["pkg.big.call_with_tool"] - ) - assert "Patch target lookup" in prompt - assert "call_with_tool" in prompt - assert "pkg.llm_planning" in prompt - assert "patch_renames" in prompt - assert "Entity migration (quick reference)" in prompt - - -def test_build_classify_prompt_with_stable_paths(): - # stable_patch_paths appear in a separate "already correct" section and - # the forking path remains in the "needs updating" section. - prompt = _build_classify_prompt( - _ctx_msg(), - "def test_f(): pass", - ["crispen.before.X"], - stable_patch_paths=["crispen.after.Y"], - ) - assert "crispen.before.X" in prompt - assert "crispen.after.Y" in prompt - assert "already correct" in prompt - assert "do not modify" in prompt - - -# --------------------------------------------------------------------------- -# _build_func_verify_prompt -# --------------------------------------------------------------------------- - - -def test_build_func_verify_prompt_basic(): - prompt = _build_func_verify_prompt( - _ctx_msg(), - "def test_f(): pass", - {"crispen.before.X": "crispen.after.X"}, - ) - assert "crispen.before.X" in prompt - assert "crispen.after.X" in prompt - assert "correct" in prompt - - -def test_build_func_verify_prompt_multiple_renames(): - prompt = _build_func_verify_prompt( - _ctx_msg(), - "def test_f(): pass", - {"crispen.before.X": "crispen.after.X", "crispen.before.Y": "crispen.after.Y"}, - ) - assert "crispen.before.X" in prompt - assert "crispen.before.Y" in prompt - assert "crispen.after.X" in prompt - assert "crispen.after.Y" in prompt - - -def test_build_func_verify_prompt_includes_patch_lookup(): - # When the context has a patch lookup section, it should be repeated near - # the verify instructions. - ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) - prompt = _build_func_verify_prompt( - ctx_msg, - "def test_f(): pass", - {"pkg.old.call_with_tool": "pkg.llm_planning.call_with_tool"}, - ) - assert "Patch target lookup" in prompt - - -# --------------------------------------------------------------------------- -# _build_no_change_verify_prompt -# --------------------------------------------------------------------------- - - -def test_build_no_change_verify_prompt_includes_migration_reminder(): - # Prompt built with a context that has migration entries should include - # the migration quick-reference block near the instructions. - prompt = _build_no_change_verify_prompt( - _ctx_msg(), - "def test_f(): pass", - ["crispen.before.X"], - ) - assert "crispen.before.X" in prompt - assert "Entity migration" in prompt - - -def test_build_no_change_verify_prompt_includes_patch_lookup(): - # When the context has a patch lookup section, it should be repeated near - # the verify instructions so the model doesn't have to scan the full context. - ctx_msg = _build_context_message([_make_ctx_with_ext_imports()]) - prompt = _build_no_change_verify_prompt( - ctx_msg, - "def test_f(): pass", - ["pkg.old.call_with_tool"], - ) - assert "Patch target lookup" in prompt - - -def test_build_no_change_verify_prompt_with_stable_paths(): - # stable_patch_paths appear in a separate "already correct" section and - # the instruction tells the verifier not to include them in corrections. - prompt = _build_no_change_verify_prompt( - _ctx_msg(), - "def test_f(): pass", - ["crispen.before.X"], - stable_patch_paths=["crispen.after.Y"], - ) - assert "crispen.before.X" in prompt - assert "crispen.after.Y" in prompt - assert "already correct" in prompt - assert "do not include" in prompt - - -# --------------------------------------------------------------------------- -# _build_rewrite_func_prompt -# --------------------------------------------------------------------------- - - -def test_build_rewrite_func_prompt_no_error(): - prompt = _build_rewrite_func_prompt( - _ctx_msg(), "def test_f(): pass", ["crispen.before.X"] - ) - assert "crispen.before.X" in prompt - assert "Previous rewrite" not in prompt - assert "Rewrite the complete function" in prompt - - -def test_build_rewrite_func_prompt_with_error(): - prompt = _build_rewrite_func_prompt( - _ctx_msg(), - "def test_f(): pass", - ["crispen.before.X"], - prev_error="SyntaxError on line 3", - ) - assert "Previous rewrite was rejected" in prompt - assert "SyntaxError on line 3" in prompt - - -def test_build_rewrite_func_prompt_with_stable_paths(): - # stable_patch_paths appear in a separate "already correct" section and - # the instruction tells the LLM not to modify them. - prompt = _build_rewrite_func_prompt( - _ctx_msg(), - "def test_f(): pass", - ["crispen.before.X"], - stable_patch_paths=["crispen.after.Y"], - ) - assert "crispen.before.X" in prompt - assert "crispen.after.Y" in prompt - assert "already correct" in prompt - assert "do not modify" in prompt.lower() - - -# --------------------------------------------------------------------------- -# _build_rewrite_verify_prompt -# --------------------------------------------------------------------------- - - -def test_build_rewrite_verify_prompt_basic(): - prompt = _build_rewrite_verify_prompt( - _ctx_msg(), - "def test_f(): pass", - '@patch("crispen.after.X")\ndef test_f(mock_x):\n pass\n', - ) - assert "Original test function" in prompt - assert "Rewritten test function" in prompt - assert "crispen.after.X" in prompt - assert "correct" in prompt - - -# --------------------------------------------------------------------------- -# _process_file_source — basic flow -# --------------------------------------------------------------------------- - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_functions(mock_call): - src = "def test_f(): pass\n" - result, changed, cross = _process_file_source( - src, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert result == src - assert changed is False - mock_call.assert_not_called() - - -@mock_patch(_PATCH_CALL_TOOL, return_value=_ok(None)) -def test_process_classify_tool_none(mock_call): - # Classify returns tool_input=None → break, no update. - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert result == _SRC_WITH_PATCH - assert changed is False - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_needed(mock_call): - # Classify returns empty renames → verify confirms no-change → no update. - mock_call.side_effect = [_ok(_CLASSIFY_NO_CHANGE), _ok(_VERIFY_OK)] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert result == _SRC_WITH_PATCH - assert changed is False - assert mock_call.call_count == 2 # classify + verify - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_verify_none_accept(mock_call): - # Classify says no change; verify returns None → accept no-change. - mock_call.side_effect = [_ok(_CLASSIFY_NO_CHANGE), _ok(None)] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is False - assert mock_call.call_count == 2 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_verify_truncated_reject(mock_call): - # No-change verify truncated → treated as rejection, not accepted as no-change. - mock_call.side_effect = [_ok(_CLASSIFY_NO_CHANGE), _truncated_ok()] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 1 - ) - assert result == _SRC_WITH_PATCH - assert changed is False - assert mock_call.call_count == 2 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_verify_rejects_then_accepts(mock_call): - # No-change verify rejects with corrections; corrections-verify accepts. - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok(_VERIFY_REJECT_WITH_CORRECTIONS), - _ok(_VERIFY_OK), # corrections-verify accepts - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 - ) - assert changed is True - assert "crispen.after.X" in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_verify_retries_exhausted(mock_call): - # llm_verify_retries=0: no escalation, accept no-change immediately. - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=1, llm_verify_retries=0) - mock_call.side_effect = [_ok(_CLASSIFY_NO_CHANGE), _ok(_VERIFY_REJECT)] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), cfg, 1 - ) - assert changed is False - assert mock_call.call_count == 2 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_exhausted_escalates_to_rewrite(mock_call): - # When llm_verify_retries>0 and no-change retries are exhausted, escalate - # to the full rewrite path seeded with the verifier's explanation. - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) - # Corrections that rename the function itself (X→Y) are filtered out by the - # name-invariant guard, so corrections_renames ends up empty, causing the - # retry to exhaust and escalate to rewrite (covers lines 2897-2901). - name_change_correction = { - "correct": False, - "issue": "wrong path", - "corrections": {"crispen.before.X": "crispen.before.Y"}, - } - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), # classify → no change - _ok(name_change_correction), # verify → reject (filtered corrections) - _ok(_CLASSIFY_NO_CHANGE), # classify (retry) → no change again - _ok(name_change_correction), # verify → reject (retries exhausted → escalate) - _ok({"rewritten_function": _VALID_REWRITE}), # rewrite (escalated) - _ok(_REWRITE_VERIFY_OK), # verify rewrite → accept - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), cfg, 3 - ) - assert changed is True - assert mock_call.call_count == 6 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_exhausted_escalate_verbose(mock_call, capsys): - # verbose=True prints 'escalating to rewrite' when escalation is triggered. - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) - name_change_correction = { - "correct": False, - "issue": "wrong path", - "corrections": {"crispen.before.X": "crispen.before.Y"}, - } - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok(name_change_correction), - _ok(_CLASSIFY_NO_CHANGE), - _ok(name_change_correction), - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(_REWRITE_VERIFY_OK), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - cfg, - 3, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "escalating to rewrite" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_applied(mock_call): - # No-change verify returns corrections → corrections-verify accepts → apply. - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok(_VERIFY_REJECT_WITH_CORRECTIONS), - _ok(_VERIFY_OK), # corrections-verify accepts - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 - ) - assert changed is True - assert "crispen.after.X" in result - assert mock_call.call_count == 3 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_verify_none_accept(mock_call): - # Corrections-verify returns tool_input=None → accept. - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok(_VERIFY_REJECT_WITH_CORRECTIONS), - _ok(None), # corrections-verify returns None → accept - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 - ) - assert changed is True - assert "crispen.after.X" in result - assert mock_call.call_count == 3 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_verify_truncated_reject(mock_call): - # Corrections-verify truncated → treated as rejection, corrections not applied. - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok(_VERIFY_REJECT_WITH_CORRECTIONS), - _truncated_ok(), # corrections-verify truncated → reject - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 2 - ) - assert result == _SRC_WITH_PATCH - assert changed is False - assert mock_call.call_count == 3 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_verify_fails_retry(mock_call): - # Corrections-verify rejects → retries left → retry classify which succeeds. - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), # classify → no change - _ok(_VERIFY_REJECT_WITH_CORRECTIONS), # verify → reject + corrections - _ok(_VERIFY_REJECT), # corrections-verify → rejected - _ok(_CLASSIFY_RENAME), # classify (retry) → rename - _ok(_VERIFY_OK), # rename verify → accept - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), cfg, 3 - ) - assert changed is True - assert "crispen.after.X" in result - assert mock_call.call_count == 5 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_verbose(mock_call, capsys): - # verbose=True prints 'verifying corrections for' and 'ACCEPTED'. - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok(_VERIFY_REJECT_WITH_CORRECTIONS), - _ok(_VERIFY_OK), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - _CFG, - 2, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "verifying corrections for" in err - assert "ACCEPTED" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_verbose_reject(mock_call, capsys): - # verbose=True prints 'REJECTED' and issue when corrections-verify rejects. - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok(_VERIFY_REJECT_WITH_CORRECTIONS), - _ok({"correct": False, "issue": "correction still wrong", "corrections": {}}), - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_OK), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - cfg, - 3, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "corrections verify REJECTED" in err - assert "correction still wrong" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_timing_detailed(mock_call, capsys): - # timing='detailed' prints elapsed/token info after corrections-verify call. - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=2, timing="detailed") - mock_call.side_effect = [ - LLMCallResult( - tool_input=_CLASSIFY_NO_CHANGE, - elapsed=0.5, - input_tokens=100, - output_tokens=10, - ), - LLMCallResult( - tool_input=_VERIFY_REJECT_WITH_CORRECTIONS, - elapsed=0.4, - input_tokens=90, - output_tokens=20, - ), - LLMCallResult( - tool_input=_VERIFY_OK, - elapsed=0.3, - input_tokens=80, - output_tokens=5, - ), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - cfg, - 2, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "0.30s" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_acc(mock_call): - # _acc accumulates calls from classify, no-change verify, and corrections-verify. - mock_call.side_effect = [ - LLMCallResult( - tool_input=_CLASSIFY_NO_CHANGE, - elapsed=0.5, - input_tokens=100, - output_tokens=10, - ), - LLMCallResult( - tool_input=_VERIFY_REJECT_WITH_CORRECTIONS, - elapsed=0.4, - input_tokens=90, - output_tokens=20, - ), - LLMCallResult( - tool_input=_VERIFY_OK, - elapsed=0.3, - input_tokens=80, - output_tokens=5, - ), - ] - acc = RewriteAccumulator() - _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2, _acc=acc - ) - assert acc.calls == 3 - assert abs(acc.elapsed - 1.2) < 1e-9 - assert acc.input_tokens == 270 - assert acc.output_tokens == 35 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_no_splice(mock_call, tmp_path): - # Corrections-verify accepts; function uses const ref → no splice; const updated. - src = ( - 'TARGET = "crispen.before.X"\n\n@patch(TARGET)\ndef test_f(mock_x):\n pass\n' - ) - scan = str(tmp_path / "test_foo.py") - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok(_VERIFY_REJECT_WITH_CORRECTIONS), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - src, {"crispen.before.X"}, "ctx", MagicMock(), _CFG, 2, scan_file=scan - ) - assert changed is True - assert "crispen.after.X" in result - assert mock_call.call_count == 3 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_name_invariant_filtered(mock_call): - # Verifier proposes corrections that rename the patched name itself - # (e.g. X → Y). These must be filtered out; with an empty corrections set - # the no-change result falls through to retry logic — here retries=1 so - # the second classify call is made and returns no-change confirmed by verify. - verify_name_change_correction = { - "correct": False, - "issue": "module moved", - "corrections": {"crispen.before.X": "crispen.before.Y"}, # name changed! - } - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok(verify_name_change_correction), - _ok(_CLASSIFY_NO_CHANGE), - _ok(_VERIFY_OK), - ] - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), cfg, 3 - ) - # Correction was filtered (name changed X→Y) — no change applied. - assert "crispen.before.Y" not in result - assert mock_call.call_count == 4 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_corrections_still_imported_guard(mock_call): - # Verifier proposes corrections that move a name listed as still-imported in - # the context message to a non-submodule path. The second still-imported - # filter drops the correction; with empty corrections the retry loop resumes - # and accepts no-change on verify. - still_imported_ctx = ( - "Names still externally imported in the modified original (check):\n" "- `X`\n" - ) - verify_still_imported_correction = { - "correct": False, - "issue": "hallucinated move", - "corrections": {"crispen.before.X": "crispen.sub.X"}, # X is still in orig - } - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok(verify_still_imported_correction), - _ok(_CLASSIFY_NO_CHANGE), - _ok(_VERIFY_OK), - ] - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - still_imported_ctx, - MagicMock(), - cfg, - 3, - still_imported={"X"}, - ) - # Correction was filtered (X still imported, non-submodule target) — - # no change applied; retry loop accepted no-change on subsequent verify. - assert "crispen.sub.X" not in result - assert mock_call.call_count == 4 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_verify_verbose(mock_call, capsys): - # verbose=True prints 'verifying no-change' and 'ACCEPTED'. - mock_call.side_effect = [_ok(_CLASSIFY_NO_CHANGE), _ok(_VERIFY_OK)] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "verifying no-change" in err - assert "ACCEPTED" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_verify_verbose_reject(mock_call, capsys): - # verbose=True prints 'REJECTED' and the issue when no-change verify rejects. - mock_call.side_effect = [ - _ok(_CLASSIFY_NO_CHANGE), - _ok({"correct": False, "issue": "patch still points to old module"}), - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_OK), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - _CFG, - 2, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "REJECTED" in err - assert "patch still points to old module" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_verify_timing_detailed(mock_call, capsys): - # timing='detailed' appends elapsed/token info after the no-change verify call. - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=1, timing="detailed") - mock_call.side_effect = [ - LLMCallResult( - tool_input=_CLASSIFY_NO_CHANGE, - elapsed=0.5, - input_tokens=100, - output_tokens=10, - ), - LLMCallResult( - tool_input=_VERIFY_OK, - elapsed=0.3, - input_tokens=80, - output_tokens=5, - ), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - cfg, - 1, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "→ done" in err - assert "0.30s" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_change_acc_accumulates(mock_call): - # _acc accumulates calls from both classify and no-change verify; no_change counted. - mock_call.side_effect = [ - LLMCallResult( - tool_input=_CLASSIFY_NO_CHANGE, - elapsed=0.5, - input_tokens=100, - output_tokens=10, - ), - LLMCallResult( - tool_input=_VERIFY_OK, - elapsed=0.3, - input_tokens=80, - output_tokens=5, - ), - ] - acc = RewriteAccumulator() - _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1, _acc=acc - ) - assert acc.calls == 2 - assert abs(acc.elapsed - 0.8) < 1e-9 - assert acc.input_tokens == 180 - assert acc.output_tokens == 15 - assert acc.no_change == 1 - assert acc.rename == 0 - assert acc.rewrite == 0 - assert acc.edit_failures == 0 - - -@mock_patch(_PATCH_CALL_TOOL, return_value=_ok(None)) -def test_process_acc_edit_failure_on_classify_none(mock_call): - # Classify returns tool_input=None → edit_failures incremented. - acc = RewriteAccumulator() - _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1, _acc=acc - ) - assert acc.edit_failures == 1 - assert acc.no_change == 0 - assert acc.rename == 0 - assert acc.rewrite == 0 - - -@mock_patch( - _PATCH_CALL_TOOL, - return_value=_ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.before.X"}, - } - ), -) -def test_process_same_path_filtered_out(mock_call): - # Rename where old == new → filtered to empty → triggers no-change verify. - # return_value repeats for both calls; verify gets wrong type → rejects; retries - # exhaust → accept no-change. - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is False - - -@mock_patch( - _PATCH_CALL_TOOL, - return_value=_ok({"needs_rewrite": False, "patch_renames": "not-a-dict"}), -) -def test_process_patch_renames_not_dict(mock_call): - # patch_renames is not a dict → treated as empty, no change. - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is False - - -@mock_patch( - _PATCH_CALL_TOOL, - return_value=_ok( - {"needs_rewrite": False, "patch_renames": {42: "crispen.after.X"}} - ), -) -def test_process_patch_renames_non_string_key(mock_call): - # Non-string key in patch_renames → filtered out. - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is False - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_patch_renames_name_invariant_filtered(mock_call): - # LLM proposes renaming crispen.before.X → crispen.before.Y (name changed from - # X to Y). A file split never renames an entity — only its module path changes. - # The rename must be filtered out, leaving no renames → triggers no-change verify. - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.before.Y"}, - } - ), - _ok(_VERIFY_OK), # no-change verify confirms - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is False - assert mock_call.call_count == 2 # classify + no-change verify - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_string_swap_verify_accepts(mock_call): - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is True - assert "crispen.after.X" in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verify_none_accept(mock_call): - # Verify call returns tool_input=None → accept proposed renames. - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _ok(None), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is True - assert "crispen.after.X" in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verify_truncated_reject(mock_call): - # Verify call truncated → treated as rejection, renames not applied. - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _truncated_ok(), # verify truncated → reject - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 1 - ) - assert result == _SRC_WITH_PATCH - assert changed is False - assert mock_call.call_count == 2 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verify_none_accept_no_splice(mock_call, tmp_path): - # Verify returns None; function uses const ref → new_text == orig_text → no splice. - src = ( - 'TARGET = "crispen.before.X"\n\n@patch(TARGET)\ndef test_f(mock_x):\n pass\n' - ) - scan = str(tmp_path / "test_foo.py") - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _ok(None), - ] - result, changed, cross = _process_file_source( - src, {"crispen.before.X"}, "ctx", MagicMock(), _CFG, 1, scan_file=scan - ) - # No splice but const should be updated via same_file_const_map. - assert changed is True - assert "crispen.after.X" in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verify_rejected_then_accept(mock_call): - # First verify rejects; second classify+verify is accepted. - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_REJECT_WITH_CORRECTIONS), - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 - ) - assert changed is True - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verify_rejected_exhausted(mock_call): - # Verify rejects with llm_verify_retries=0 → function skipped. - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_REJECT), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 1 - ) - assert result == _SRC_WITH_PATCH - assert changed is False - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verify_rejected_exhausted_escalates_to_rewrite(mock_call): - # When llm_verify_retries>0 and rename verify retries are exhausted, - # escalate to the full rewrite path seeded with the verifier's explanation. - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=3, llm_verify_retries=1) - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), # classify → rename - _ok(_VERIFY_REJECT_WITH_CORRECTIONS), # verify → reject (retries left) - _ok(_CLASSIFY_RENAME), # classify (retry) → rename again - _ok( - _VERIFY_REJECT_WITH_CORRECTIONS - ), # verify → reject (retries exhausted → escalate) - _ok({"rewritten_function": _VALID_REWRITE}), # rewrite (escalated) - _ok(_REWRITE_VERIFY_OK), # verify rewrite → accept - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), cfg, 3 - ) - assert changed is True - assert mock_call.call_count == 6 - - -# --------------------------------------------------------------------------- -# _process_file_source — full rewrite path -# --------------------------------------------------------------------------- - -_VALID_REWRITE = ( - '@patch("crispen.after.X")\n' - '@patch("crispen.after.Y")\n' - "def test_f(mock_x, mock_y):\n" - " pass\n" -) - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_needs_rewrite_success(mock_call): - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(_REWRITE_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is True - assert "crispen.after.X" in result - assert "crispen.after.Y" in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_needs_rewrite_tool_none(mock_call): - # Rewrite call returns tool_input=None → no update. - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok(None), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert result == _SRC_WITH_PATCH - assert changed is False - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_needs_rewrite_empty_text(mock_call): - # Rewrite returns empty string → no update. - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": ""}), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is False - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_needs_rewrite_non_string(mock_call): - # Rewrite returns non-string value → no update. - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": 42}), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is False - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_needs_rewrite_compile_error_retry(mock_call): - # First rewrite has syntax error; second is valid. - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": "def f(:\n pass\n"}), # invalid - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(_REWRITE_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 - ) - assert changed is True - assert "crispen.after.X" in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_needs_rewrite_compile_error_exhausted(mock_call): - # Both rewrite attempts fail to compile → no update. - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": "def f(:\n pass\n"}), # invalid - _ok({"rewritten_function": "def f(:\n pass\n"}), # still invalid - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 - ) - assert changed is False - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_needs_rewrite_verify_none_accept(mock_call): - # Verify returns tool_input=None → accept the rewrite. - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(None), # verify returns None → accept - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is True - assert "crispen.after.X" in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_needs_rewrite_verify_truncated_reject(mock_call): - # Rewrite verify truncated → treated as rejection, rewrite not accepted. - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": _VALID_REWRITE}), - _truncated_ok(), # verify truncated → reject - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 1 - ) - assert result == _SRC_WITH_PATCH - assert changed is False - assert mock_call.call_count == 3 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_needs_rewrite_verify_rejected_then_accept(mock_call): - # Verify rejects first rewrite; second rewrite+verify is accepted. - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(_REWRITE_VERIFY_REJECT), - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(_REWRITE_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 2 - ) - assert changed is True - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_needs_rewrite_verify_rejected_exhausted(mock_call): - # Verify rejects with llm_verify_retries=0 → no update. - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(_REWRITE_VERIFY_REJECT), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG_NO_LLM_VERIFY, 1 - ) - assert result == _SRC_WITH_PATCH - assert changed is False - - -# --------------------------------------------------------------------------- -# _process_file_source — per-function processing (forking case) -# --------------------------------------------------------------------------- - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_per_function_different_renames(mock_call): - """Two functions with the same @patch string can receive different renames. - - This is the forking case: test_a tests an entity that moved to mod1, - test_b tests an entity that moved to mod2. Each gets classified and - renamed independently. - """ - src = ( - '@patch("crispen.before.X")\ndef test_a(m):\n call_a()\n\n' - '@patch("crispen.before.X")\ndef test_b(m):\n call_b()\n' - ) - mock_call.side_effect = [ - # test_a: classify → rename to mod1 - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.mod1.X"}, - } - ), - _ok(_VERIFY_OK), - # test_b: classify → rename to mod2 - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.mod2.X"}, - } - ), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - src, {"crispen.before.X"}, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is True - assert "crispen.mod1.X" in result - assert "crispen.mod2.X" in result - assert mock_call.call_count == 4 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_per_function_both_updated(mock_call): - """Two functions with the same @patch string both get the same rename.""" - src = ( - '@patch("crispen.before.X")\ndef test_a(m):\n pass\n\n' - '@patch("crispen.before.X")\ndef test_b(m):\n pass\n' - ) - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_OK), - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - src, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1 - ) - assert changed is True - assert result.count("crispen.after.X") == 2 - - -# --------------------------------------------------------------------------- -# _process_file_source — accumulator and verbose -# --------------------------------------------------------------------------- - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_acc_accumulates(mock_call): - """_process_file_source accumulates calls, elapsed, and tokens into _acc.""" - mock_call.side_effect = [ - LLMCallResult( - tool_input=_CLASSIFY_RENAME, - elapsed=1.2, - input_tokens=200, - output_tokens=40, - ), - LLMCallResult( - tool_input=_VERIFY_OK, - elapsed=0.3, - input_tokens=150, - output_tokens=5, - ), - ] - acc = RewriteAccumulator() - _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1, _acc=acc - ) - assert acc.calls == 2 - assert abs(acc.elapsed - 1.5) < 1e-9 - assert acc.input_tokens == 350 - assert acc.output_tokens == 45 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_acc_rewrite_accumulates(mock_call): - """Full rewrite path accumulates classify, rewrite, and verify calls.""" - mock_call.side_effect = [ - LLMCallResult( - tool_input=_CLASSIFY_REWRITE, - elapsed=0.5, - input_tokens=100, - output_tokens=10, - ), - LLMCallResult( - tool_input={"rewritten_function": _VALID_REWRITE}, - elapsed=1.5, - input_tokens=300, - output_tokens=60, - ), - LLMCallResult( - tool_input=_REWRITE_VERIFY_OK, - elapsed=0.2, - input_tokens=80, - output_tokens=5, - ), - ] - acc = RewriteAccumulator() - _process_file_source( - _SRC_WITH_PATCH, _FORKING_PATHS, "ctx", MagicMock(), _CFG, 1, _acc=acc - ) - assert acc.calls == 3 - assert abs(acc.elapsed - 2.2) < 1e-9 - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verbose_prints_to_stderr(mock_call, capsys): - """verbose=True emits per-call messages to stderr.""" - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_OK), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "patch_rewriter" in err - assert "classifying" in err - assert "verifying renames" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verbose_detailed_timing(mock_call, capsys): - """timing='detailed' appends elapsed/token info after each call.""" - mock_call.side_effect = [ - LLMCallResult( - tool_input=_CLASSIFY_RENAME, - elapsed=1.23, - input_tokens=100, - output_tokens=20, - ), - LLMCallResult( - tool_input=_VERIFY_OK, - elapsed=0.45, - input_tokens=80, - output_tokens=5, - ), - ] - from crispen.config import CrispenConfig - - cfg = CrispenConfig(patch_update_retries=1, timing="detailed") - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - cfg, - 1, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "→ done" in err - assert "1.23s" in err - assert "0.45s" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verbose_retry_label(mock_call, capsys): - """Retry attempts include '(retry)' in the verbose message.""" - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_REJECT), - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_OK), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - _CFG, - 2, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "(retry)" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verbose_verify_accepted(mock_call, capsys): - """verbose=True prints 'ACCEPTED' when verify succeeds.""" - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_OK), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "ACCEPTED" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verbose_verify_rejected_prints_issue(mock_call, capsys): - """verbose=True prints 'REJECTED' and the issue when verify rejects.""" - mock_call.side_effect = [ - _ok(_CLASSIFY_RENAME), - _ok( - { - "correct": False, - "issue": "wrong module path", - "corrections": {"crispen.before.X": "crispen.after.X"}, - } - ), - _ok(_CLASSIFY_RENAME), - _ok(_VERIFY_OK), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - _CFG, - 2, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "REJECTED" in err - assert "wrong module path" in err - assert "ACCEPTED" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verbose_rewrite_path(mock_call, capsys): - """verbose=True prints 'rewriting', 'verifying rewrite', and 'rewrote'.""" - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(_REWRITE_VERIFY_OK), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "rewriting" in err - assert "verifying rewrite" in err - assert "rewrote" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verbose_rewrite_verify_rejected(mock_call, capsys): - """verbose=True prints 'REJECTED' and issue when rewrite verify fails.""" - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(_REWRITE_VERIFY_REJECT), - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(_REWRITE_VERIFY_OK), - ] - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - _CFG, - 2, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "REJECTED" in err - assert "wrong mock setup" in err - assert "ACCEPTED" in err - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_verbose_rewrite_compile_retry(mock_call, capsys): - """verbose=True prints '(retry)' when rewrite compile fails.""" - mock_call.side_effect = [ - _ok(_CLASSIFY_REWRITE), - _ok({"rewritten_function": "def f(:\n pass\n"}), # invalid - _ok({"rewritten_function": _VALID_REWRITE}), - _ok(_REWRITE_VERIFY_OK), - ] - cfg = CrispenConfig(patch_update_retries=1, timing="detailed") - _process_file_source( - _SRC_WITH_PATCH, - _FORKING_PATHS, - "ctx", - MagicMock(), - cfg, - 2, - scan_file="tests/test_foo.py", - verbose=True, - ) - err = capsys.readouterr().err - assert "rewriting" in err - assert "(retry)" in err - - -# --------------------------------------------------------------------------- -# _process_file_source — const ref restoration after full rewrite -# --------------------------------------------------------------------------- - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_rewrite_restores_unchanged_const_ref(mock_call): - """After full rewrite, @patch("value") left unchanged → reverted to @patch(NAME).""" - src = ( - 'STABLE = "pkg.stable.X"\n' - 'TARGET = "pkg.big.A"\n\n' - "@patch(STABLE)\n" - "@patch(TARGET)\n" - "def test_f(mock_stable, mock_target):\n" - " pass\n" - ) - # LLM updates TARGET but leaves STABLE's substituted literal unchanged. - rewritten = ( - '@patch("pkg.stable.X")\n' - '@patch("pkg.sub_a.A")\n' - "def test_f(mock_stable, mock_target):\n" - " pass\n" - ) - mock_call.side_effect = [ - _ok({"needs_rewrite": True}), - _ok({"rewritten_function": rewritten}), - _ok({"correct": True, "issue": ""}), - ] - result, changed, _ = _process_file_source( - src, - {"pkg.big.A"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file="tests/test_foo.py", - ) - assert changed is True - # STABLE decorator reverted to named constant form. - assert "@patch(STABLE)" in result - assert '@patch("pkg.stable.X")' not in result - # TARGET decorator keeps the LLM's updated literal value. - assert '@patch("pkg.sub_a.A")' in result - - -# --------------------------------------------------------------------------- -# apply_patch_rewrite -# --------------------------------------------------------------------------- - - -def test_rewrite_empty_contexts(): - msgs = list(apply_patch_rewrite([], {}, "/repo", _CFG)) - assert msgs == [] - - -def test_rewrite_no_forking_paths(): - ctx = _make_fl_ctx(forking_old_paths=set()) - msgs = list(apply_patch_rewrite([ctx], {}, "/repo", _CFG)) - assert msgs == [] - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_per_file_update(mock_key, mock_client, mock_call): - mock_call.side_effect = [ - _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), - _ok(_VERIFY_OK), - ] - src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' - per_file = {"/repo/tests/test_big.py": {"source": src, "msgs": []}} - ctx = _make_fl_ctx() - msgs = list(apply_patch_rewrite([ctx], per_file, None, _CFG)) - updated = per_file["/repo/tests/test_big.py"]["source"] - assert "pkg.sub_a.A" in updated - assert any("patch_update" in m for m in per_file["/repo/tests/test_big.py"]["msgs"]) - assert msgs == [] # no disk messages since repo_root=None - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_no_repo_root_no_disk_scan(mock_key, mock_client, mock_call): - # repo_root=None → exits after per_file; empty per_file → no LLM calls. - msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, None, _CFG)) - assert msgs == [] - mock_call.assert_not_called() - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_disk_file_update(mock_key, mock_client, mock_call, tmp_path): - test_file = tmp_path / "test_big.py" - src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' - test_file.write_text(src, encoding="utf-8") - mock_call.side_effect = [ - _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), - _ok(_VERIFY_OK), - ] - msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) - assert "pkg.sub_a.A" in test_file.read_text(encoding="utf-8") - assert len(msgs) == 1 - assert "patch_update" in msgs[0] - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_skip_excluded_dir(mock_key, mock_client, mock_call, tmp_path): - venv = tmp_path / "venv" - venv.mkdir() - f = venv / "test_big.py" - f.write_text('@patch("pkg.big.A")\ndef test_f(): pass\n', encoding="utf-8") - msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) - assert msgs == [] - mock_call.assert_not_called() - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_skip_per_file_abs(mock_key, mock_client, mock_call, tmp_path): - # A file already in per_file should NOT be re-processed from disk. - test_file = tmp_path / "test_big.py" - test_file.write_text('@patch("pkg.big.A")\ndef test_f(): pass\n', encoding="utf-8") - original_disk = test_file.read_text(encoding="utf-8") - # per_file entry uses a source without matching patches (no LLM call needed). - per_file = {str(test_file): {"source": "# no patches\n", "msgs": []}} - list(apply_patch_rewrite([_make_fl_ctx()], per_file, str(tmp_path), _CFG)) - # Disk file untouched since it was in per_file_abs. - assert test_file.read_text(encoding="utf-8") == original_disk - mock_call.assert_not_called() - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_oserror_skipped(mock_key, mock_client, mock_call, tmp_path): - test_file = tmp_path / "test_big.py" - test_file.write_text('@patch("pkg.big.A")\ndef test_f(): pass\n', encoding="utf-8") - test_file.chmod(0o000) - try: - msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) - assert msgs == [] - mock_call.assert_not_called() - finally: - test_file.chmod(0o644) - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_disk_file_no_match_not_updated( - mock_key, mock_client, mock_call, tmp_path -): - # Disk file exists but has no matching @patch decorators → changed=False, - # file is not written, no yield message (covers the `if changed: False` branch). - test_file = tmp_path / "no_patches.py" - test_file.write_text("def test_unrelated(): pass\n", encoding="utf-8") - msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) - assert msgs == [] - assert test_file.read_text(encoding="utf-8") == "def test_unrelated(): pass\n" - mock_call.assert_not_called() - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_no_py_files_in_repo(mock_key, mock_client, mock_call, tmp_path): - # tmp_path has no .py files → disk scan loop body never executes. - msgs = list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) - assert msgs == [] - mock_call.assert_not_called() - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_acc_tracks_calls_and_files(mock_key, mock_client, mock_call, tmp_path): - """RewriteAccumulator is populated with call counts and files_updated.""" - test_file = tmp_path / "test_big.py" - src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' - test_file.write_text(src, encoding="utf-8") - mock_call.side_effect = [ - LLMCallResult( - tool_input={ - "needs_rewrite": False, - "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}, - }, - elapsed=1.5, - input_tokens=100, - output_tokens=50, - ), - LLMCallResult( - tool_input=_VERIFY_OK, - elapsed=0.5, - input_tokens=80, - output_tokens=10, - ), - ] - acc = RewriteAccumulator() - list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG, _acc=acc)) - assert acc.calls == 2 - assert acc.elapsed == 2.0 - assert acc.input_tokens == 180 - assert acc.output_tokens == 60 - assert acc.files_updated == 1 - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_acc_per_file_files_updated(mock_key, mock_client, mock_call): - """files_updated is incremented for in-memory per_file changes.""" - src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' - per_file = {"/repo/tests/test_big.py": {"source": src, "msgs": []}} - mock_call.side_effect = [ - _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), - _ok(_VERIFY_OK), - ] - acc = RewriteAccumulator() - list(apply_patch_rewrite([_make_fl_ctx()], per_file, None, _CFG, _acc=acc)) - assert acc.files_updated == 1 - - -# --------------------------------------------------------------------------- -# _build_local_const_map -# --------------------------------------------------------------------------- - - -def test_local_const_map_string_assignment(): - src = 'TARGET = "myapp.service.MyClass"\n' - result = _build_local_const_map(src) - assert result == {"TARGET": "myapp.service.MyClass"} - - -def test_local_const_map_non_string_excluded(): - src = "TARGET = 42\n" - assert _build_local_const_map(src) == {} - - -def test_local_const_map_multi_target_excluded(): - # a = b = "value" has two targets → not included. - src = 'a = b = "value"\n' - assert _build_local_const_map(src) == {} - - -def test_local_const_map_syntax_error(): - assert _build_local_const_map("def f(:\n") == {} - - -def test_local_const_map_empty_source(): - assert _build_local_const_map("") == {} - - -def test_local_const_map_last_wins(): - src = 'X = "first"\nX = "second"\n' - assert _build_local_const_map(src)["X"] == "second" - - -def test_local_const_map_annotated_assignment(): - src = 'TARGET: str = "myapp.service.MyClass"\n' - assert _build_local_const_map(src) == {"TARGET": "myapp.service.MyClass"} - - -def test_local_const_map_annotated_non_string_excluded(): - src = "TARGET: int = 42\n" - assert _build_local_const_map(src) == {} - - -def test_local_const_map_annotated_no_value_excluded(): - # Bare annotation with no value: ``TARGET: str`` — ast.AnnAssign with value=None - src = "TARGET: str\n" - assert _build_local_const_map(src) == {} - - -# --------------------------------------------------------------------------- -# _resolve_import_to_file -# --------------------------------------------------------------------------- - - -def test_resolve_relative_level1_py(tmp_path): - # from .sub import NAME — sub.py exists - (tmp_path / "sub.py").write_text("X = 1\n", encoding="utf-8") - scan = str(tmp_path / "test_foo.py") - result = _resolve_import_to_file("sub", 1, scan, None) - assert result == str(tmp_path / "sub.py") - - -def test_resolve_relative_level1_init(tmp_path): - # from .pkg import NAME — pkg/__init__.py exists - pkg = tmp_path / "pkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("", encoding="utf-8") - scan = str(tmp_path / "test_foo.py") - result = _resolve_import_to_file("pkg", 1, scan, None) - assert result == str(pkg / "__init__.py") - - -def test_resolve_relative_level1_no_module(tmp_path): - # from . import NAME — finds __init__.py in same dir - (tmp_path / "__init__.py").write_text("", encoding="utf-8") - scan = str(tmp_path / "test_foo.py") - result = _resolve_import_to_file(None, 1, scan, None) - assert result == str(tmp_path / "__init__.py") - - -def test_resolve_relative_level2(tmp_path): - # from ..sub import NAME — goes up one level - parent = tmp_path / "parent" - parent.mkdir() - child = parent / "child" - child.mkdir() - (parent / "sub.py").write_text("X = 1\n", encoding="utf-8") - scan = str(child / "test_foo.py") - result = _resolve_import_to_file("sub", 2, scan, None) - assert result == str(parent / "sub.py") - - -def test_resolve_relative_not_found(tmp_path): - scan = str(tmp_path / "test_foo.py") - assert _resolve_import_to_file("missing", 1, scan, None) is None - - -def test_resolve_relative_no_module_no_init(tmp_path): - scan = str(tmp_path / "test_foo.py") - assert _resolve_import_to_file(None, 1, scan, None) is None - - -def test_resolve_absolute_found(tmp_path): - pkg = tmp_path / "mypkg" - pkg.mkdir() - (pkg / "helpers.py").write_text("X = 1\n", encoding="utf-8") - scan = str(tmp_path / "tests" / "test_foo.py") - result = _resolve_import_to_file("mypkg.helpers", 0, scan, str(tmp_path)) - assert result == str(pkg / "helpers.py") - - -def test_resolve_absolute_no_repo_root(tmp_path): - scan = str(tmp_path / "test_foo.py") - assert _resolve_import_to_file("mypkg.helpers", 0, scan, None) is None - - -def test_resolve_absolute_no_module(tmp_path): - scan = str(tmp_path / "test_foo.py") - assert _resolve_import_to_file(None, 0, scan, str(tmp_path)) is None - - -def test_resolve_absolute_not_found(tmp_path): - scan = str(tmp_path / "test_foo.py") - assert _resolve_import_to_file("no.such.module", 0, scan, str(tmp_path)) is None - - -# --------------------------------------------------------------------------- -# _build_const_map -# --------------------------------------------------------------------------- - - -def test_build_const_map_same_file(tmp_path): - src = 'TARGET = "myapp.service.MyClass"\n' - scan = str(tmp_path / "test_foo.py") - result = _build_const_map(src, scan, None) - val, def_file = result["TARGET"] - assert val == "myapp.service.MyClass" - assert def_file == str((tmp_path / "test_foo.py").resolve()) - - -def test_build_const_map_cross_file(tmp_path): - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "myapp.service.MyClass"\n', encoding="utf-8") - src = "from .helpers import TARGET\n" - scan = str(tmp_path / "test_foo.py") - result = _build_const_map(src, scan, None) - val, def_file = result["TARGET"] - assert val == "myapp.service.MyClass" - assert def_file == str(helpers.resolve()) - - -def test_build_const_map_alias(tmp_path): - helpers = tmp_path / "helpers.py" - helpers.write_text('X = "myapp.service.MyClass"\n', encoding="utf-8") - src = "from .helpers import X as MY_TARGET\n" - scan = str(tmp_path / "test_foo.py") - result = _build_const_map(src, scan, None) - assert "MY_TARGET" in result - assert result["MY_TARGET"][0] == "myapp.service.MyClass" - - -def test_build_const_map_local_priority(tmp_path): - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "imported.value"\n', encoding="utf-8") - src = 'TARGET = "local.value"\nfrom .helpers import TARGET\n' - scan = str(tmp_path / "test_foo.py") - result = _build_const_map(src, scan, None) - assert result["TARGET"][0] == "local.value" - - -def test_build_const_map_star_import_skipped(tmp_path): - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "myapp.service.MyClass"\n', encoding="utf-8") - src = "from .helpers import *\n" - scan = str(tmp_path / "test_foo.py") - result = _build_const_map(src, scan, None) - assert result == {} - - -def test_build_const_map_import_file_not_found(tmp_path): - src = "from .missing import TARGET\n" - scan = str(tmp_path / "test_foo.py") - result = _build_const_map(src, scan, None) - assert result == {} - - -def test_build_const_map_import_oserror(tmp_path): - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "val"\n', encoding="utf-8") - helpers.chmod(0o000) - try: - src = "from .helpers import TARGET\n" - scan = str(tmp_path / "test_foo.py") - result = _build_const_map(src, scan, None) - assert result == {} - finally: - helpers.chmod(0o644) - - -def test_build_const_map_syntax_error(): - result = _build_const_map("def f(:\n", "/some/file.py", None) - assert result == {} - - -def test_build_const_map_no_const_in_import(tmp_path): - helpers = tmp_path / "helpers.py" - helpers.write_text("def some_func(): pass\n", encoding="utf-8") - src = "from .helpers import some_func\n" - scan = str(tmp_path / "test_foo.py") - result = _build_const_map(src, scan, None) - assert result == {} - - -# --------------------------------------------------------------------------- -# _build_attr_const_map -# --------------------------------------------------------------------------- - - -def test_build_attr_const_map_basic(tmp_path): - """``import constants`` resolves string constants from the module file.""" - constants_file = tmp_path / "constants.py" - constants_file.write_text('TARGET = "myapp.service.MyClass"\n', encoding="utf-8") - src = "import constants\n" - scan = str(tmp_path / "test_foo.py") - result = _build_attr_const_map(src, scan, str(tmp_path)) - assert "constants" in result - val, def_file = result["constants"]["TARGET"] - assert val == "myapp.service.MyClass" - assert def_file == str(constants_file.resolve()) - - -def test_build_attr_const_map_with_alias(tmp_path): - """``import pkg.constants as C`` maps alias ``C`` to module constants.""" - pkg = tmp_path / "pkg" - pkg.mkdir() - constants_file = pkg / "constants.py" - constants_file.write_text('TARGET = "myapp.svc.MyClass"\n', encoding="utf-8") - src = "import pkg.constants as C\n" - scan = str(tmp_path / "test_foo.py") - result = _build_attr_const_map(src, scan, str(tmp_path)) - assert "C" in result - assert result["C"]["TARGET"][0] == "myapp.svc.MyClass" - - -def test_build_attr_const_map_no_file(tmp_path): - """Import that doesn't resolve to a file → skipped, empty result.""" - src = "import missing_module\n" - scan = str(tmp_path / "test_foo.py") - result = _build_attr_const_map(src, scan, str(tmp_path)) - assert result == {} - - -def test_build_attr_const_map_oserror(tmp_path): - """Module file exists but is unreadable → skipped.""" - constants_file = tmp_path / "constants.py" - constants_file.write_text('TARGET = "val"\n', encoding="utf-8") - constants_file.chmod(0o000) - try: - src = "import constants\n" - scan = str(tmp_path / "test_foo.py") - result = _build_attr_const_map(src, scan, str(tmp_path)) - assert result == {} - finally: - constants_file.chmod(0o644) - - -def test_build_attr_const_map_syntax_error(): - """SyntaxError in source → empty result.""" - assert _build_attr_const_map("def f(:\n", "/some/file.py", None) == {} - - -def test_build_attr_const_map_non_import_skipped(tmp_path): - """Non-``import`` statements (from-imports, assignments) are skipped.""" - constants_file = tmp_path / "constants.py" - constants_file.write_text('TARGET = "val"\n', encoding="utf-8") - # Only a from-import and an assignment; no plain ``import`` → empty. - src = 'from .constants import TARGET\nX = "y"\n' - scan = str(tmp_path / "test_foo.py") - result = _build_attr_const_map(src, scan, str(tmp_path)) - assert result == {} - - -# --------------------------------------------------------------------------- -# _substitute_consts_in_func_text -# --------------------------------------------------------------------------- - - -def test_substitute_replaces_const(): - code = "@patch(TARGET)\ndef test_f(mock): pass\n" - result = _substitute_consts_in_func_text(code, {"TARGET": "myapp.svc.MyClass"}) - assert '@patch("myapp.svc.MyClass")' in result - assert "TARGET" not in result - - -def test_substitute_no_subs_unchanged(): - code = "@patch(TARGET)\ndef test_f(mock): pass\n" - assert _substitute_consts_in_func_text(code, {}) == code - - -def test_substitute_parse_error_returns_original(): - code = "def f(:\n" - assert _substitute_consts_in_func_text(code, {"X": "val"}) == code - - -def test_substitute_non_patch_call_unchanged(): - # other_func(TARGET) inside the body is not a patch call → left as-is. - code = "@patch(TARGET)\ndef test_f(mock):\n other_func(TARGET)\n" - result = _substitute_consts_in_func_text(code, {"TARGET": "myapp.svc.MyClass"}) - assert '@patch("myapp.svc.MyClass")' in result - assert "other_func(TARGET)" in result - - -def test_substitute_name_not_in_subs_unchanged(): - # @patch(OTHER) where OTHER is not in substitutions → left as-is (line 311). - code = "@patch(TARGET)\n@patch(OTHER)\ndef test_f(m1, m2):\n pass\n" - result = _substitute_consts_in_func_text(code, {"TARGET": "myapp.svc.MyClass"}) - assert '@patch("myapp.svc.MyClass")' in result - assert "@patch(OTHER)" in result - - -def test_substitute_attr_in_subs(): - """@patch(module.CONSTANT) with dotted key in subs → substituted.""" - code = "@patch(constants.TARGET)\ndef test_f(mock):\n pass\n" - result = _substitute_consts_in_func_text( - code, {"constants.TARGET": "myapp.svc.MyClass"} - ) - assert '@patch("myapp.svc.MyClass")' in result - assert "constants.TARGET" not in result - - -def test_substitute_attr_not_in_subs(): - """@patch(constants.OTHER) where dotted key not in subs → unchanged.""" - code = ( - "@patch(constants.TARGET)\n" - "@patch(constants.OTHER)\n" - "def test_f(m1, m2):\n pass\n" - ) - result = _substitute_consts_in_func_text( - code, {"constants.TARGET": "myapp.svc.MyClass"} - ) - assert '@patch("myapp.svc.MyClass")' in result - assert "@patch(constants.OTHER)" in result - - -def test_substitute_attr_non_name_base(): - """@patch(a.b.c) where base is Attribute (not Name) → else branch, unchanged.""" - code = "@patch(a.b.c)\ndef test_f(mock):\n pass\n" - result = _substitute_consts_in_func_text(code, {"a.b.c": "should.not.replace"}) - assert "@patch(a.b.c)" in result - - -# --------------------------------------------------------------------------- -# _restore_const_refs -# --------------------------------------------------------------------------- - - -def _make_ref(const_name: str, resolved_value: str) -> _ConstRef: - return _ConstRef( - const_name=const_name, - source_file="/proj/tests/helpers.py", - resolved_value=resolved_value, - patch_dec_idx=0, - ) - - -def test_restore_reverts_unchanged_plain_name(): - """@patch("value") whose value matches a const_ref → reverted to @patch(NAME).""" - code = '@patch("myapp.svc.MyClass")\ndef test_f(mock): pass\n' - refs = [_make_ref("TARGET", "myapp.svc.MyClass")] - result = _restore_const_refs(code, refs) - assert "@patch(TARGET)" in result - assert '"myapp.svc.MyClass"' not in result - - -def test_restore_reverts_unchanged_attr_form(): - """@patch("value") matching module.CONST ref → reverted to @patch(module.CONST).""" - code = '@patch("myapp.svc.MyClass")\ndef test_f(mock): pass\n' - refs = [_make_ref("constants.TARGET", "myapp.svc.MyClass")] - result = _restore_const_refs(code, refs) - assert "@patch(constants.TARGET)" in result - assert '"myapp.svc.MyClass"' not in result - - -def test_restore_leaves_changed_value_as_literal(): - """@patch("new.value") where new.value is not in const_refs → kept as literal.""" - code = '@patch("myapp.new.MyClass")\ndef test_f(mock): pass\n' - refs = [_make_ref("TARGET", "myapp.old.MyClass")] - result = _restore_const_refs(code, refs) - assert '@patch("myapp.new.MyClass")' in result - - -def test_restore_empty_refs_unchanged(): - """No const_refs → text returned as-is.""" - code = '@patch("myapp.svc.MyClass")\ndef test_f(mock): pass\n' - assert _restore_const_refs(code, []) == code - - -def test_restore_parse_error_returns_original(): - """Unparseable text → original returned unchanged.""" - code = "def f(:\n" - refs = [_make_ref("TARGET", "myapp.svc.X")] - assert _restore_const_refs(code, refs) == code - - -def test_restore_empty_args_patch_unchanged(): - """@patch() with no args → left as-is.""" - code = "@patch()\ndef test_f(): pass\n" - refs = [_make_ref("TARGET", "myapp.svc.MyClass")] - assert _restore_const_refs(code, refs) == code - - -def test_restore_non_string_arg_unchanged(): - """@patch(NAME) where arg is a Name node (not SimpleString) → left as-is.""" - code = "@patch(OTHER_NAME)\ndef test_f(mock): pass\n" - refs = [_make_ref("TARGET", "myapp.svc.MyClass")] - result = _restore_const_refs(code, refs) - assert "@patch(OTHER_NAME)" in result - - -def test_restore_non_patch_call_untouched(): - """other_func("value") is not a patch call → left as-is.""" - code = ( - '@patch("myapp.svc.MyClass")\n' - "def test_f(mock):\n" - ' other_func("myapp.svc.OtherClass")\n' - ) - refs = [ - _make_ref("TARGET", "myapp.svc.MyClass"), - _make_ref("OTHER", "myapp.svc.OtherClass"), - ] - result = _restore_const_refs(code, refs) - assert "@patch(TARGET)" in result - assert 'other_func("myapp.svc.OtherClass")' in result - - -def test_restore_single_quote_string(): - """SimpleString with single quotes → still reverted.""" - code = "@patch('myapp.svc.MyClass')\ndef test_f(mock): pass\n" - refs = [_make_ref("TARGET", "myapp.svc.MyClass")] - result = _restore_const_refs(code, refs) - assert "@patch(TARGET)" in result - - -def test_restore_partial_revert_mixed(): - """One decorator changed, one unchanged → only unchanged one is reverted.""" - code = ( - '@patch("myapp.svc.MyClass")\n' - '@patch("myapp.new.Y")\n' - "def test_f(m1, m2): pass\n" - ) - # MyClass unchanged (should revert), Y was updated by LLM (keep literal) - refs = [ - _make_ref("TARGET", "myapp.svc.MyClass"), - _make_ref("Y_CONST", "myapp.old.Y"), # old value; new value won't match - ] - result = _restore_const_refs(code, refs) - assert "@patch(TARGET)" in result - assert '@patch("myapp.new.Y")' in result - - -# --------------------------------------------------------------------------- -# _find_test_functions_to_update — constant reference handling -# --------------------------------------------------------------------------- - - -def test_find_const_ref_same_file(tmp_path): - """@patch(CONST) where CONST is in the same file → collected, substituted.""" - src = ( - 'TARGET = "crispen.before.X"\n\n@patch(TARGET)\ndef test_f(mock_x):\n pass\n' - ) - scan = str(tmp_path / "test_foo.py") - result = _find_test_functions_to_update(src, {"crispen.before.X"}, scan_file=scan) - assert len(result) == 1 - assert result[0].function_name == "test_f" - # full_text sent to LLM has the value inlined - assert '"crispen.before.X"' in result[0].full_text - assert "TARGET" not in result[0].full_text - # const_ref recorded - assert len(result[0].const_refs) == 1 - assert result[0].const_refs[0].const_name == "TARGET" - assert result[0].const_refs[0].resolved_value == "crispen.before.X" - assert result[0].const_refs[0].patch_dec_idx == 0 - - -def test_find_const_ref_not_in_map_not_collected(tmp_path): - """@patch(UNRESOLVED) where name not in const_map → not collected.""" - src = "@patch(UNRESOLVED)\ndef test_f(mock): pass\n" - scan = str(tmp_path / "test_foo.py") - result = _find_test_functions_to_update(src, {"crispen.before.X"}, scan_file=scan) - assert result == [] - - -def test_find_const_ref_value_no_match(tmp_path): - """@patch(CONST) where const value doesn't match old_paths → not collected.""" - src = 'TARGET = "other.mod.Y"\n\n@patch(TARGET)\ndef test_f(mock): pass\n' - scan = str(tmp_path / "test_foo.py") - result = _find_test_functions_to_update(src, {"crispen.before.X"}, scan_file=scan) - assert result == [] - - -def test_find_mix_literal_and_const(tmp_path): - """Function with both a literal @patch and a const @patch → both collected.""" - src = ( - 'TARGET = "crispen.before.X"\n\n' - '@patch("crispen.before.X")\n' - "@patch(TARGET)\n" - "def test_f(m1, m2):\n pass\n" - ) - scan = str(tmp_path / "test_foo.py") - result = _find_test_functions_to_update(src, {"crispen.before.X"}, scan_file=scan) - assert len(result) == 1 - assert len(result[0].old_patch_paths) == 2 - assert len(result[0].const_refs) == 1 - # patch_dec_idx of the const ref is 1 (second @patch decorator) - assert result[0].const_refs[0].patch_dec_idx == 1 - - -def test_find_non_matching_decorator_split_into_stable(tmp_path): - """Non-matching decorators go to stable_patch_paths, not old_patch_paths. - - A test that patches get_api_key (already correct) and call_with_tool - (forking, needs rewrite) should have only call_with_tool in old_patch_paths - and get_api_key in stable_patch_paths so the LLM is not asked to evaluate - the already-correct path. - """ - src = ( - 'KEY = "crispen.mod.get_api_key"\n' - 'CALL = "crispen.mod.call_with_tool"\n\n' - "@patch(KEY)\n" - "@patch(CALL)\n" - "def test_f(mock_call, mock_key):\n pass\n" - ) - scan = str(tmp_path / "test_foo.py") - # Only CALL's value is in old_paths; KEY's value is already correct. - result = _find_test_functions_to_update( - src, {"crispen.mod.call_with_tool"}, scan_file=scan - ) - assert len(result) == 1 - # Forking path goes to old_patch_paths only. - assert result[0].old_patch_paths == ["crispen.mod.call_with_tool"] - # Already-correct path goes to stable_patch_paths. - assert result[0].stable_patch_paths == ["crispen.mod.get_api_key"] - # Both const refs must be recorded so their definitions can be updated. - assert len(result[0].const_refs) == 2 - - -def test_find_patch_no_args_increments_idx(tmp_path): - """@patch() with no args increments patch_dec_idx before the const @patch.""" - src = ( - 'TARGET = "crispen.before.X"\n\n' - "@patch()\n" - "@patch(TARGET)\n" - "def test_f(m):\n pass\n" - ) - scan = str(tmp_path / "test_foo.py") - result = _find_test_functions_to_update(src, {"crispen.before.X"}, scan_file=scan) - assert len(result) == 1 - assert result[0].const_refs[0].patch_dec_idx == 1 - - -def test_find_cross_file_const(tmp_path): - """@patch(CONST) where CONST comes from a relative import → collected.""" - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") - src = "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(mock):\n pass\n" - scan = str(tmp_path / "test_foo.py") - result = _find_test_functions_to_update( - src, {"crispen.before.X"}, scan_file=scan, repo_root=str(tmp_path) - ) - assert len(result) == 1 - assert result[0].const_refs[0].source_file == str(helpers.resolve()) - - -def test_find_attr_const_ref_collected(tmp_path): - """@patch(constants.TARGET) where ``import constants`` resolves → collected.""" - constants_file = tmp_path / "constants.py" - constants_file.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") - src = "import constants\n\n@patch(constants.TARGET)\ndef test_f(mock):\n pass\n" - scan = str(tmp_path / "test_foo.py") - result = _find_test_functions_to_update( - src, {"crispen.before.X"}, scan_file=scan, repo_root=str(tmp_path) - ) - assert len(result) == 1 - assert result[0].function_name == "test_f" - assert result[0].const_refs[0].const_name == "constants.TARGET" - assert result[0].const_refs[0].resolved_value == "crispen.before.X" - assert result[0].const_refs[0].patch_dec_idx == 0 - assert result[0].const_refs[0].source_file == str(constants_file.resolve()) - # LLM sees inlined value, not the attribute access form. - assert '"crispen.before.X"' in result[0].full_text - assert "constants.TARGET" not in result[0].full_text - - -def test_find_attr_const_module_not_in_map(tmp_path): - """@patch(unknown.TARGET) where module not in attr_const_map → not collected.""" - src = "@patch(unknown.TARGET)\ndef test_f(mock):\n pass\n" - scan = str(tmp_path / "test_foo.py") - # No ``import unknown`` in source → attr_const_map empty → no match. - result = _find_test_functions_to_update( - src, {"crispen.before.X"}, scan_file=scan, repo_root=str(tmp_path) - ) - assert result == [] - - -def test_find_attr_const_attr_not_in_module(tmp_path): - """@patch(constants.UNKNOWN) where attr not in module constants → not collected.""" - constants_file = tmp_path / "constants.py" - constants_file.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") - src = "import constants\n\n@patch(constants.UNKNOWN)\ndef test_f(mock):\n pass\n" - scan = str(tmp_path / "test_foo.py") - result = _find_test_functions_to_update( - src, {"crispen.before.X"}, scan_file=scan, repo_root=str(tmp_path) - ) - assert result == [] - - -def test_find_attr_const_value_no_match(tmp_path): - """@patch(constants.OTHER) where value doesn't match old_paths → not collected.""" - constants_file = tmp_path / "constants.py" - constants_file.write_text('OTHER = "unrelated.path.Class"\n', encoding="utf-8") - src = "import constants\n\n@patch(constants.OTHER)\ndef test_f(mock):\n pass\n" - scan = str(tmp_path / "test_foo.py") - result = _find_test_functions_to_update( - src, {"crispen.before.X"}, scan_file=scan, repo_root=str(tmp_path) - ) - assert result == [] - - -def test_find_attr_multi_level_not_handled(tmp_path): - """@patch(a.b.c) multi-level attribute (base not Name) → not collected.""" - src = "@patch(a.b.c)\ndef test_f(mock):\n pass\n" - scan = str(tmp_path / "test_foo.py") - result = _find_test_functions_to_update( - src, {"a.b.c"}, scan_file=scan, repo_root=str(tmp_path) - ) - assert result == [] - - -# --------------------------------------------------------------------------- -# _process_file_source — constant reference post-processing -# --------------------------------------------------------------------------- - -_SRC_WITH_CONST = ( - 'TARGET = "crispen.before.X"\n\n' - "@patch(TARGET)\n" - "def test_f(mock_x):\n" - " pass\n" -) - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_const_same_file_update(mock_call, tmp_path): - """Same-file const ref → same_file_const_map updates the const definition.""" - scan = str(tmp_path / "test_foo.py") - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.after.X"}, - } - ), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - _SRC_WITH_CONST, - {"crispen.before.X"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - ) - assert changed is True - # apply_patch_strings updates the const definition. - assert '"crispen.after.X"' in result - assert '"crispen.before.X"' not in result - # No cross-file updates for same-file const. - assert cross == {} - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_const_cross_file_update(mock_call, tmp_path): - """Const ref from imported file → cross_file_patch_maps returned.""" - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") - src = "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(mock):\n pass\n" - scan = str(tmp_path / "test_foo.py") - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.after.X"}, - } - ), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - src, - {"crispen.before.X"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - repo_root=str(tmp_path), - ) - helpers_abs = str(helpers.resolve()) - assert helpers_abs in cross - assert cross[helpers_abs] == {"crispen.before.X": "crispen.after.X"} - - -@mock_patch( - _PATCH_CALL_TOOL, return_value=_ok({"needs_rewrite": False, "patch_renames": {}}) -) -def test_process_const_no_change_no_cross(mock_call, tmp_path): - """LLM returns no renames → no change, cross is empty.""" - scan = str(tmp_path / "test_foo.py") - result, changed, cross = _process_file_source( - _SRC_WITH_CONST, - {"crispen.before.X"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - ) - assert changed is False - assert cross == {} - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_cross_file_const_ref_not_in_renames(mock_call, tmp_path): - """Cross-file const whose patch path is not in accepted renames → skipped. - - Scenario: function has two @patch decorators with different old paths. One - is a cross-file const ref (path A) and the other is a literal (path B). - Classify returns rename only for B; A is not in accepted renames. - The const ref for A should be skipped. - """ - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET_A = "crispen.before.A"\n', encoding="utf-8") - src = ( - "from .helpers import TARGET_A\n\n" - '@patch(TARGET_A)\n@patch("crispen.before.B")\n' - "def test_f(m1, m2):\n pass\n" - ) - scan = str(tmp_path / "test_foo.py") - # Classify: only rename crispen.before.B → crispen.after.B; - # crispen.before.A unchanged. - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.B": "crispen.after.B"}, - } - ), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - src, - {"crispen.before.A", "crispen.before.B"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - repo_root=str(tmp_path), - ) - assert changed is True - assert "crispen.after.B" in result - # crispen.before.A not in accepted renames → no cross-file update for helpers.py. - helpers_abs = str(helpers.resolve()) - assert helpers_abs not in cross - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_no_scan_file_no_const_processing(mock_call): - """scan_file="" → const_map is empty, const post-processing skipped.""" - # Even with a const-ref style source, no scan_file means no const resolution. - src = 'TARGET = "crispen.before.X"\n\n@patch(TARGET)\ndef test_f(m):\n pass\n' - # With scan_file="", const_map is empty, @patch(TARGET) is not collected. - result, changed, cross = _process_file_source( - src, {"crispen.before.X"}, "ctx", MagicMock(), _CFG, 1 - ) - assert result == src - assert changed is False - assert cross == {} - mock_call.assert_not_called() - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_attr_const_cross_file_update(mock_call, tmp_path): - """@patch(constants.TARGET) resolved via import → cross-file proposal returned.""" - constants_file = tmp_path / "constants.py" - constants_file.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") - src = ( - "import constants\n\n" - "@patch(constants.TARGET)\n" - "def test_f(mock_x):\n" - " pass\n" - ) - scan = str(tmp_path / "test_foo.py") - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.after.X"}, - } - ), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - src, - {"crispen.before.X"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - repo_root=str(tmp_path), - ) - # Cross-file proposal recorded for constants.py. - constants_abs = str(constants_file.resolve()) - assert constants_abs in cross - assert cross[constants_abs] == {"crispen.before.X": "crispen.after.X"} - - -# --------------------------------------------------------------------------- -# _apply_cross_file_const_updates -# --------------------------------------------------------------------------- - - -def test_cross_file_empty_proposals(): - msgs = list(_apply_cross_file_const_updates({}, {})) - assert msgs == [] - - -def test_cross_file_conflicting_proposals(tmp_path): - """Multiple new values for the same constant → resolved is empty → skip.""" - f = tmp_path / "helpers.py" - f.write_text('TARGET = "old.val"\n', encoding="utf-8") - proposals = {str(f.resolve()): {"old.val": {"new.val1", "new.val2"}}} - msgs = list(_apply_cross_file_const_updates(proposals, {})) - assert msgs == [] - # File unchanged. - assert f.read_text(encoding="utf-8") == 'TARGET = "old.val"\n' - - -def test_cross_file_per_file_entry_updated(tmp_path): - """Const source file is in per_file → updates in-memory source, no disk write.""" - f = tmp_path / "helpers.py" - f.write_text('TARGET = "old.val"\n', encoding="utf-8") - per_file = {str(f): {"source": 'TARGET = "old.val"\n', "msgs": []}} - proposals = {str(f.resolve()): {"old.val": {"new.val"}}} - msgs = list(_apply_cross_file_const_updates(proposals, per_file)) - assert msgs == [] - assert '"new.val"' in per_file[str(f)]["source"] - assert any("constant definition" in m for m in per_file[str(f)]["msgs"]) - # Disk file unchanged. - assert f.read_text(encoding="utf-8") == 'TARGET = "old.val"\n' - - -def test_cross_file_per_file_entry_no_change(tmp_path): - """Resolved new value equals old → apply_patch_strings makes no change → no msg.""" - f = tmp_path / "helpers.py" - src = 'TARGET = "new.val"\n' # already has new value - per_file = {str(f): {"source": src, "msgs": []}} - proposals = {str(f.resolve()): {"old.val": {"new.val"}}} - # apply_patch_strings("TARGET = "new.val"\n", {"old.val": "new.val"}) → unchanged - msgs = list(_apply_cross_file_const_updates(proposals, per_file)) - assert msgs == [] - assert per_file[str(f)]["msgs"] == [] - - -def test_cross_file_disk_file_updated(tmp_path): - """Const source file is a disk file → written, message yielded.""" - f = tmp_path / "helpers.py" - f.write_text('TARGET = "old.val"\n', encoding="utf-8") - proposals = {str(f.resolve()): {"old.val": {"new.val"}}} - msgs = list(_apply_cross_file_const_updates(proposals, {})) - assert len(msgs) == 1 - assert "constant definition" in msgs[0] - assert '"new.val"' in f.read_text(encoding="utf-8") - - -def test_cross_file_disk_file_no_change(tmp_path): - """Disk file already has the new value → no write, no message.""" - f = tmp_path / "helpers.py" - f.write_text('TARGET = "new.val"\n', encoding="utf-8") - proposals = {str(f.resolve()): {"old.val": {"new.val"}}} - msgs = list(_apply_cross_file_const_updates(proposals, {})) - assert msgs == [] - - -def test_cross_file_disk_oserror(tmp_path): - """OSError reading disk file → skipped silently.""" - f = tmp_path / "helpers.py" - f.write_text('TARGET = "old.val"\n', encoding="utf-8") - f.chmod(0o000) - try: - proposals = {str(f.resolve()): {"old.val": {"new.val"}}} - msgs = list(_apply_cross_file_const_updates(proposals, {})) - assert msgs == [] - finally: - f.chmod(0o644) - - -# --------------------------------------------------------------------------- -# apply_patch_rewrite — cross-file constant integration -# --------------------------------------------------------------------------- - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_cross_file_const_per_file(mock_key, mock_client, mock_call, tmp_path): - """Cross-file const whose source is in per_file gets updated in-memory.""" - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "pkg.big.A"\n', encoding="utf-8") - # test_foo.py imports TARGET from helpers and uses it in @patch. - test_src = ( - "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(m):\n pass\n" - ) - helpers_state = {"source": 'TARGET = "pkg.big.A"\n', "msgs": []} - per_file = { - str(tmp_path / "test_foo.py"): {"source": test_src, "msgs": []}, - str(helpers): helpers_state, - } - mock_call.side_effect = [ - _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), - _ok(_VERIFY_OK), - ] - list(apply_patch_rewrite([_make_fl_ctx()], per_file, None, _CFG)) - # The constant definition in helpers.py (per_file entry) should be updated. - assert '"pkg.sub_a.A"' in helpers_state["source"] - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_cross_file_const_disk(mock_key, mock_client, mock_call, tmp_path): - """Cross-file const on disk (not in per_file) gets written directly.""" - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "pkg.big.A"\n', encoding="utf-8") - test_src = ( - "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(m):\n pass\n" - ) - test_file = tmp_path / "test_foo.py" - test_file.write_text(test_src, encoding="utf-8") - mock_call.side_effect = [ - _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), - _ok(_VERIFY_OK), - ] - list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG)) - # The constant definition on disk should be updated. - assert '"pkg.sub_a.A"' in helpers.read_text(encoding="utf-8") - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_cross_file_const_per_file_acc( - mock_key, mock_client, mock_call, tmp_path -): - """_acc.files_updated is incremented when a cross-file const in per_file changes.""" - (tmp_path / "pyproject.toml").write_text("", encoding="utf-8") - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "pkg.big.A"\n', encoding="utf-8") - test_src = ( - "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(m):\n pass\n" - ) - helpers_state = {"source": 'TARGET = "pkg.big.A"\n', "msgs": []} - per_file = { - str(tmp_path / "test_foo.py"): {"source": test_src, "msgs": []}, - str(helpers): helpers_state, - } - mock_call.side_effect = [ - _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), - _ok(_VERIFY_OK), - ] - acc = RewriteAccumulator() - list(apply_patch_rewrite([_make_fl_ctx()], per_file, None, _CFG, _acc=acc)) - # One file_updated for the test_foo.py source change, one for helpers const. - assert acc.files_updated >= 1 - assert '"pkg.sub_a.A"' in helpers_state["source"] - - -@mock_patch(_PATCH_CALL_TOOL) -@mock_patch(_PATCH_MAKE_CLIENT, return_value=MagicMock()) -@mock_patch(_PATCH_GET_KEY, return_value="fake_key") -def test_rewrite_cross_file_const_disk_acc(mock_key, mock_client, mock_call, tmp_path): - """_acc.files_updated is incremented when a cross-file const on disk changes.""" - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "pkg.big.A"\n', encoding="utf-8") - test_src = ( - "from .helpers import TARGET\n\n@patch(TARGET)\ndef test_f(m):\n pass\n" - ) - test_file = tmp_path / "test_foo.py" - test_file.write_text(test_src, encoding="utf-8") - mock_call.side_effect = [ - _ok({"needs_rewrite": False, "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}}), - _ok(_VERIFY_OK), - ] - acc = RewriteAccumulator() - list(apply_patch_rewrite([_make_fl_ctx()], {}, str(tmp_path), _CFG, _acc=acc)) - assert acc.files_updated >= 1 - - -# --------------------------------------------------------------------------- -# Same-file constant: passthrough votes "keep old" — conflicts with rename -# --------------------------------------------------------------------------- - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_passthrough_votes_conflict_with_rename_proposal(mock_call, tmp_path): - """One test (A) renames Y but not X → casts "keep old" vote for X. - Another test (B) renames X → casts "rename" vote for X. - "keep old" + "rename" → conflicting proposals → inline test_b with new value; - test_a's decorator unchanged. TARGET2 (Y) has a single rename vote → updated - via same_file_const_map. - - Covers: - - "keep old" vote (new_val is None) entered into same_file_proposals - - conflict detection (len > 1) → conflicting_old_vals - - per-function inline for test_b (existing_idx is None → append) - - test_a in conflicting inline loop with new_val=None → inline_subs empty - → continue - - single-proposal for TARGET2 (value != old) → same_file_const_map update - """ - src = ( - 'TARGET = "crispen.before.X"\n' - 'TARGET2 = "crispen.before.Y"\n' - "\n" - "@patch(TARGET)\n" - "@patch(TARGET2)\n" - "def test_a(mock_y, mock_x):\n" - " pass\n" - "\n" - "@patch(TARGET)\n" - "def test_b(mock_x):\n" - " pass\n" - ) - scan = str(tmp_path / "test_foo.py") - # test_a renames Y but NOT X → X gets a "keep old" vote, Y gets a rename vote. - # test_b renames X → X gets a "rename to after.X" vote. - # X proposals: {old, after.X} → conflicting → inline test_b, test_a unchanged. - # Y proposals: {after.Y} → single, != old → same_file_const_map update. - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.Y": "crispen.after.Y"}, - } - ), - _ok(_VERIFY_OK), - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.after.X"}, - } - ), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - src, - {"crispen.before.X", "crispen.before.Y"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - ) - assert changed is True - # X has conflicting votes → TARGET NOT updated globally. - assert 'TARGET = "crispen.before.X"' in result - # Y has single vote → TARGET2 updated via same_file_const_map. - assert 'TARGET2 = "crispen.after.Y"' in result - # test_b's X decorator is inlined individually. - assert '@patch("crispen.after.X")' in result - # test_a's decorator unchanged (its inline_subs were empty). - assert "@patch(TARGET)" in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_passthrough_identity_proposal_skipped(mock_call, tmp_path): - """One test renames Y but not X. X only receives a "keep old" identity vote. - Expected: TARGET not updated (identity guard: proposed == old); TARGET2 updated. - - Covers the ``next(iter(new_set)) != old`` identity guard in same_file_const_map - that drops entries where the sole proposal equals the existing value. - """ - src = ( - 'TARGET = "crispen.before.X"\n' - 'TARGET2 = "crispen.before.Y"\n' - "\n" - "@patch(TARGET)\n" - "@patch(TARGET2)\n" - "def test_a(mock_y, mock_x):\n" - " pass\n" - ) - scan = str(tmp_path / "test_foo.py") - # test_a: renames Y → after.Y, does not rename X. - # X proposals: {"crispen.before.X"} → len==1, value==old → identity skip. - # Y proposals: {"crispen.after.Y"} → len==1, value!=old → const_map update. - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.Y": "crispen.after.Y"}, - } - ), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - src, - {"crispen.before.X", "crispen.before.Y"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - ) - assert changed is True - # X got only an identity vote → not in same_file_const_map → TARGET unchanged. - assert 'TARGET = "crispen.before.X"' in result - # Y got a rename vote → TARGET2 updated. - assert 'TARGET2 = "crispen.after.Y"' in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_conflict_two_renames_existing_splice(mock_call, tmp_path): - """Two tests rename the same constant to *different* targets → conflict. - test_a also renames a literal patch → it gets a func_splice from string_swap. - Expected: both functions get inlined with their respective literals; - test_a's existing splice is *updated in place* (existing_idx path). - - Covers: - - lines 1763-1772 (loop, build inline_subs) - - line 1787-False (inlined != base_text) - - line 1789-True (existing_idx not None → update splice) - - line 1792 (existing_idx is None → append splice, for test_b) - """ - src = ( - 'TARGET = "crispen.before.X"\n' - "\n" - "@patch(TARGET)\n" - '@patch("crispen.before.Z")\n' - "def test_a(mock_z, mock_x):\n" - " pass\n" - "\n" - "@patch(TARGET)\n" - "def test_b(mock_x):\n" - " pass\n" - ) - scan = str(tmp_path / "test_foo.py") - # test_a renames X → after_a.X and Z → after.Z. - # test_b renames X → after_b.X. - # Two different targets for X → conflict → inline each function individually. - # test_a's Z literal rename creates an existing func_splice; the inline step - # must update that existing splice rather than appending a new one. - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": { - "crispen.before.X": "crispen.after_a.X", - "crispen.before.Z": "crispen.after.Z", - }, - } - ), - _ok(_VERIFY_OK), - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.after_b.X"}, - } - ), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - src, - {"crispen.before.X", "crispen.before.Z"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - ) - assert changed is True - # The shared TARGET constant must NOT be updated (conflict). - assert 'TARGET = "crispen.before.X"' in result - # test_a: Z literal renamed, X constant inlined. - assert '@patch("crispen.after_a.X")' in result - assert '@patch("crispen.after.Z")' in result - # test_b: X constant inlined with its own target. - assert '@patch("crispen.after_b.X")' in result - # No original constant-style decorator survives. - assert "@patch(TARGET)" not in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_process_conflict_two_proposals_passthrough_function_continue( - mock_call, tmp_path -): - """Two functions propose *different* values for TARGET → conflicting_old_vals. - A third function also uses TARGET but only renames a different const (TARGET_Y). - Expected: the third function is in string_swap_results but triggers the - ``continue`` branch in the conflicting_old_vals inline loop (inline_subs - empty for X); the other two get their decorators inlined individually. - - Covers the ``if not inline_subs: continue`` branch inside the - ``if conflicting_old_vals:`` block (via two sub-paths): - - ref.resolved_value NOT in conflicting_old_vals (Y ref → loop continues) - - ref.resolved_value in conflicting_old_vals but new_val is None (X ref) - """ - src = ( - 'TARGET = "crispen.before.X"\n' - 'TARGET_Y = "crispen.before.Y"\n' - "\n" - "@patch(TARGET)\n" - "def test_a(mock_x):\n" - " pass\n" - "\n" - "@patch(TARGET)\n" - "def test_b(mock_x):\n" - " pass\n" - "\n" - "@patch(TARGET)\n" - "@patch(TARGET_Y)\n" - "def test_c(mock_y, mock_x):\n" - " pass\n" - ) - scan = str(tmp_path / "test_foo.py") - # test_a → after_a.X; test_b → after_b.X (two different proposals → conflicting) - # test_c → renames Y only (not X) → in string_swap_results but inline_subs empty - # for X → continue. Y gets a single proposal → same_file_const_map update. - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.after_a.X"}, - } - ), - _ok(_VERIFY_OK), - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.after_b.X"}, - } - ), - _ok(_VERIFY_OK), - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.Y": "crispen.after.Y"}, - } - ), - _ok(_VERIFY_OK), - ] - result, changed, cross = _process_file_source( - src, - {"crispen.before.X", "crispen.before.Y"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - ) - assert changed is True - # X: conflicting (two proposals) → const unchanged, test_a and test_b inlined. - assert 'TARGET = "crispen.before.X"' in result - assert '@patch("crispen.after_a.X")' in result - assert '@patch("crispen.after_b.X")' in result - # Y: single proposal → const updated via same_file_const_map. - assert 'TARGET_Y = "crispen.after.Y"' in result - # test_c: in string_swap_results (renamed Y) but X inline_subs empty → continue. - assert "@patch(TARGET)" in result - - -# --------------------------------------------------------------------------- -# Call-graph helpers: _cg_collect_called_names -# --------------------------------------------------------------------------- - - -def test_cg_collect_called_names_name_and_attr(): - src = "foo()\nobj.bar()\n" - result = _cg_collect_called_names(src) - assert "foo" in result - assert "bar" in result - - -def test_cg_collect_called_names_complex_func(): - # f()() — outer call's func is a Call node (neither Name nor Attribute). - src = "f()()\n" - result = _cg_collect_called_names(src) - # Only the inner call's name is collected (f), the outer call is skipped. - assert "f" in result - - -def test_cg_collect_called_names_parse_error(): - assert _cg_collect_called_names("def f(:\n") == set() - - -def test_cg_collect_called_names_no_calls(): - assert _cg_collect_called_names("x = 1\n") == set() - - -# --------------------------------------------------------------------------- -# _cg_collect_func_body_calls -# --------------------------------------------------------------------------- - - -def test_cg_collect_func_body_calls_found(): - src = "def helper(): foo()\ndef other(): bar()\n" - result = _cg_collect_func_body_calls(src, "helper") - assert "foo" in result - assert "bar" not in result - - -def test_cg_collect_func_body_calls_not_found(): - src = "def helper(): foo()\n" - assert _cg_collect_func_body_calls(src, "missing") == set() - - -def test_cg_collect_func_body_calls_parse_error(): - assert _cg_collect_func_body_calls("def f(:\n", "f") == set() - - -def test_cg_collect_func_body_calls_attribute_call(): - src = "def helper(): obj.method()\n" - result = _cg_collect_func_body_calls(src, "helper") - assert "method" in result - - -def test_cg_collect_func_body_calls_complex_func(): - # f()() inside a function body — outer call's func is a Call, not Name/Attribute. - src = "def helper(): f()()\n" - result = _cg_collect_func_body_calls(src, "helper") - assert "f" in result # inner call collected; outer (complex func) silently skipped - - -def test_cg_collect_func_body_calls_skips_non_function_nodes(): - # Module-level assignment before the function — should be skipped. - src = "X = 1\ndef helper(): foo()\n" - result = _cg_collect_func_body_calls(src, "helper") - assert "foo" in result - - -# --------------------------------------------------------------------------- -# _cg_collect_called_names — alias-access form -# --------------------------------------------------------------------------- - - -def test_cg_collect_called_names_alias_access_emits_pair(): - # ``m.func()`` should emit both ``"func"`` and ``"m.func"``. - src = "import mymod as m\nm.func()\n" - result = _cg_collect_called_names(src) - assert "func" in result - assert "m.func" in result - - -def test_cg_collect_called_names_nested_attr_no_alias_pair(): - # ``a.b.c()`` — the receiver of ``.c`` is itself an Attribute, not a Name; - # only the bare attr name is emitted (no alias pair for chained access). - src = "a.b.c()\n" - result = _cg_collect_called_names(src) - assert "c" in result - assert "b.c" not in result # receiver is Attribute, not Name - - -# --------------------------------------------------------------------------- -# _cg_collect_func_body_calls — alias-access form -# --------------------------------------------------------------------------- - - -def test_cg_collect_func_body_calls_alias_access_emits_pair(): - src = "import mymod as m\ndef helper(): m.process()\n" - result = _cg_collect_func_body_calls(src, "helper") - assert "process" in result - assert "m.process" in result - - -def test_cg_collect_func_body_calls_nested_attr_no_alias_pair(): - # Chained access ``a.b.c()`` — receiver of attr c is not a Name. - src = "def helper(): a.b.c()\n" - result = _cg_collect_func_body_calls(src, "helper") - assert "c" in result - assert "b.c" not in result - - -# --------------------------------------------------------------------------- -# _cg_resolve_call_to_import -# --------------------------------------------------------------------------- - - -def test_cg_resolve_call_plain_name(): - imports = {"foo": ("pkg.sub", "foo"), "bar": ("pkg.other", "bar")} - assert _cg_resolve_call_to_import("foo", imports) == ("pkg.sub", "foo") - - -def test_cg_resolve_call_alias_attr(): - # ``m.process()`` — alias ``m`` maps to module ``mymod``; resolves to - # ``(mymod, "process")``. - imports = {"m": ("mymod", "mymod")} - assert _cg_resolve_call_to_import("m.process", imports) == ("mymod", "process") - - -def test_cg_resolve_call_alias_attr_unknown_alias(): - # Alias not in imports → None. - assert _cg_resolve_call_to_import("unknown.func", {"m": ("mymod", "mymod")}) is None - - -def test_cg_resolve_call_plain_not_found(): - assert _cg_resolve_call_to_import("missing", {"foo": ("pkg", "foo")}) is None - - -# --------------------------------------------------------------------------- -# _cg_collect_defined_names -# --------------------------------------------------------------------------- - - -def test_cg_collect_defined_names_functions_and_classes(): - src = "def foo(): pass\nclass Bar: pass\nasync def baz(): pass\n" - result = _cg_collect_defined_names(src) - assert result == {"foo", "Bar", "baz"} - - -def test_cg_collect_defined_names_parse_error(): - assert _cg_collect_defined_names("def f(:\n") == set() - - -def test_cg_collect_defined_names_empty(): - assert _cg_collect_defined_names("x = 1\n") == set() - - -# --------------------------------------------------------------------------- -# _cg_file_to_module_and_package -# --------------------------------------------------------------------------- - - -def test_cg_file_to_module_regular(tmp_path): - pkg = tmp_path / "pkg" - pkg.mkdir() - f = pkg / "helpers.py" - f.touch() - mod, pkg_path = _cg_file_to_module_and_package(f, tmp_path) - assert mod == "pkg.helpers" - assert pkg_path == "pkg" - - -def test_cg_file_to_module_init(tmp_path): - d = tmp_path / "pkg" / "utils" - d.mkdir(parents=True) - f = d / "__init__.py" - f.touch() - mod, pkg_path = _cg_file_to_module_and_package(f, tmp_path) - assert mod == "pkg.utils" - assert pkg_path == "pkg.utils" - - -def test_cg_file_to_module_top_level(tmp_path): - f = tmp_path / "helpers.py" - f.touch() - mod, pkg_path = _cg_file_to_module_and_package(f, tmp_path) - assert mod == "helpers" - assert pkg_path == "" - - -def test_cg_file_to_module_nested(tmp_path): - d = tmp_path / "a" / "b" - d.mkdir(parents=True) - f = d / "c.py" - f.touch() - mod, pkg_path = _cg_file_to_module_and_package(f, tmp_path) - assert mod == "a.b.c" - assert pkg_path == "a.b" - - -# --------------------------------------------------------------------------- -# _cg_parse_imports -# --------------------------------------------------------------------------- - - -def test_cg_parse_imports_from_import(): - assert _cg_parse_imports("from pkg.sub import foo\n", "pkg") == { - "foo": ("pkg.sub", "foo") - } - - -def test_cg_parse_imports_import_simple(): - result = _cg_parse_imports("import os\n", "pkg") - assert result["os"] == ("os", "os") - - -def test_cg_parse_imports_import_dotted(): - result = _cg_parse_imports("import pkg.sub\n", "") - assert result["pkg"] == ("pkg.sub", "pkg.sub") - - -def test_cg_parse_imports_import_as(): - result = _cg_parse_imports("import os as o\n", "pkg") - assert result["o"] == ("os", "os") - - -def test_cg_parse_imports_from_import_as(): - assert _cg_parse_imports("from pkg import foo as bar\n", "pkg") == { - "bar": ("pkg", "foo") - } - - -def test_cg_parse_imports_relative_level1(): - # `from . import helper` with package "pkg.sub" → mod = "pkg.sub" - assert _cg_parse_imports("from . import helper\n", "pkg.sub") == { - "helper": ("pkg.sub", "helper") - } - - -def test_cg_parse_imports_relative_level2(): - # `from .. import foo` with package "pkg.sub" → base = "pkg" - assert _cg_parse_imports("from .. import foo\n", "pkg.sub") == { - "foo": ("pkg", "foo") - } - - -def test_cg_parse_imports_relative_with_module(): - # `from .utils import helper` with package "pkg" → mod = "pkg.utils" - assert _cg_parse_imports("from .utils import helper\n", "pkg") == { - "helper": ("pkg.utils", "helper") - } - - -def test_cg_parse_imports_relative_with_empty_base(): - # `from .sub import foo` with empty package → base="" → mod = "sub" - assert _cg_parse_imports("from .sub import foo\n", "") == {"foo": ("sub", "foo")} - - -def test_cg_parse_imports_relative_no_module(): - # `from . import bar` with package "pkg.sub" → mod = "pkg.sub" - assert _cg_parse_imports("from . import bar\n", "pkg.sub") == { - "bar": ("pkg.sub", "bar") - } - - -def test_cg_parse_imports_star_skipped(): - assert _cg_parse_imports("from pkg import *\n", "pkg") == {} - - -def test_cg_parse_imports_syntax_error(): - assert _cg_parse_imports("def f(:\n", "pkg") == {} - - -def test_cg_parse_imports_too_deep_relative(): - # level=3 with package="pkg" → go_up=2 > len(["pkg"])=1 → skipped - assert _cg_parse_imports("from ... import foo\n", "pkg") == {} - - -def test_cg_parse_imports_level2_with_submodule(): - # `from ..utils import foo` with package "pkg.sub" → base="pkg" → "pkg.utils" - assert _cg_parse_imports("from ..utils import foo\n", "pkg.sub") == { - "foo": ("pkg.utils", "foo") - } - - -# --------------------------------------------------------------------------- -# _CgIndex.get_imports -# --------------------------------------------------------------------------- - - -def test_cg_index_get_imports_cached(): - index = _CgIndex( - module_to_source={"pkg.mod": "from pkg.sub import foo\n"}, - module_to_package={"pkg.mod": "pkg"}, - module_to_defs={"pkg.mod": set()}, - file_to_module={}, - ) - r1 = index.get_imports("pkg.mod") - r2 = index.get_imports("pkg.mod") # second call — cached - assert r1 == r2 == {"foo": ("pkg.sub", "foo")} - assert "pkg.mod" in index._import_cache - - -def test_cg_index_get_imports_missing_module(): - index = _CgIndex( - module_to_source={}, - module_to_package={}, - module_to_defs={}, - file_to_module={}, - ) - assert index.get_imports("nonexistent") == {} - - -# --------------------------------------------------------------------------- -# _cg_build_index -# --------------------------------------------------------------------------- - - -def test_cg_build_index_from_repo(tmp_path): - pkg = tmp_path / "pkg" - pkg.mkdir() - (pkg / "mod.py").write_text("def foo(): pass\n", encoding="utf-8") - index = _cg_build_index(str(tmp_path), {}, []) - assert "pkg.mod" in index.module_to_source - assert "foo" in index.module_to_defs["pkg.mod"] - assert index.module_to_package["pkg.mod"] == "pkg" - - -def test_cg_build_index_per_file_override(tmp_path): - pkg = tmp_path / "pkg" - pkg.mkdir() - f = pkg / "mod.py" - f.write_text("def old(): pass\n", encoding="utf-8") - abs_path = str(f.resolve()) - index = _cg_build_index(str(tmp_path), {abs_path: "def new(): pass\n"}, []) - assert "new" in index.module_to_defs["pkg.mod"] - assert "old" not in index.module_to_defs["pkg.mod"] - - -def test_cg_build_index_no_repo_root(): - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="", - modified_source="", - new_files={"placement.py": "def helper(): pass\n"}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths=set(), - ) - index = _cg_build_index(None, {}, [ctx]) - assert "pkg.placement" in index.module_to_source - assert index.file_to_module == {} - - -def test_cg_build_index_excluded_dirs(tmp_path): - venv = tmp_path / ".venv" - venv.mkdir() - (venv / "mod.py").write_text("def foo(): pass\n", encoding="utf-8") - index = _cg_build_index(str(tmp_path), {}, []) - assert "mod" not in index.module_to_source - - -def test_cg_build_index_new_files_from_context(): - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="", - modified_source="", - new_files={"placement.py": "def helper(): pass\n"}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths=set(), - ) - index = _cg_build_index(None, {}, [ctx]) - assert "helper" in index.module_to_defs["pkg.placement"] - assert index.module_to_package["pkg.placement"] == "pkg" - - -def test_cg_build_index_init_package(tmp_path): - pkg = tmp_path / "pkg" - pkg.mkdir() - (pkg / "__init__.py").write_text("from .sub import foo\n", encoding="utf-8") - (pkg / "sub.py").write_text("def foo(): pass\n", encoding="utf-8") - index = _cg_build_index(str(tmp_path), {}, []) - assert "pkg" in index.module_to_source - assert index.module_to_package["pkg"] == "pkg" - - -def test_cg_build_index_already_in_index(): - ctx1 = _FLContext( - filepath="/proj/orig.py", - old_module="orig", - original_source="", - modified_source="", - new_files={"placement.py": "def first(): pass\n"}, - new_module_paths={"placement.py": "pkg.shared"}, - entity_to_target={}, - forking_old_paths=set(), - ) - ctx2 = _FLContext( - filepath="/proj/orig2.py", - old_module="orig2", - original_source="", - modified_source="", - new_files={"placement.py": "def second(): pass\n"}, - new_module_paths={"placement.py": "pkg.shared"}, # same module path - entity_to_target={}, - forking_old_paths=set(), - ) - index = _cg_build_index(None, {}, [ctx1, ctx2]) - assert "first" in index.module_to_defs["pkg.shared"] - assert "second" not in index.module_to_defs["pkg.shared"] - - -def test_cg_build_index_oserror(tmp_path): - pkg = tmp_path / "pkg" - pkg.mkdir() - bad = pkg / "bad.py" - bad.write_text("def foo(): pass\n", encoding="utf-8") - bad.chmod(0o000) - try: - index = _cg_build_index(str(tmp_path), {}, []) - assert "pkg.bad" not in index.module_to_source - finally: - bad.chmod(0o644) - - -def test_cg_build_index_missing_module_path(): - ctx = _FLContext( - filepath="/proj/orig.py", - old_module="orig", - original_source="", - modified_source="", - new_files={"placement.py": "def helper(): pass\n"}, - new_module_paths={}, # rel_path missing → new_mod = None → skip - entity_to_target={}, - forking_old_paths=set(), - ) - index = _cg_build_index(None, {}, [ctx]) - assert "placement.py" not in index.module_to_source - - -def test_cg_build_index_empty_src(): - ctx = _FLContext( - filepath="/proj/orig.py", - old_module="orig", - original_source="", - modified_source="", - new_files={"placement.py": ""}, # empty src → skipped - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths=set(), - ) - index = _cg_build_index(None, {}, [ctx]) - assert "pkg.placement" not in index.module_to_source - - -def test_cg_build_index_init_package_new_file(): - # __init__.py as a new file: pkg = new_mod (not rsplit) - ctx = _FLContext( - filepath="/proj/orig.py", - old_module="orig", - original_source="", - modified_source="", - new_files={"__init__.py": "def init_fn(): pass\n"}, - new_module_paths={"__init__.py": "pkg.sub"}, - entity_to_target={}, - forking_old_paths=set(), - ) - index = _cg_build_index(None, {}, [ctx]) - assert index.module_to_package["pkg.sub"] == "pkg.sub" - - -def test_cg_build_index_top_level_new_file(): - # new_mod without a dot → package = "" - ctx = _FLContext( - filepath="/proj/orig.py", - old_module="orig", - original_source="", - modified_source="", - new_files={"placement.py": "def helper(): pass\n"}, - new_module_paths={"placement.py": "placement"}, # no dot - entity_to_target={}, - forking_old_paths=set(), - ) - index = _cg_build_index(None, {}, [ctx]) - assert index.module_to_package.get("placement") == "" - - -# --------------------------------------------------------------------------- -# _resolve_forking_path_via_callgraph — BFS helpers -# --------------------------------------------------------------------------- - - -def _make_bfs_ctx() -> _FLContext: - """Context with placement.py (helper) and conflict.py (resolve) using use_fn.""" - return _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="from .placement import helper\n", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", - "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", - }, - new_module_paths={ - "placement.py": "pkg.placement", - "conflict.py": "pkg.conflict", - }, - entity_to_target={"helper": "placement.py"}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - - -def _make_bfs_index(test_src: str, calling_module: str = "pkg.test_mod") -> _CgIndex: - """Minimal index: only the calling module's source (for import resolution).""" - parts = calling_module.split(".") - pkg = ".".join(parts[:-1]) if len(parts) > 1 else "" - return _CgIndex( - module_to_source={calling_module: test_src}, - module_to_package={calling_module: pkg}, - module_to_defs={calling_module: set()}, - file_to_module={}, - ) - - -# --------------------------------------------------------------------------- -# _resolve_forking_path_via_callgraph — tests -# --------------------------------------------------------------------------- - - -def test_resolve_callgraph_no_calling_module(): - ctx = _make_bfs_ctx() - index = _make_bfs_index("from pkg.placement import helper\n") - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): helper()\n", ctx, index, "" - ) - assert result is None - - -def test_resolve_callgraph_pre_check_fails(): - # original_source has no external import of 'use_fn' → pre-check fails - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="def helper(): use_fn()\n", # not imported externally - modified_source="", - new_files={"placement.py": "def helper(): use_fn()\n"}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - index = _make_bfs_index("from pkg.placement import helper\n") - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_no_terminal(): - # New files don't reference 'use_fn' → terminal empty → None - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": "def helper(): pass\n"}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - index = _make_bfs_index("from pkg.placement import helper\n") - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_direct_call(): - # Test directly calls 'helper'; helper in placement uses use_fn. - ctx = _make_bfs_ctx() - index = _make_bfs_index("from pkg.placement import helper\n") - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" - ) - assert result == "pkg.placement.use_fn" - - -def test_resolve_callgraph_multi_hop(): - # Test → intermediary → helper → terminal (placement.use_fn) - ctx = _make_bfs_ctx() - middle_src = "from pkg.placement import helper\ndef intermediary(): helper()\n" - test_src = "from pkg.middle import intermediary\n" - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, - module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, - module_to_defs={"pkg.test_mod": set(), "pkg.middle": {"intermediary"}}, - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): intermediary()\n", ctx, index, "pkg.test_mod" - ) - assert result == "pkg.placement.use_fn" - - -def test_resolve_callgraph_reexport(): - # Test imports helper from pkg.orig; pkg.orig re-exports helper from placement. - # Re-export is followed without incrementing depth. - ctx = _make_bfs_ctx() - orig_src = "from .placement import helper\n" # re-exports (fn not defined) - test_src = "from pkg.orig import helper\n" - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src, "pkg.orig": orig_src}, - module_to_package={"pkg.test_mod": "pkg", "pkg.orig": "pkg"}, - module_to_defs={"pkg.test_mod": set(), "pkg.orig": set()}, # helper not defined - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" - ) - assert result == "pkg.placement.use_fn" - - -def test_resolve_callgraph_multiple_candidates(): - # Both placement.helper and conflict.resolve are reachable → ambiguous → None. - ctx = _make_bfs_ctx() - test_src = "from pkg.placement import helper\n" "from pkg.conflict import resolve\n" - index = _make_bfs_index(test_src) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): helper(); resolve()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_not_reachable(): - # Test doesn't import anything relevant → BFS queue empty → None. - ctx = _make_bfs_ctx() - index = _make_bfs_index("") # no imports - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): unrelated()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_new_submodule_non_terminal(): - # Test imports 'other' from placement; 'other' doesn't use use_fn. - # 'other' is in a new sub-module but NOT in terminal → skipped. - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "placement.py": ( - "from external import use_fn\n" - "def helper(): use_fn()\n" - "def other(): pass\n" - ) - }, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = "from pkg.placement import other\n" - index = _make_bfs_index(test_src) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): other()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_init_reexport(): - # pkg/orig.py split into pkg/orig/__init__.py (re-exports helper) and - # pkg/orig/placement.py (defines helper, uses use_fn). - # Test imports helper from pkg.orig (the new __init__). - # __init__ is excluded from new_module_set so BFS traverses through it - # and follows the re-export to placement.py, finding the terminal. - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "orig/__init__.py": "from .placement import helper\n", - "orig/placement.py": ( - "from external import use_fn\ndef helper(): use_fn()\n" - ), - }, - new_module_paths={ - "orig/__init__.py": "pkg.orig", - "orig/placement.py": "pkg.orig.placement", - }, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - init_src = "from .placement import helper\n" - test_src = "from pkg.orig import helper\n" - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src, "pkg.orig": init_src}, - module_to_package={"pkg.test_mod": "pkg", "pkg.orig": "pkg.orig"}, - module_to_defs={"pkg.test_mod": set(), "pkg.orig": set()}, - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" - ) - assert result == "pkg.orig.placement.use_fn" - - -def test_resolve_callgraph_visited_dedup(): - # 'intermediary' and 'inter2' both map to same (module, func); processed once. - ctx = _make_bfs_ctx() - middle_src = "from pkg.placement import helper\ndef intermediary(): helper()\n" - test_src = ( - "from pkg.middle import intermediary\n" - "from pkg.middle import intermediary as inter2\n" - ) - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, - module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, - module_to_defs={"pkg.test_mod": set(), "pkg.middle": {"intermediary"}}, - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", - "def test_f(): intermediary(); inter2()\n", - ctx, - index, - "pkg.test_mod", - ) - assert result == "pkg.placement.use_fn" - - -def test_resolve_callgraph_empty_new_file(): - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "empty.py": "", - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", - }, - new_module_paths={"empty.py": "pkg.empty", "placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - index = _make_bfs_index("from pkg.placement import helper\n") - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" - ) - assert result == "pkg.placement.use_fn" - - -def test_resolve_callgraph_missing_module_path(): - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": "from external import use_fn\ndef f(): use_fn()\n"}, - new_module_paths={}, # missing → terminal empty → None - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - index = _make_bfs_index("from pkg.placement import f\n") - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): f()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_missing_source(): - # Called function's module is not in the index → src=None → continue - ctx = _make_bfs_ctx() - test_src = "from pkg.missing import something\n" - index = _make_bfs_index(test_src) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): something()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_func_defined_no_calls(): - # Function IS defined but has no imported calls → BFS dead-end → None - ctx = _make_bfs_ctx() - middle_src = "def standalone(): pass\n" - test_src = "from pkg.middle import standalone\n" - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, - module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, - module_to_defs={"pkg.test_mod": set(), "pkg.middle": {"standalone"}}, - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): standalone()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_body_call_not_importable(): - # Function's body calls something not in its import map → BFS dead-end → None - ctx = _make_bfs_ctx() - middle_src = "def fn(): bar()\n" # bar not imported - test_src = "from pkg.middle import fn\n" - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, - module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, - module_to_defs={"pkg.test_mod": set(), "pkg.middle": {"fn"}}, - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): fn()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_func_not_defined_not_reexported(): - # func_name not defined and not re-exported → BFS dead-end → None - ctx = _make_bfs_ctx() - middle_src = "def other(): pass\n" # 'fn' not defined, not re-exported - test_src = "from pkg.middle import fn\n" - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, - module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, - module_to_defs={"pkg.test_mod": set(), "pkg.middle": {"other"}}, - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): fn()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_body_call_already_visited(): - # fn_a → fn_b → fn_a (mutual recursion); fn_a already visited when fn_b adds it - ctx = _make_bfs_ctx() - m_a = "from pkg.m_b import fn_b\ndef fn_a(): fn_b()\n" - m_b = "from pkg.m_a import fn_a\ndef fn_b(): fn_a()\n" - test_src = "from pkg.m_a import fn_a\n" - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src, "pkg.m_a": m_a, "pkg.m_b": m_b}, - module_to_package={ - "pkg.test_mod": "pkg", - "pkg.m_a": "pkg", - "pkg.m_b": "pkg", - }, - module_to_defs={ - "pkg.test_mod": set(), - "pkg.m_a": {"fn_a"}, - "pkg.m_b": {"fn_b"}, - }, - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): fn_a()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_reexport_already_visited(): - # m_x re-exports fn_x from m_y; m_y re-exports fn_x from m_x (cycle). - # When m_y checks re-export, (m_x, fn_x) is already visited → skip. - ctx = _make_bfs_ctx() - m_x = "from pkg.m_y import fn_x\n" # re-exports fn_x from m_y - m_y = "from pkg.m_x import fn_x\n" # re-exports fn_x from m_x (cycle) - test_src = "from pkg.m_x import fn_x\n" - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src, "pkg.m_x": m_x, "pkg.m_y": m_y}, - module_to_package={ - "pkg.test_mod": "pkg", - "pkg.m_x": "pkg", - "pkg.m_y": "pkg", - }, - module_to_defs={"pkg.test_mod": set(), "pkg.m_x": set(), "pkg.m_y": set()}, - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): fn_x()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_depth_limit(): - # Chain of _CG_MAX_DEPTH + 1 hops; last function calls terminal but is cut off. - n = _CG_MAX_DEPTH + 1 # 13 intermediate functions - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"end.py": "from external import use_fn\ndef end_fn(): use_fn()\n"}, - new_module_paths={"end.py": "pkg.end"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - modules = {} - for i in range(n): - if i < n - 1: - src = f"from pkg.m{i + 1} import f{i + 1}\ndef f{i}(): f{i + 1}()\n" - else: - src = f"from pkg.end import end_fn\ndef f{i}(): end_fn()\n" - modules[f"pkg.m{i}"] = src - modules["pkg.test_mod"] = "from pkg.m0 import f0\n" - defs = {m: _cg_collect_defined_names(s) for m, s in modules.items()} - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs=defs, - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): f0()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_module_limit(): - # Re-export chain of _CG_MAX_MODULES + 1 unique modules; 51st is cut off. - n = _CG_MAX_MODULES # 50 re-export hops before the cut-off - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"end.py": "from external import use_fn\ndef final(): use_fn()\n"}, - new_module_paths={"end.py": "pkg.end"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - modules = {} - for i in range(n + 1): - if i < n: - modules[f"pkg.m{i}"] = f"from pkg.m{i + 1} import fn\n" - else: - modules[f"pkg.m{i}"] = "from pkg.end import final\ndef fn(): final()\n" - modules["pkg.test_mod"] = "from pkg.m0 import fn\n" - defs = {m: _cg_collect_defined_names(s) for m, s in modules.items()} - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs=defs, - file_to_module={}, - ) - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): fn()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -def test_resolve_callgraph_same_module_twice(): - """Two called names both resolve to the same intermediate module. - - The second BFS entry hits the 'module already in modules_seen' fast path - (branch 1250->1255 in patch_rewriter.py). - """ - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"end.py": "from external import use_fn\ndef final(): use_fn()\n"}, - new_module_paths={"end.py": "pkg.end"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - # Both fn_a and fn_b live in pkg.middle; neither calls anything reachable. - middle_src = "def fn_a(): pass\ndef fn_b(): pass\n" - modules = { - "pkg.test_mod": "from pkg.middle import fn_a, fn_b\n", - "pkg.middle": middle_src, - } - defs = {m: _cg_collect_defined_names(s) for m, s in modules.items()} - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs=defs, - file_to_module={}, - ) - # BFS enqueues (pkg.middle, fn_a, 0) and (pkg.middle, fn_b, 0). - # First pop adds pkg.middle to modules_seen; second pop hits the fast path. - result = _resolve_forking_path_via_callgraph( - "use_fn", "def test_f(): fn_a(); fn_b()\n", ctx, index, "pkg.test_mod" - ) - assert result is None - - -# --------------------------------------------------------------------------- -# _callgraph_update_file — helpers -# --------------------------------------------------------------------------- - - -def _make_cuf_contexts() -> list: - """FL context with placement (helper) and conflict (resolve) using use_fn.""" - return [ - _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\ndef helper(): use_fn()\n", - modified_source="from .placement import helper\n", - new_files={ - "placement.py": ( - "from external import use_fn\ndef helper(): use_fn()\n" - ), - "conflict.py": ( - "from external import use_fn\ndef resolve(): use_fn()\n" - ), - }, - new_module_paths={ - "placement.py": "pkg.placement", - "conflict.py": "pkg.conflict", - }, - entity_to_target={"helper": "placement.py"}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - ] - - -def _make_cuf_index(scan_abs: str, test_src: str) -> _CgIndex: - """Minimal index for _callgraph_update_file: maps scan_abs → 'pkg.test_mod'.""" - return _CgIndex( - module_to_source={"pkg.test_mod": test_src}, - module_to_package={"pkg.test_mod": "pkg"}, - module_to_defs={"pkg.test_mod": set()}, - file_to_module={scan_abs: "pkg.test_mod"}, - ) - - -# --------------------------------------------------------------------------- -# _callgraph_update_file — tests -# --------------------------------------------------------------------------- - - -def test_callgraph_update_file_no_functions(): - src = "x = 1\n" - result, changed, _unresolved = _callgraph_update_file( - src, {"pkg.orig.use_fn"}, _make_cuf_contexts() - ) - assert not changed - assert result == src - - -def test_callgraph_update_file_index_none(tmp_path): - # index=None → BFS skipped → no resolution even if test calls helper. - test_src = ( - "from pkg.placement import helper\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(mock_use_fn):\n" - " helper()\n" - ) - scan = str(tmp_path / "test_foo.py") - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - _make_cuf_contexts(), - scan_file=scan, - index=None, - ) - assert not changed - - -def test_callgraph_update_file_string_literal_resolved(tmp_path): - test_src = ( - "from pkg.placement import helper\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(mock_use_fn):\n" - " helper()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - _make_cuf_contexts(), - scan_file=scan, - index=index, - ) - assert changed - assert '@patch("pkg.placement.use_fn")' in result - - -def test_callgraph_update_file_acc_cg_resolved(tmp_path): - # _acc.cg_resolved incremented for each resolved path. - test_src = ( - "from pkg.placement import helper\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(mock_use_fn):\n" - " helper()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - acc = RewriteAccumulator() - _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - _make_cuf_contexts(), - scan_file=scan, - index=index, - _acc=acc, - ) - assert acc.cg_resolved == 1 - - -def test_callgraph_update_file_no_resolution(tmp_path): - # Test calls 'unrelated' — not imported → BFS queue empty → no resolution. - # Static fallback has 2 candidates (placement + conflict) → unresolved saved. - test_src = ( - '@patch("pkg.orig.use_fn")\n' "def test_f(mock_use_fn):\n" " unrelated()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - _make_cuf_contexts(), - scan_file=scan, - index=index, - ) - assert not changed - cands = unresolved.get("test_f", {}).get("pkg.orig.use_fn", []) - assert sorted(cands) == ["pkg.conflict.use_fn", "pkg.placement.use_fn"] - - -def test_callgraph_update_file_zero_cands_single_static_auto_resolve(tmp_path): - # BFS finds 0 candidates but static terminal has exactly 1 → auto-resolve. - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = ( - '@patch("pkg.orig.use_fn")\n' "def test_f(mock_use_fn):\n" " unrelated()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - [ctx], - scan_file=scan, - index=index, - ) - assert changed - assert '@patch("pkg.placement.use_fn")' in result - assert "test_f" not in unresolved - - -def test_callgraph_update_file_zero_cands_single_static_clears_unresolved(tmp_path): - # ctx_ambig: BFS finds 2 candidates (saves to unresolved). - # ctx_uniq_static: BFS finds 0, static has 1 → auto-resolves AND clears the - # previously saved unresolved entry (exercises the delete-entry branch). - test_src = ( - "from pkg.placement import helper\n" - "from pkg.conflict import resolve\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(m):\n" - " helper()\n" - " resolve()\n" - ) - ctx_ambig = _make_cuf_contexts()[0] # placement + conflict → 2 BFS candidates - ctx_uniq_static = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"singleton.py": "from external import use_fn\ndef fn(): use_fn()\n"}, - new_module_paths={"singleton.py": "pkg.singleton"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - scan = str(tmp_path / "test_foo.py") - # Index only knows pkg.test_mod; pkg.placement/conflict have no source so - # ctx_uniq_static's BFS reaches 0 candidates while static_cands = 1. - index = _make_cuf_index(scan, test_src) - result, changed, unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - [ctx_ambig, ctx_uniq_static], - scan_file=scan, - index=index, - ) - assert changed - assert '@patch("pkg.singleton.use_fn")' in result - assert "test_f" not in unresolved # static single-cand cleared the entry - - -def test_callgraph_update_file_const_ref_unanimous(tmp_path): - test_src = ( - "from pkg.placement import helper\n" - '_PATCH_USE = "pkg.orig.use_fn"\n' - "@patch(_PATCH_USE)\n" - "def test_a(mock_use_fn):\n" - " helper()\n" - "\n" - "@patch(_PATCH_USE)\n" - "def test_b(mock_use_fn):\n" - " helper()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - _make_cuf_contexts(), - scan_file=scan, - index=index, - ) - assert changed - assert '_PATCH_USE = "pkg.placement.use_fn"' in result - assert "@patch(_PATCH_USE)" in result - - -def test_callgraph_update_file_const_ref_conflicting(tmp_path): - # test_a: helper() → placement; test_b: resolve() → conflict → conflicting. - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", - "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", - }, - new_module_paths={ - "placement.py": "pkg.placement", - "conflict.py": "pkg.conflict", - }, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = ( - "from pkg.placement import helper\n" - "from pkg.conflict import resolve\n" - '_PATCH_USE = "pkg.orig.use_fn"\n' - "@patch(_PATCH_USE)\n" - "def test_a(mock_use_fn):\n" - " helper()\n" - "\n" - "@patch(_PATCH_USE)\n" - "def test_b(mock_use_fn):\n" - " resolve()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, {"pkg.orig.use_fn"}, [ctx], scan_file=scan, index=index - ) - assert changed - assert '_PATCH_USE = "pkg.orig.use_fn"' in result # const def unchanged - assert '@patch("pkg.placement.use_fn")' in result # test_a inlined - assert '@patch("pkg.conflict.use_fn")' in result # test_b inlined - - -def test_callgraph_update_file_non_forking_path_skipped(tmp_path): - test_src = ( - "from pkg.placement import helper\n" - '@patch("pkg.stable.some_func")\n' - '@patch("pkg.orig.use_fn")\n' - "def test_f(mock_use_fn, mock_some):\n" - " helper()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - _make_cuf_contexts(), - scan_file=scan, - index=index, - ) - assert changed - assert '@patch("pkg.placement.use_fn")' in result - assert '@patch("pkg.stable.some_func")' in result # unchanged - - -def test_callgraph_update_file_multi_context_second_matches(tmp_path): - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - ctx_a = _FLContext( - filepath="/proj/pkg/other.py", - old_module="pkg.other", - original_source="from external import other_fn\n", - modified_source="", - new_files={}, - new_module_paths={}, - entity_to_target={}, - forking_old_paths={"pkg.other.other_fn"}, - ) - ctx_b = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = ( - "from pkg.placement import helper\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(mock_use_fn):\n" - " helper()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn", "pkg.other.other_fn"}, - [ctx_a, ctx_b], - scan_file=scan, - index=index, - ) - assert changed - assert '@patch("pkg.placement.use_fn")' in result - - -def test_callgraph_update_file_const_ref_no_resolution_passthrough(tmp_path): - test_src = ( - '_PATCH_USE = "pkg.orig.use_fn"\n' - "@patch(_PATCH_USE)\n" - "def test_f(mock_use_fn):\n" - " unrelated()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - _make_cuf_contexts(), - scan_file=scan, - index=index, - ) - assert not changed - assert '_PATCH_USE = "pkg.orig.use_fn"' in result - - -def test_callgraph_update_file_const_ref_passthrough_single_proposal_updates_const( - tmp_path, -): - # test_a: BFS fails (calls unrelated()) → passthrough (if not resolved → continue). - # test_b: BFS → placement → single proposal for _PATCH_USE. - # Old: passthrough + single proposal → conflicting → inline test_b. - # New: single proposal (passthrough no longer blocks) → const def updated, no - # inline. - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", - }, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = ( - "from pkg.placement import helper\n" - '_PATCH_USE = "pkg.orig.use_fn"\n' - "@patch(_PATCH_USE)\n" - "def test_a(m):\n" - " unrelated()\n" - "\n" - "@patch(_PATCH_USE)\n" - "def test_b(m):\n" - " helper()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, {"pkg.orig.use_fn"}, [ctx], scan_file=scan, index=index - ) - assert changed - # Const definition updated (single proposal, passthrough no longer blocks). - assert '_PATCH_USE = "pkg.placement.use_fn"' in result - # Decorators stay as const refs — no per-function inlining. - assert "@patch(_PATCH_USE)" in result - assert '@patch("pkg.placement.use_fn")' not in result - - -def test_callgraph_update_file_const_ref_partial_resolution(tmp_path): - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn, other_fn\n", - modified_source="", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n" - }, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn", "pkg.orig.other_fn"}, - ) - test_src = ( - "from pkg.placement import helper\n" - '_PATCH_USE = "pkg.orig.use_fn"\n' - '_PATCH_OTHER = "pkg.orig.other_fn"\n' - "@patch(_PATCH_OTHER)\n" - "@patch(_PATCH_USE)\n" - "def test_f(mock_use, mock_other):\n" - " helper()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn", "pkg.orig.other_fn"}, - [ctx], - scan_file=scan, - index=index, - ) - assert changed - assert '_PATCH_USE = "pkg.placement.use_fn"' in result - assert '_PATCH_OTHER = "pkg.orig.other_fn"' in result - - -def test_callgraph_update_file_inline_no_inline_subs_continue(tmp_path): - # test_a: string literal (no const_refs → inline_subs empty → continue) - # test_b: const ref → placement; test_c: const ref → conflict → conflicting - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", - "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", - }, - new_module_paths={ - "placement.py": "pkg.placement", - "conflict.py": "pkg.conflict", - }, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = ( - "from pkg.placement import helper\n" - "from pkg.conflict import resolve\n" - '@patch("pkg.orig.use_fn")\n' - "def test_a(m):\n" - " helper()\n" - "\n" - '_PATCH_USE = "pkg.orig.use_fn"\n' - "@patch(_PATCH_USE)\n" - "def test_b(m):\n" - " helper()\n" - "\n" - "@patch(_PATCH_USE)\n" - "def test_c(m):\n" - " resolve()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, {"pkg.orig.use_fn"}, [ctx], scan_file=scan, index=index - ) - assert changed - assert '@patch("pkg.placement.use_fn")' in result # test_a updated - assert '@patch("pkg.conflict.use_fn")' in result # test_c inlined - - -def test_callgraph_update_file_inline_ref_from_different_file(tmp_path): - # Const ref from constants.py (≠ scan_file) → inline_subs empty → no change. - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", - "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", - }, - new_module_paths={ - "placement.py": "pkg.placement", - "conflict.py": "pkg.conflict", - }, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - constants_file = tmp_path / "constants.py" - constants_file.write_text('_PATCH_USE = "pkg.orig.use_fn"\n', encoding="utf-8") - test_src = ( - "from constants import _PATCH_USE\n" - "from pkg.placement import helper\n" - "from pkg.conflict import resolve\n" - "@patch(_PATCH_USE)\n" - "def test_b(m):\n" - " helper()\n" - "\n" - "@patch(_PATCH_USE)\n" - "def test_c(m):\n" - " resolve()\n" - ) - scan_file = tmp_path / "test_cases.py" - scan_file.write_text(test_src, encoding="utf-8") - scan = str(scan_file) - # Build index from disk so file_to_module is populated for test_cases.py - index = _cg_build_index(str(tmp_path), {}, [ctx]) - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - [ctx], - scan_file=scan, - repo_root=str(tmp_path), - index=index, - ) - assert not changed - - -def test_callgraph_update_file_inline_new_val_same_as_old(tmp_path): - # placement.py → "pkg.orig" (same as old_module); test_b→helper→same val; skipped. - # test_c → resolve → "pkg.conflict" → different val → inlined → changed. - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", - "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", - }, - new_module_paths={ - "placement.py": "pkg.orig", # same as old_module → new_val == old_val - "conflict.py": "pkg.conflict", - }, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = ( - "from pkg.orig import helper\n" - "from pkg.conflict import resolve\n" - '_PATCH_USE = "pkg.orig.use_fn"\n' - "@patch(_PATCH_USE)\n" - "def test_b(m):\n" - " helper()\n" - "\n" - "@patch(_PATCH_USE)\n" - "def test_c(m):\n" - " resolve()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, {"pkg.orig.use_fn"}, [ctx], scan_file=scan, index=index - ) - assert changed # test_c inlined to pkg.conflict.use_fn - - -def test_callgraph_update_file_inline_existing_splice_updated(tmp_path): - # test_a: use_fn → func_splice; other_fn const ref conflicting → inline. - # Inline finds existing splice and updates it. test_b: const → new splice. - placement_src = ( - "from external import use_fn, other_fn\n" "def helper(): use_fn(); other_fn()\n" - ) - conflict2_src = "from external import other_fn\ndef resolve2(): other_fn()\n" - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn, other_fn\n", - modified_source="", - new_files={"placement.py": placement_src, "conflict2.py": conflict2_src}, - new_module_paths={ - "placement.py": "pkg.placement", - "conflict2.py": "pkg.conflict2", - }, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn", "pkg.orig.other_fn"}, - ) - test_src = ( - "from pkg.placement import helper\n" - "from pkg.conflict2 import resolve2\n" - '_PATCH_OTHER = "pkg.orig.other_fn"\n' - '@patch("pkg.orig.use_fn")\n' - "@patch(_PATCH_OTHER)\n" - "def test_a(m_other, m_use):\n" - " helper()\n" - "\n" - "@patch(_PATCH_OTHER)\n" - "def test_b(m_other):\n" - " resolve2()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn", "pkg.orig.other_fn"}, - [ctx], - scan_file=scan, - index=index, - ) - assert changed - assert '@patch("pkg.placement.use_fn")' in result - assert '@patch("pkg.placement.other_fn")' in result - assert '@patch("pkg.conflict2.other_fn")' in result - - -def test_callgraph_update_file_verbose(tmp_path, capsys): - test_src = ( - "from pkg.placement import helper\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(mock_use_fn):\n" - " helper()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - _make_cuf_contexts(), - scan_file=scan, - index=index, - verbose=True, - ) - captured = capsys.readouterr() - assert "patch_callgraph" in captured.err - - -def test_callgraph_update_file_truncated_warns(tmp_path, capsys): - # Depth limit of 0 forces truncation for indirect calls; warning must be printed. - # Test calls an intermediate function (not a terminal); with max_depth=0 the - # first BFS hop immediately hits the limit before reaching the terminal. - test_src = ( - "from pkg.middle import middle_fn\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(mock_use_fn):\n" - " middle_fn()\n" - ) - scan = str(tmp_path / "test_foo.py") - scan_abs = str((tmp_path / "test_foo.py").resolve()) - # middle_fn → helper (terminal in pkg.placement), but BFS cuts off before that. - middle_src = "from pkg.placement import helper\ndef middle_fn(): helper()\n" - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src, "pkg.middle": middle_src}, - module_to_package={"pkg.test_mod": "pkg", "pkg.middle": "pkg"}, - module_to_defs={ - "pkg.test_mod": set(), - "pkg.middle": _cg_collect_defined_names(middle_src), - }, - file_to_module={scan_abs: "pkg.test_mod"}, - ) - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - _make_cuf_contexts(), - scan_file=scan, - index=index, - max_depth=0, - ) - assert not changed - captured = capsys.readouterr() - assert "traversal limit reached" in captured.err - assert "pkg.orig.use_fn" in captured.err - - -# --------------------------------------------------------------------------- -# _resolve_forking_path_candidates — full result (truncation / candidates) -# --------------------------------------------------------------------------- - - -def test_resolve_forking_path_candidates_single(): - # Single candidate: path returned, candidates=[path], truncated=False. - ctx = _make_bfs_ctx() - index = _make_bfs_index("from pkg.placement import helper\n") - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" - ) - assert path == "pkg.placement.use_fn" - assert cands == ["pkg.placement.use_fn"] - assert not truncated - - -def test_resolve_forking_path_candidates_multiple(): - # Multiple candidates → path=None, cands=[...], truncated=False. - ctx = _make_bfs_ctx() - test_src = "from pkg.placement import helper\nfrom pkg.conflict import resolve\n" - index = _make_bfs_index(test_src) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", - "def test_f(): helper(); resolve()\n", - ctx, - index, - "pkg.test_mod", - ) - assert path is None - assert sorted(cands) == ["pkg.conflict.use_fn", "pkg.placement.use_fn"] - assert not truncated - - -def test_resolve_forking_path_candidates_no_calling_module(): - ctx = _make_bfs_ctx() - index = _make_bfs_index("from pkg.placement import helper\n") - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", "def test_f(): helper()\n", ctx, index, "" - ) - assert path is None - assert cands == [] - assert not truncated - - -def test_resolve_forking_path_candidates_truncated_depth(): - # Chain of exactly _CG_MAX_DEPTH + 1 hops; the last hop is cut off → truncated=True. - # Chain: test_mod -[f0]-> mid0 -> mid1 -> ... -> mid{n-1} -[helper]-> placement - # n = _CG_MAX_DEPTH + 1 intermediate modules; helper is at depth n-1 = 13, - # but the depth limit cuts off at depth 12 before enqueuing helper. - n = _CG_MAX_DEPTH + 1 # 13 hops from test_mod to placement - ctx = _make_bfs_ctx() - all_src: dict = {} - all_src["pkg.test_mod"] = "from pkg.mid0 import f0\n" - for i in range(n): - caller = f"f{i}" - if i < n - 1: - callee = f"f{i + 1}" - callee_mod = f"pkg.mid{i + 1}" - else: - callee = "helper" - callee_mod = "pkg.placement" - all_src[f"pkg.mid{i}"] = ( - f"from {callee_mod} import {callee}\n" f"def {caller}(): {callee}()\n" - ) - all_src["pkg.placement"] = "from external import use_fn\ndef helper(): use_fn()\n" - index = _CgIndex( - module_to_source=all_src, - module_to_package={m: "pkg" for m in all_src}, - module_to_defs={m: _cg_collect_defined_names(s) for m, s in all_src.items()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", "def test_f(): f0()\n", ctx, index, "pkg.test_mod" - ) - assert path is None - assert truncated - - -def test_resolve_forking_path_candidates_truncated_modules(): - # Re-export chain of _CG_MAX_MODULES + 1 intermediate modules; the last one - # is cut off before pkg.placement (a terminal) is ever reached. - n = _CG_MAX_MODULES + 1 - ctx = _make_bfs_ctx() - src_map: dict = {} - for i in range(n): - next_mod = f"pkg.m{i + 1}" if i < n - 1 else "pkg.placement" - src_map[f"pkg.m{i}"] = f"from {next_mod} import helper\n" - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - src_map["pkg.placement"] = placement_src - test_src = "from pkg.m0 import helper\n" - all_src = {"pkg.test_mod": test_src, **src_map} - index = _CgIndex( - module_to_source=all_src, - module_to_package={m: "pkg" for m in all_src}, - module_to_defs={m: _cg_collect_defined_names(s) for m, s in all_src.items()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", "def test_f(): helper()\n", ctx, index, "pkg.test_mod" - ) - assert path is None - assert truncated - - -def test_resolve_forking_path_candidates_original_module_only(): - # modified_source still has a function using use_fn; no new sub-file uses it. - # → only terminal is (pkg.orig, func_a) → unique resolution to original path. - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source=("from external import use_fn\n" "def func_a(): use_fn()\n"), - new_files={ - "placement.py": "from external import other\ndef helper(): other()\n" - }, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - orig_src = ctx.modified_source - test_src = "from pkg.orig import func_a\n" - modules = {"pkg.test_mod": test_src, "pkg.orig": orig_src} - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", "def test_f(): func_a()\n", ctx, index, "pkg.test_mod" - ) - assert path == "pkg.orig.use_fn" - assert cands == ["pkg.orig.use_fn"] - assert not truncated - - -def test_resolve_forking_path_candidates_original_and_new_both_candidates(): - # modified_source keeps func_a (uses use_fn); placement.py moves func_b - # (also uses use_fn). Test calls both → 2 candidates → ambiguous. - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source=("from external import use_fn\n" "def func_a(): use_fn()\n"), - new_files={ - "placement.py": ( - "from external import use_fn\n" "def func_b(): use_fn()\n" - ), - }, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - orig_src = ctx.modified_source - placement_src = ctx.new_files["placement.py"] - test_src = "from pkg.orig import func_a\nfrom pkg.placement import func_b\n" - modules = { - "pkg.test_mod": test_src, - "pkg.orig": orig_src, - "pkg.placement": placement_src, - } - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", - "def test_f(): func_a(); func_b()\n", - ctx, - index, - "pkg.test_mod", - ) - assert path is None # ambiguous - assert sorted(cands) == ["pkg.orig.use_fn", "pkg.placement.use_fn"] - assert not truncated - - -# --------------------------------------------------------------------------- -# _expand_module_terminals -# --------------------------------------------------------------------------- - - -def test_expand_module_terminals_no_direct(): - # No direct terminal in this module → nothing added. - terminal: dict = {} - _expand_module_terminals( - "def a(): b()\ndef b(): pass\n", "pkg.mod", "use_fn", terminal - ) - assert terminal == {} - - -def test_expand_module_terminals_direct_only(): - # A direct terminal is seeded before calling; only transitive callers are added. - terminal: dict = {("pkg.mod", "b"): "pkg.mod.use_fn"} - _expand_module_terminals( - "def a(): b()\ndef b(): use_fn()\n", "pkg.mod", "use_fn", terminal - ) - # a calls b (direct terminal) → a becomes transitive terminal. - assert terminal[("pkg.mod", "a")] == "pkg.mod.use_fn" - # Original direct entry unchanged. - assert terminal[("pkg.mod", "b")] == "pkg.mod.use_fn" - - -def test_expand_module_terminals_multi_level(): - # c → b → a (direct); all three end up in terminal. - terminal: dict = {("pkg.mod", "a"): "pkg.mod.use_fn"} - src = "def a(): use_fn()\ndef b(): a()\ndef c(): b()\n" - _expand_module_terminals(src, "pkg.mod", "use_fn", terminal) - assert ("pkg.mod", "b") in terminal - assert ("pkg.mod", "c") in terminal - - -def test_expand_module_terminals_syntax_error(): - # Unparseable source → silently returns without modifying terminal. - terminal: dict = {("pkg.mod", "a"): "pkg.mod.use_fn"} - _expand_module_terminals("def (broken\n", "pkg.mod", "use_fn", terminal) - # Only the original entry remains. - assert list(terminal.keys()) == [("pkg.mod", "a")] - - -def test_expand_module_terminals_unrelated_module(): - # Direct terminal is in a different module → nothing added for pkg.other. - terminal: dict = {("pkg.mod", "a"): "pkg.mod.use_fn"} - _expand_module_terminals("def b(): a()\n", "pkg.other", "use_fn", terminal) - # b is in pkg.other which has no direct terminals → not added. - assert ("pkg.other", "b") not in terminal - - -# --------------------------------------------------------------------------- -# BFS local-call following (intra-module) -# --------------------------------------------------------------------------- - - -def test_resolve_forking_path_candidates_intra_module_chain(): - # BFS follows locally-defined calls within a non-terminal intermediate module. - # Chain: test_mod → pkg.service.public_func - # (local) ↓ - # pkg.service._local_helper - # (import) ↓ - # pkg.placement.use_target ← terminal (calls use_fn) - # - # pkg.service is neither orig nor a new sub-file, so _expand_module_terminals - # never seeds it. The elif branch in the BFS must queue _local_helper from - # public_func's body so we eventually reach the terminal in pkg.placement. - placement_src = "from external import use_fn\ndef use_target(): use_fn()\n" - service_src = ( - "from pkg.placement import use_target\n" - "def _local_helper(): use_target()\n" - "def public_func(): _local_helper()\n" - ) - ctx2 = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="from .placement import use_target\n", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={"use_target": "placement.py"}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = "from pkg.service import public_func\n" - modules = { - "pkg.test_mod": test_src, - "pkg.service": service_src, - "pkg.placement": placement_src, - } - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", - "def test_f(): public_func()\n", - ctx2, - index, - "pkg.test_mod", - ) - assert path == "pkg.placement.use_fn" - assert cands == ["pkg.placement.use_fn"] - assert not truncated - - -def test_resolve_forking_path_candidates_intra_module_local_already_visited(): - # The elif branch fires but the visited guard suppresses re-queuing. - # A recursive function calls itself: when processing its body calls, itself - # is already in visited → (module, called_name) in visited → branch skipped. - placement_src = "from external import use_fn\ndef use_target(): use_fn()\n" - service_src = ( - "from pkg.placement import use_target\n" - # recursive_func calls use_target (imported) AND itself (local, recursive) - "def recursive_func(n): use_target() if n <= 0 else recursive_func(n-1)\n" - ) - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="from .placement import use_target\n", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={"use_target": "placement.py"}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = "from pkg.service import recursive_func\n" - modules = { - "pkg.test_mod": test_src, - "pkg.service": service_src, - "pkg.placement": placement_src, - } - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", - "def test_f(): recursive_func(5)\n", - ctx, - index, - "pkg.test_mod", - ) - # recursive_func's body calls reach use_target (terminal in pkg.placement). - assert path == "pkg.placement.use_fn" - assert not truncated - - -def test_resolve_forking_path_candidates_new_module_intra_chain(): - # BFS follows intra-module calls within a new submodule to reach a terminal. - # Chain: test_mod → pkg.placement.wrapper (new-module, not terminal) - # (local) ↓ - # pkg.placement._inner ← terminal (calls use_fn directly) - # - # Before the fix, the BFS hit pkg.placement in new_module_set and stopped at - # wrapper without following _inner — no candidate was found. After the fix, - # it follows the local call to _inner and discovers pkg.placement.use_fn. - placement_src = ( - "from external import use_fn\n" - "def _inner(): use_fn()\n" - "def wrapper(): _inner()\n" - ) - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={"wrapper": "placement.py", "_inner": "placement.py"}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = "from pkg.placement import wrapper\n" - modules = { - "pkg.test_mod": test_src, - "pkg.placement": placement_src, - } - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", - "def test_f(): wrapper()\n", - ctx, - index, - "pkg.test_mod", - ) - assert path == "pkg.placement.use_fn" - assert cands == ["pkg.placement.use_fn"] - assert not truncated - - -def test_resolve_forking_path_candidates_new_module_intra_chain_cycle(): - # Intra-module traversal inside a new submodule respects the visited guard: - # a mutually recursive pair (a calls b, b calls a) does not loop. - placement_src = ( - "from external import use_fn\n" - "def _inner(): use_fn()\n" - "def a(): b()\n" - "def b(): a(); _inner()\n" # b is terminal (uses use_fn via _inner) - ) - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={"a": "placement.py"}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = "from pkg.placement import a\n" - modules = { - "pkg.test_mod": test_src, - "pkg.placement": placement_src, - } - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", - "def test_f(): a()\n", - ctx, - index, - "pkg.test_mod", - ) - assert path == "pkg.placement.use_fn" - assert not truncated - - -def test_resolve_forking_path_candidates_new_module_cross_module_terminal(): - # BFS follows cross-module calls FROM a terminal function inside a new submodule. - # Scenario: - # pkg.main: _run_step() calls use_fn() directly (terminal) - # orchestrate() calls _run_step() [local] + do_step() [pkg.steps] - # _expand_module_terminals makes orchestrate terminal for pkg.main.use_fn # noqa: E501 - # pkg.steps: do_step() calls use_fn() (terminal for pkg.steps.use_fn) - # - # When BFS hits (pkg.main, orchestrate) — which IS in terminal — it should - # record pkg.main.use_fn AND then follow the cross-module call to - # (pkg.steps, do_step), discovering pkg.steps.use_fn as a second candidate. - # Line 1472 in the BFS is covered only by this cross-module append. - main_src = ( - "from external import use_fn\n" - "from pkg.steps import do_step\n" - "def _run_step(): use_fn()\n" - "def orchestrate(): _run_step(); do_step()\n" - ) - steps_src = "from external import use_fn\n" "def do_step(): use_fn()\n" - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"main.py": main_src, "steps.py": steps_src}, - new_module_paths={"main.py": "pkg.main", "steps.py": "pkg.steps"}, - entity_to_target={ - "_run_step": "main.py", - "orchestrate": "main.py", - "do_step": "steps.py", - }, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = "from pkg.main import orchestrate\n" - modules = { - "pkg.test_mod": test_src, - "pkg.main": main_src, - "pkg.steps": steps_src, - } - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", - "def test_f(): orchestrate()\n", - ctx, - index, - "pkg.test_mod", - ) - # Both submodules use use_fn — two candidates, no single resolved path. - assert path is None - assert sorted(cands) == ["pkg.main.use_fn", "pkg.steps.use_fn"] - assert not truncated - - -# --------------------------------------------------------------------------- -# BFS — import alias traversal -# --------------------------------------------------------------------------- - - -def test_resolve_forking_path_candidates_import_alias_direct(): - # Test uses ``import pkg.placement as pl; pl.helper()`` to call the terminal. - # The BFS must follow ``pl.helper`` by resolving alias ``pl`` → ``pkg.placement``. - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="from .placement import helper\n", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={"helper": "placement.py"}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = "import pkg.placement as pl\n" - # Import map: "pl" → ("pkg.placement", "pkg.placement"); call "pl.helper()" - # → _cg_collect_called_names emits "pl.helper"; BFS resolves alias pl → - # module pkg.placement, queues (pkg.placement, "helper") → terminal hit. - index = _CgIndex( - module_to_source={"pkg.test_mod": test_src}, - module_to_package={"pkg.test_mod": "pkg"}, - module_to_defs={"pkg.test_mod": set()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", - "def test_f(): pl.helper()\n", - ctx, - index, - "pkg.test_mod", - ) - assert path == "pkg.placement.use_fn" - assert cands == ["pkg.placement.use_fn"] - assert not truncated - - -def test_resolve_forking_path_candidates_body_call_via_alias(): - # An intermediate function uses ``mod.helper()`` (module alias) to reach - # the terminal. The BFS body-call step must follow the alias. - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - service_src = "import pkg.placement as pl\n" "def public_func(): pl.helper()\n" - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="from .placement import helper\n", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={"helper": "placement.py"}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = "from pkg.service import public_func\n" - modules = { - "pkg.test_mod": test_src, - "pkg.service": service_src, - "pkg.placement": placement_src, - } - index = _CgIndex( - module_to_source=modules, - module_to_package={m: "pkg" for m in modules}, - module_to_defs={m: _cg_collect_defined_names(s) for m, s in modules.items()}, - file_to_module={}, - ) - path, cands, truncated, _static = _resolve_forking_path_candidates( - "use_fn", - "def test_f(): public_func()\n", - ctx, - index, - "pkg.test_mod", - ) - assert path == "pkg.placement.use_fn" - assert cands == ["pkg.placement.use_fn"] - assert not truncated - - -# --------------------------------------------------------------------------- -# _candidates_check -# --------------------------------------------------------------------------- - - -def test_candidates_check_no_candidates(): - # No candidates for any path → None. - assert _candidates_check({"pkg.orig.A": "pkg.sub.A"}, ["pkg.orig.A"], {}) is None - - -def test_candidates_check_rename_valid(): - # Rename is in candidates → None. - cands = {"pkg.orig.A": ["pkg.placement.A", "pkg.helpers.A"]} - assert ( - _candidates_check({"pkg.orig.A": "pkg.placement.A"}, ["pkg.orig.A"], cands) - is None - ) - - -def test_candidates_check_rename_invalid(): - # Rename proposes a path not in candidates → error message. - cands = {"pkg.orig.A": ["pkg.placement.A"]} - result = _candidates_check({"pkg.orig.A": "pkg.wrong.A"}, ["pkg.orig.A"], cands) - assert result is not None - assert "pkg.wrong.A" in result - assert "pkg.placement.A" in result - - -def test_candidates_check_no_change_with_candidates(): - # No rename proposed for a path that has candidates → error message. - cands = {"pkg.orig.A": ["pkg.placement.A"]} - result = _candidates_check({}, ["pkg.orig.A"], cands) - assert result is not None - assert "pkg.orig.A" in result - assert "pkg.placement.A" in result - - -def test_candidates_check_path_not_in_candidates(): - # Another path has no candidates → passes; only paths with candidates are checked. - cands = {"pkg.orig.A": ["pkg.placement.A"]} - # pkg.orig.B has no candidates; even though no rename proposed → None - assert _candidates_check({}, ["pkg.orig.B"], cands) is None - - -def test_candidates_check_no_change_when_old_in_candidates(): - # No rename proposed but old path is itself one of the candidates (e.g. the entity - # is still accessible at the original module via __init__.py re-export) → None. - cands = {"pkg.orig.A": ["pkg.orig.A", "pkg.resolver.A"]} - assert _candidates_check({}, ["pkg.orig.A"], cands) is None - - -# --------------------------------------------------------------------------- -# _callgraph_update_file — candidates collected when multiple found -# --------------------------------------------------------------------------- - - -def test_callgraph_update_file_multiple_candidates_saved(tmp_path): - # Both placement and conflict are reachable → 2 candidates → saved. - test_src = ( - "from pkg.placement import helper\n" - "from pkg.conflict import resolve\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(m):\n" - " helper()\n" - " resolve()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - _make_cuf_contexts(), - scan_file=scan, - index=index, - ) - assert not changed # ambiguous → no update - assert "test_f" in unresolved - assert "pkg.orig.use_fn" in unresolved["test_f"] - cands = unresolved["test_f"]["pkg.orig.use_fn"] - assert sorted(cands) == ["pkg.conflict.use_fn", "pkg.placement.use_fn"] - - -def test_callgraph_update_file_resolved_clears_candidates(tmp_path): - # Single ctx with unique resolution → no candidates saved. - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = ( - "from pkg.placement import helper\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(m):\n" - " helper()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - [ctx], - scan_file=scan, - index=index, - ) - assert changed - assert "test_f" not in unresolved # unique resolution → no candidates saved - - -def test_callgraph_update_file_resolved_clears_function_entry(tmp_path): - # ctx_ambig gives 2 candidates (saves to unresolved); ctx_uniq resolves uniquely → - # unresolved entry for the function is deleted (line 2695). - test_src = ( - "from pkg.placement import helper\n" - "from pkg.conflict import resolve\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(m):\n" - " helper()\n" - " resolve()\n" - ) - ctx_ambig = _make_cuf_contexts()[0] # both placement and conflict → 2 candidates - ctx_uniq = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="from .placement import helper\n", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", - }, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - [ctx_ambig, ctx_uniq], - scan_file=scan, - index=index, - ) - assert changed - assert "test_f" not in unresolved # ctx_uniq resolved → entry deleted - - -# --------------------------------------------------------------------------- -# apply_patch_callgraph — candidates_out parameter -# --------------------------------------------------------------------------- - - -def test_apply_patch_callgraph_candidates_out_per_file(tmp_path): - # Multiple candidates → saved in candidates_out for per_file entry. - test_src = ( - "from pkg.placement import helper\n" - "from pkg.conflict import resolve\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(m):\n" - " helper()\n" - " resolve()\n" - ) - test_file = tmp_path / "test_orig.py" - test_file.write_text(test_src, encoding="utf-8") - per_file = {str(test_file): {"source": test_src, "msgs": []}} - candidates_out: dict = {} - list( - apply_patch_callgraph( - _make_cuf_contexts(), per_file, str(tmp_path), candidates_out=candidates_out - ) - ) - abs_fp = str(test_file.resolve()) - assert abs_fp in candidates_out - assert "test_f" in candidates_out[abs_fp] - - -def test_apply_patch_callgraph_candidates_out_disk_file(tmp_path): - # Multiple candidates → saved in candidates_out for disk file. - test_src = ( - "from pkg.placement import helper\n" - "from pkg.conflict import resolve\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(m):\n" - " helper()\n" - " resolve()\n" - ) - test_file = tmp_path / "test_orig.py" - test_file.write_text(test_src, encoding="utf-8") - candidates_out: dict = {} - list( - apply_patch_callgraph( - _make_cuf_contexts(), {}, str(tmp_path), candidates_out=candidates_out - ) - ) - abs_fp = str(test_file.resolve()) - assert abs_fp in candidates_out - assert "test_f" in candidates_out[abs_fp] - - -# --------------------------------------------------------------------------- -# Prompt builders — candidates_per_path parameter -# --------------------------------------------------------------------------- - - -def _make_fl_ctx_simple(): - """Minimal FLContext for prompt builder tests.""" - return _FLContext( - filepath="/repo/pkg/big.py", - old_module="pkg.big", - original_source="from external import A\ndef f(): A()\n", - modified_source="from .sub_a import f\n", - new_files={"sub_a.py": "from external import A\ndef f(): A()\n"}, - new_module_paths={"sub_a.py": "pkg.sub_a"}, - entity_to_target={"f": "sub_a.py"}, - forking_old_paths={"pkg.big.A"}, - ) - - -def test_build_classify_prompt_with_candidates(): - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - prompt = _build_classify_prompt( - context_msg, - "def test_f(): pass\n", - ["pkg.big.A"], - candidates_per_path={"pkg.big.A": ["pkg.sub_a.A", "pkg.sub_b.A"]}, - ) - assert "Call-graph candidate paths" in prompt - assert "pkg.sub_a.A" in prompt - assert "pkg.sub_b.A" in prompt - - -def test_build_classify_prompt_candidates_above_threshold(): - # Candidates count > threshold → section not included. - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - many_cands = [f"pkg.sub_{i}.A" for i in range(_CG_CANDIDATES_LLM_THRESHOLD + 1)] - prompt = _build_classify_prompt( - context_msg, - "def test_f(): pass\n", - ["pkg.big.A"], - candidates_per_path={"pkg.big.A": many_cands}, - ) - assert "Call-graph candidate paths" not in prompt - - -def test_build_func_verify_prompt_with_candidates(): - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - prompt = _build_func_verify_prompt( - context_msg, - "def test_f(): pass\n", - {"pkg.big.A": "pkg.sub_a.A"}, - candidates_per_path={"pkg.big.A": ["pkg.sub_a.A", "pkg.sub_b.A"]}, - ) - assert "Call-graph candidate paths" in prompt - assert "pkg.sub_a.A" in prompt - - -def test_build_func_verify_prompt_candidates_above_threshold(): - # All candidate lists exceed the threshold → section not included. - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - many_cands = [f"pkg.sub_{i}.A" for i in range(_CG_CANDIDATES_LLM_THRESHOLD + 1)] - prompt = _build_func_verify_prompt( - context_msg, - "def test_f(): pass\n", - {"pkg.big.A": "pkg.sub_a.A"}, - candidates_per_path={"pkg.big.A": many_cands}, - ) - assert "Call-graph candidate paths" not in prompt - - -def test_build_no_change_verify_prompt_with_candidates(): - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - prompt = _build_no_change_verify_prompt( - context_msg, - "def test_f(): pass\n", - ["pkg.big.A"], - candidates_per_path={"pkg.big.A": ["pkg.sub_a.A"]}, - ) - assert "Call-graph candidate paths" in prompt - assert "pkg.sub_a.A" in prompt - - -def test_build_no_change_verify_prompt_candidates_above_threshold(): - # All candidate lists exceed the threshold → section not included. - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - many_cands = [f"pkg.sub_{i}.A" for i in range(_CG_CANDIDATES_LLM_THRESHOLD + 1)] - prompt = _build_no_change_verify_prompt( - context_msg, - "def test_f(): pass\n", - ["pkg.big.A"], - candidates_per_path={"pkg.big.A": many_cands}, - ) - assert "Call-graph candidate paths" not in prompt - - -def test_build_rewrite_func_prompt_with_candidates(): - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - prompt = _build_rewrite_func_prompt( - context_msg, - "def test_f(): pass\n", - ["pkg.big.A"], - candidates_per_path={"pkg.big.A": ["pkg.sub_a.A", "pkg.helpers.A"]}, - ) - assert "Call-graph candidate paths" in prompt - assert "pkg.sub_a.A" in prompt - assert "pkg.helpers.A" in prompt - - -def test_build_rewrite_func_prompt_candidates_above_threshold(): - # All candidate lists exceed the threshold → section not included. - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - many_cands = [f"pkg.sub_{i}.A" for i in range(_CG_CANDIDATES_LLM_THRESHOLD + 1)] - prompt = _build_rewrite_func_prompt( - context_msg, - "def test_f(): pass\n", - ["pkg.big.A"], - candidates_per_path={"pkg.big.A": many_cands}, - ) - assert "Call-graph candidate paths" not in prompt - - -# --------------------------------------------------------------------------- -# _patch_strings_in_text -# --------------------------------------------------------------------------- - - -def test_patch_strings_in_text_decorator(): - text = '@patch("pkg.mod.A")\ndef test_f(m): pass\n' - assert _patch_strings_in_text(text) == {"pkg.mod.A"} - - -def test_patch_strings_in_text_attribute_decorator(): - text = '@mock.patch("pkg.mod.B")\ndef test_f(m): pass\n' - assert _patch_strings_in_text(text) == {"pkg.mod.B"} - - -def test_patch_strings_in_text_context_manager(): - text = 'def test_f():\n with patch("pkg.mod.C") as m: pass\n' - assert _patch_strings_in_text(text) == {"pkg.mod.C"} - - -def test_patch_strings_in_text_multiple(): - text = ( - '@patch("pkg.mod.A")\n' '@mock.patch("pkg.mod.B")\n' "def test_f(a, b): pass\n" - ) - assert _patch_strings_in_text(text) == {"pkg.mod.A", "pkg.mod.B"} - - -def test_patch_strings_in_text_empty(): - assert _patch_strings_in_text("def test_f(): pass\n") == set() - - -# --------------------------------------------------------------------------- -# _rewrite_candidates_check -# --------------------------------------------------------------------------- - - -def test_rewrite_candidates_check_no_candidates(): - # No candidates for any path → None. - text = '@patch("pkg.mod.A")\ndef test_f(m): pass\n' - assert _rewrite_candidates_check(["pkg.mod.A"], text, {}) is None - - -def test_rewrite_candidates_check_valid_rename(): - # Old path absent, one candidate present → None. - text = '@patch("pkg.placement.A")\ndef test_f(m): pass\n' - cands = {"pkg.mod.A": ["pkg.placement.A", "pkg.other.A"]} - assert _rewrite_candidates_check(["pkg.mod.A"], text, cands) is None - - -def test_rewrite_candidates_check_old_still_present(): - # Old path still present even though candidates exist → error. - text = '@patch("pkg.mod.A")\ndef test_f(m): pass\n' - cands = {"pkg.mod.A": ["pkg.placement.A"]} - result = _rewrite_candidates_check(["pkg.mod.A"], text, cands) - assert result is not None - assert "pkg.mod.A" in result - assert "pkg.placement.A" in result - - -def test_rewrite_candidates_check_renamed_to_unknown(): - # Old path absent, no known candidate appears — could be a wrong rename or a - # dead-code removal. Let the LLM verify step decide; no error returned here. - text = '@patch("pkg.wrong.A")\ndef test_f(m): pass\n' - cands = {"pkg.mod.A": ["pkg.placement.A", "pkg.other.A"]} - assert _rewrite_candidates_check(["pkg.mod.A"], text, cands) is None - - -def test_rewrite_candidates_check_deleted_patch(): - # Old path absent and decorator was removed entirely → dead-code removal is - # allowed; let the LLM verify step confirm correctness. - text = "def test_f(): pass\n" - cands = {"pkg.mod.A": ["pkg.placement.A", "pkg.other.A"]} - assert _rewrite_candidates_check(["pkg.mod.A"], text, cands) is None - - -def test_rewrite_candidates_check_path_without_candidates_ignored(): - # A path with no candidates in the dict → skip it. - text = '@patch("pkg.mod.B")\ndef test_f(m): pass\n' - cands = {"pkg.mod.A": ["pkg.placement.A"]} # A has candidates, B does not - assert _rewrite_candidates_check(["pkg.mod.B"], text, cands) is None - - -# --------------------------------------------------------------------------- -# _build_rewrite_verify_prompt — candidates_per_path -# --------------------------------------------------------------------------- - - -def test_build_rewrite_verify_prompt_with_candidates(): - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - prompt = _build_rewrite_verify_prompt( - context_msg, - "def test_f(): pass\n", - "def test_f(): pass\n", - candidates_per_path={"pkg.big.A": ["pkg.sub_a.A", "pkg.sub_b.A"]}, - ) - assert "Call-graph candidate paths" in prompt - assert "pkg.sub_a.A" in prompt - - -def test_build_rewrite_verify_prompt_candidates_above_threshold(): - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - many_cands = [f"pkg.sub_{i}.A" for i in range(_CG_CANDIDATES_LLM_THRESHOLD + 1)] - prompt = _build_rewrite_verify_prompt( - context_msg, - "def test_f(): pass\n", - "def test_f(): pass\n", - candidates_per_path={"pkg.big.A": many_cands}, - ) - assert "Call-graph candidate paths" not in prompt - - -def test_build_rewrite_verify_prompt_no_candidates(): - ctx = _make_fl_ctx_simple() - context_msg = _build_context_message([ctx]) - prompt = _build_rewrite_verify_prompt( - context_msg, - "def test_f(): pass\n", - "def test_f(): pass\n", - ) - assert "Call-graph candidate paths" not in prompt - assert "Verify that the rewrite is correct" in prompt - - -# --------------------------------------------------------------------------- -# _process_file_source — candidates pre-check -# --------------------------------------------------------------------------- - - -_PATCH_MAKE_CLIENT = "crispen.patch_rewriter.make_client" -_PATCH_GET_KEY_PR = "crispen.patch_rewriter.get_api_key" -_PATCH_CALL_PR = "crispen.patch_rewriter.call_with_tool" - - -def _make_process_cfg(): - return CrispenConfig(patch_update_retries=1, llm_verify_retries=0) - - -@mock_patch(_PATCH_CALL_PR) -@mock_patch(_PATCH_MAKE_CLIENT) -@mock_patch(_PATCH_GET_KEY_PR, return_value="key") -def test_process_file_source_candidates_reject_no_change( - mock_key, mock_client, mock_call -): - # LLM proposes no change but candidates exist → reject and retry. - # First classify: no rename → rejected by candidates check. - # Second classify: correct rename in candidates → verify → accepted. - src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' - ctx = _FLContext( - filepath="/repo/pkg/big.py", - old_module="pkg.big", - original_source="from external import A\ndef f(): A()\n", - modified_source="from .sub_a import f\n", - new_files={"sub_a.py": "from external import A\ndef f(): A()\n"}, - new_module_paths={"sub_a.py": "pkg.sub_a"}, - entity_to_target={"f": "sub_a.py"}, - forking_old_paths={"pkg.big.A"}, - ) - context_msg = _build_context_message([ctx]) - mock_call.side_effect = [ - # First classify: no rename (LLM says no change needed) - LLMCallResult( - tool_input={"needs_rewrite": False, "patch_renames": {}}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # Second classify (after candidates rejection): correct rename - LLMCallResult( - tool_input={ - "needs_rewrite": False, - "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}, - }, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # Verify rename - LLMCallResult( - tool_input={"correct": True, "corrections": {}, "issue": ""}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - ] - cg_candidates = {"test_f": {"pkg.big.A": ["pkg.sub_a.A"]}} - new_src, changed, _ = _process_file_source( - src, - {"pkg.big.A"}, - context_msg, - mock_client.return_value, - _make_process_cfg(), - max_attempts=2, - cg_candidates=cg_candidates, - ) - assert changed - assert "pkg.sub_a.A" in new_src - # Two classify calls + one verify call = 3 - assert mock_call.call_count == 3 - - -@mock_patch(_PATCH_CALL_PR) -@mock_patch(_PATCH_MAKE_CLIENT) -@mock_patch(_PATCH_GET_KEY_PR, return_value="key") -def test_process_file_source_candidates_reject_verbose( - mock_key, mock_client, mock_call, capsys -): - # verbose=True prints 'candidates check rejected' when cand_issue fires. - src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' - ctx = _FLContext( - filepath="/repo/pkg/big.py", - old_module="pkg.big", - original_source="from external import A\ndef f(): A()\n", - modified_source="from .sub_a import f\n", - new_files={"sub_a.py": "from external import A\ndef f(): A()\n"}, - new_module_paths={"sub_a.py": "pkg.sub_a"}, - entity_to_target={"f": "sub_a.py"}, - forking_old_paths={"pkg.big.A"}, - ) - context_msg = _build_context_message([ctx]) - mock_call.side_effect = [ - # First classify: no rename → rejected by candidates check. - LLMCallResult( - tool_input={"needs_rewrite": False, "patch_renames": {}}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # Second classify: correct rename - LLMCallResult( - tool_input={ - "needs_rewrite": False, - "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}, - }, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # Verify rename - LLMCallResult( - tool_input={"correct": True, "corrections": {}, "issue": ""}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - ] - cg_candidates = {"test_f": {"pkg.big.A": ["pkg.sub_a.A"]}} - _process_file_source( - src, - {"pkg.big.A"}, - context_msg, - mock_client.return_value, - _make_process_cfg(), - max_attempts=2, - cg_candidates=cg_candidates, - verbose=True, - ) - err = capsys.readouterr().err - assert "candidates check rejected" in err - - -@mock_patch(_PATCH_CALL_PR) -@mock_patch(_PATCH_MAKE_CLIENT) -@mock_patch(_PATCH_GET_KEY_PR, return_value="key") -def test_process_file_source_candidates_reject_bad_rename( - mock_key, mock_client, mock_call -): - # LLM proposes a rename not in candidates → rejected → retry with correct answer. - src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' - ctx = _FLContext( - filepath="/repo/pkg/big.py", - old_module="pkg.big", - original_source="from external import A\ndef f(): A()\n", - modified_source="from .sub_a import f\n", - new_files={ - "sub_a.py": "from external import A\ndef f(): A()\n", - "sub_b.py": "from external import A\ndef g(): A()\n", - }, - new_module_paths={"sub_a.py": "pkg.sub_a", "sub_b.py": "pkg.sub_b"}, - entity_to_target={"f": "sub_a.py"}, - forking_old_paths={"pkg.big.A"}, - ) - context_msg = _build_context_message([ctx]) - mock_call.side_effect = [ - # First classify: wrong rename (not in candidates) - LLMCallResult( - tool_input={ - "needs_rewrite": False, - "patch_renames": {"pkg.big.A": "pkg.sub_b.A"}, - }, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # Second classify: correct rename - LLMCallResult( - tool_input={ - "needs_rewrite": False, - "patch_renames": {"pkg.big.A": "pkg.sub_a.A"}, - }, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # Verify - LLMCallResult( - tool_input={"correct": True, "corrections": {}, "issue": ""}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - ] - cg_candidates = {"test_f": {"pkg.big.A": ["pkg.sub_a.A"]}} - new_src, changed, _ = _process_file_source( - src, - {"pkg.big.A"}, - context_msg, - mock_client.return_value, - _make_process_cfg(), - max_attempts=2, - cg_candidates=cg_candidates, - ) - assert changed - assert "pkg.sub_a.A" in new_src - assert mock_call.call_count == 3 - - -@mock_patch(_PATCH_CALL_PR) -@mock_patch(_PATCH_MAKE_CLIENT) -@mock_patch(_PATCH_GET_KEY_PR, return_value="key") -def test_process_file_source_rewrite_candidates_reject_and_retry( - mock_key, mock_client, mock_call -): - # Rewrite returns old path still present → rewrite candidates check rejects - # without calling verify → retry; second rewrite uses valid candidate → accepted. - src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n complex_logic()\n' - ctx = _FLContext( - filepath="/repo/pkg/big.py", - old_module="pkg.big", - original_source="from external import A\ndef f(): A()\n", - modified_source="from .sub_a import f\n", - new_files={"sub_a.py": "from external import A\ndef f(): A()\n"}, - new_module_paths={"sub_a.py": "pkg.sub_a"}, - entity_to_target={"f": "sub_a.py"}, - forking_old_paths={"pkg.big.A"}, - ) - context_msg = _build_context_message([ctx]) - bad_rewrite = '@patch("pkg.big.A")\ndef test_f(mock_a):\n complex_logic()\n' - good_rewrite = '@patch("pkg.sub_a.A")\ndef test_f(mock_a):\n complex_logic()\n' - mock_call.side_effect = [ - # classify → needs rewrite - LLMCallResult( - tool_input={"needs_rewrite": True}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # rewrite 1: old path still present → rejected by _rewrite_candidates_check - LLMCallResult( - tool_input={"rewritten_function": bad_rewrite}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # rewrite 2: valid candidate - LLMCallResult( - tool_input={"rewritten_function": good_rewrite}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # verify - LLMCallResult( - tool_input={"correct": True, "issue": ""}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - ] - cg_candidates = {"test_f": {"pkg.big.A": ["pkg.sub_a.A"]}} - new_src, changed, _ = _process_file_source( - src, - {"pkg.big.A"}, - context_msg, - mock_client.return_value, - _make_process_cfg(), - max_attempts=2, - cg_candidates=cg_candidates, - ) - assert changed - assert "pkg.sub_a.A" in new_src - assert mock_call.call_count == 4 # classify + bad_rw + good_rw + verify - - -@mock_patch(_PATCH_CALL_PR) -@mock_patch(_PATCH_MAKE_CLIENT) -@mock_patch(_PATCH_GET_KEY_PR, return_value="key") -def test_process_file_source_candidates_all_retries_escalates_to_rewrite( - mock_key, mock_client, mock_call, capsys -): - # All classify retries exhausted with persistent candidates check rejections → - # escalate to full rewrite rather than silently leaving the test broken. - src = '@patch("pkg.big.A")\ndef test_f(mock_a):\n pass\n' - ctx = _FLContext( - filepath="/repo/pkg/big.py", - old_module="pkg.big", - original_source="from external import A\ndef f(): A()\n", - modified_source="from .sub_a import f\n", - new_files={"sub_a.py": "from external import A\ndef f(): A()\n"}, - new_module_paths={"sub_a.py": "pkg.sub_a"}, - entity_to_target={"f": "sub_a.py"}, - forking_old_paths={"pkg.big.A"}, - ) - context_msg = _build_context_message([ctx]) - good_rewrite = '@patch("pkg.sub_a.A")\ndef test_f(mock_a):\n pass\n' - mock_call.side_effect = [ - # First classify: no rename → rejected by candidates check (not last attempt). - LLMCallResult( - tool_input={"needs_rewrite": False, "patch_renames": {}}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # Second classify: still no rename → last attempt → escalate to rewrite. - LLMCallResult( - tool_input={"needs_rewrite": False, "patch_renames": {}}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # Rewrite (escalated from candidates check failure): - LLMCallResult( - tool_input={"rewritten_function": good_rewrite}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - # Rewrite verify: - LLMCallResult( - tool_input={"correct": True, "issue": ""}, - elapsed=0.1, - input_tokens=10, - output_tokens=5, - ), - ] - # Two candidates → ambiguous → LLM keeps returning no_change. - cg_candidates = {"test_f": {"pkg.big.A": ["pkg.sub_a.A", "pkg.sub_b.A"]}} - new_src, changed, _ = _process_file_source( - src, - {"pkg.big.A"}, - context_msg, - mock_client.return_value, - _make_process_cfg(), - max_attempts=2, - cg_candidates=cg_candidates, - verbose=True, - ) - assert changed - assert "pkg.sub_a.A" in new_src - assert mock_call.call_count == 4 # classify x2 + rewrite + verify - err = capsys.readouterr().err - assert "candidates check retries exhausted" in err - - -# --------------------------------------------------------------------------- -# apply_patch_callgraph -# --------------------------------------------------------------------------- - - -def test_apply_patch_callgraph_empty_contexts(): - result = list(apply_patch_callgraph([], {}, "/repo")) - assert result == [] - - -def test_apply_patch_callgraph_no_forking_paths(): - ctx = _make_fl_ctx(forking_old_paths=set()) - result = list(apply_patch_callgraph([ctx], {}, "/repo")) - assert result == [] - - -def test_apply_patch_callgraph_per_file_update(tmp_path): - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - conflict_src = "from external import use_fn\ndef resolve(): use_fn()\n" - ctx = _FLContext( - filepath=str(tmp_path / "pkg" / "orig.py"), - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src, "conflict.py": conflict_src}, - new_module_paths={ - "placement.py": "pkg.placement", - "conflict.py": "pkg.conflict", - }, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - file_src = ( - "from pkg.placement import helper\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(mock_use_fn):\n" - " helper()\n" - ) - test_file = tmp_path / "test_orig.py" - test_file.write_text(file_src, encoding="utf-8") - per_file = {str(test_file): {"source": file_src, "msgs": []}} - list(apply_patch_callgraph([ctx], per_file, str(tmp_path))) - assert '@patch("pkg.placement.use_fn")' in per_file[str(test_file)]["source"] - - -def test_apply_patch_callgraph_repo_scan(tmp_path): - test_file = tmp_path / "test_something.py" - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - test_file.write_text( - "from pkg.placement import helper\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(mock_use_fn):\n" - " helper()\n", - encoding="utf-8", - ) - ctx = _FLContext( - filepath=str(tmp_path / "pkg" / "orig.py"), - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - msgs = list(apply_patch_callgraph([ctx], {}, str(tmp_path))) - updated = test_file.read_text(encoding="utf-8") - assert '@patch("pkg.placement.use_fn")' in updated - assert any("call-graph" in m for m in msgs) - - -def test_apply_patch_callgraph_repo_scan_no_change(tmp_path): - test_file = tmp_path / "test_something.py" - test_file.write_text("def test_f(): pass\n", encoding="utf-8") - ctx = _FLContext( - filepath=str(tmp_path / "pkg" / "orig.py"), - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": "from external import use_fn\ndef f(): use_fn()\n"}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - msgs = list(apply_patch_callgraph([ctx], {}, str(tmp_path))) - assert msgs == [] - - -def test_apply_patch_callgraph_repo_root_none(): - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={}, - new_module_paths={}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - result = list(apply_patch_callgraph([ctx], {}, None)) - assert result == [] - - -def test_apply_patch_callgraph_per_file_no_change(tmp_path): - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - ctx = _FLContext( - filepath=str(tmp_path / "pkg" / "orig.py"), - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - file_src = "# pkg.orig.use_fn mentioned here but no test functions.\nx = 1\n" - key = str(tmp_path / "module.py") - per_file = {key: {"source": file_src, "msgs": []}} - list(apply_patch_callgraph([ctx], per_file, None)) - assert per_file[key]["source"] == file_src - - -def test_apply_patch_callgraph_per_file_no_match(tmp_path): - """per_file entry whose source contains no forking path string → continue.""" - ctx = _FLContext( - filepath=str(tmp_path / "pkg" / "orig.py"), - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n" - }, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - file_src = "x = 1\n" # no mention of forking path - key = str(tmp_path / "module.py") - per_file = {key: {"source": file_src, "msgs": []}} - list(apply_patch_callgraph([ctx], per_file, None)) - assert per_file[key]["source"] == file_src - - -def test_apply_patch_callgraph_repo_scan_oserror(tmp_path): - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - ctx = _FLContext( - filepath=str(tmp_path / "orig.py"), - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - bad_file = tmp_path / "test_bad.py" - bad_file.write_text( - '@patch("pkg.orig.use_fn")\ndef test_f(): helper()\n', encoding="utf-8" - ) - bad_file.chmod(0o000) - try: - msgs = list(apply_patch_callgraph([ctx], {}, str(tmp_path))) - assert msgs == [] - finally: - bad_file.chmod(0o644) - - -def test_apply_patch_callgraph_repo_scan_file_no_change(tmp_path): - test_file = tmp_path / "helper.py" - test_file.write_text( - "# references pkg.orig.use_fn in a comment\nx = 1\n", - encoding="utf-8", - ) - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - ctx = _FLContext( - filepath=str(tmp_path / "orig.py"), - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - msgs = list(apply_patch_callgraph([ctx], {}, str(tmp_path))) - assert msgs == [] - assert "x = 1" in test_file.read_text(encoding="utf-8") - - -def test_apply_patch_callgraph_excluded_dirs(tmp_path): - venv_dir = tmp_path / ".venv" - venv_dir.mkdir() - excluded_file = venv_dir / "test_something.py" - excluded_file.write_text( - '@patch("pkg.orig.use_fn")\ndef test_f(): helper()\n', encoding="utf-8" - ) - placement_src = "from external import use_fn\ndef helper(): use_fn()\n" - ctx = _FLContext( - filepath=str(tmp_path / "orig.py"), - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={"placement.py": placement_src}, - new_module_paths={"placement.py": "pkg.placement"}, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - msgs = list(apply_patch_callgraph([ctx], {}, str(tmp_path))) - assert '@patch("pkg.orig.use_fn")' in excluded_file.read_text(encoding="utf-8") - assert msgs == [] - - -# --------------------------------------------------------------------------- -# _get_const_votes_from_rewrite -# --------------------------------------------------------------------------- - - -def test_get_const_votes_empty_refs(): - """No const_refs → empty dict, no parsing needed.""" - assert _get_const_votes_from_rewrite("def test_f(): pass\n", []) == {} - - -def test_get_const_votes_syntax_error(): - """Unparseable func_text → empty dict (SyntaxError branch).""" - refs = [_make_ref("TARGET", "pkg.old.X")] - assert _get_const_votes_from_rewrite("def f(:\n", refs) == {} - - -def test_get_const_votes_no_function_in_body(): - """Valid Python but no FunctionDef/AsyncFunctionDef → empty dict.""" - refs = [_make_ref("TARGET", "pkg.old.X")] - result = _get_const_votes_from_rewrite("x = 1\n", refs) - assert result == {} - - -def test_get_const_votes_non_call_decorator_skipped(): - """A bare-name decorator (not a Call node) is skipped without error.""" - code = "@pytest.mark.slow\n@patch(TARGET)\ndef test_f(m): pass\n" - refs = [_make_ref("TARGET", "pkg.old.X")] - # TARGET still present as Name → no vote entry (const unchanged). - result = _get_const_votes_from_rewrite(code, refs) - assert result == {} - - -def test_get_const_votes_non_patch_call_skipped(): - """A Call decorator whose func is not 'patch' is skipped.""" - code = "@other_decorator('pkg.old.X')\ndef test_f(m): pass\n" - refs = [_make_ref("TARGET", "pkg.old.X")] - result = _get_const_votes_from_rewrite(code, refs) - assert result == {} - - -def test_get_const_votes_no_args_decorator_skipped(): - """@patch() with no args → skipped (no args branch).""" - code = "@patch()\ndef test_f(): pass\n" - refs = [_make_ref("TARGET", "pkg.old.X")] - result = _get_const_votes_from_rewrite(code, refs) - assert result == {} - - -def test_get_const_votes_module_attr_const_name(): - """@patch(module.CONST) style (Attribute node) → const name recorded correctly.""" - # Attribute form used when const is module-aliased after _restore_const_refs. - code = "@patch(module.TARGET)\ndef test_f(m): pass\n" - refs = [_make_ref("module.TARGET", "pkg.old.X")] - result = _get_const_votes_from_rewrite(code, refs) - # const still present as module.TARGET → no vote entry. - assert result == {} - - -def test_get_const_votes_successful_vote(): - """LLM updated the path → new literal collected, vote returned.""" - refs = [_make_ref("TARGET", "pkg.mod.X")] - code = '@patch("pkg.mod.sub.X")\ndef test_f(m): pass\n' - result = _get_const_votes_from_rewrite(code, refs) - assert result == {"pkg.mod.X": "pkg.mod.sub.X"} - - -def test_get_const_votes_deeply_nested_attr_skipped(): - """@patch(module.sub.CONST) where arg0 is Attribute(Attribute) — falls through - all elif branches (663->647 coverage: the third elif is False for this form).""" - # module.sub.CONST: arg0.value is Attribute, not Name → elif at 661 is False; - # arg0 is not Constant → elif at 663 is False → no match, loop continues. - code = "@patch(module.sub.CONST)\ndef test_f(m): pass\n" - refs = [_make_ref("TARGET", "pkg.mod.X")] - result = _get_const_votes_from_rewrite(code, refs) - # No string literal collected, TARGET still absent → no vote. - assert result == {} - - -# --------------------------------------------------------------------------- -# _update_file_patch_strings — non-participant keep-old vote (lines 3170-3172) -# --------------------------------------------------------------------------- - - -@mock_patch(_PATCH_CALL_TOOL) -def test_rewrite_non_participant_casts_keep_old_vote(mock_call, tmp_path): - """A function that fails the rewrite (edit_failure) still casts a keep-old - vote, preventing a const from being updated when only one of two users - successfully renamed it. - - Scenario: - test_a: classify → rename X → after.X; verify OK → string_swap_results. - test_b: classify → needs_rewrite → rewrite → LLM returns None (failure) - → edit_failure → NOT in string_swap_results. - - Without the keep-old fix: X proposals = {"after.X"} (single) → const updated. - With the keep-old fix: X proposals = {"after.X", "old.X"} → conflicting → - test_a inlined, const definition unchanged. - """ - src = ( - 'TARGET = "crispen.before.X"\n' - "\n" - "@patch(TARGET)\n" - "def test_a(mock_x):\n" - " pass\n" - "\n" - "@patch(TARGET)\n" - "def test_b(mock_x):\n" - " pass\n" - ) - scan = str(tmp_path / "test_foo.py") - # test_a: classify → rename → verify OK. - # test_b: classify → needs_rewrite → rewrite attempt → None response (failure). - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.after.X"}, - } - ), - _ok(_VERIFY_OK), - _ok({"needs_rewrite": True}), - LLMCallResult(tool_input=None, elapsed=0.0, input_tokens=0, output_tokens=0), - ] - result, changed, cross = _process_file_source( - src, - {"crispen.before.X"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - ) - # test_b failed → keep-old vote → conflicting → const NOT updated. - assert 'TARGET = "crispen.before.X"' in result - # test_a's decorator inlined individually. - assert '@patch("crispen.after.X")' in result - - -@mock_patch(_PATCH_CALL_TOOL) -def test_rewrite_non_participant_cross_file_ref_skipped(mock_call, tmp_path): - """Non-participant with a cross-file const ref: the ref.source_file != scan_file_abs - branch is False, so no same-file keep-old vote is cast (3171->3170 branch). - - test_a: succeeds (in string_swap_results). - test_b: fails (not in string_swap_results). test_b's const is defined in - helpers.py (cross-file) so the keep-old loop skips it — no - same_file_proposals entry for that ref. - """ - helpers = tmp_path / "helpers.py" - helpers.write_text('TARGET = "crispen.before.X"\n', encoding="utf-8") - src = ( - "from .helpers import TARGET\n" - "\n" - "@patch(TARGET)\n" - "def test_a(mock_x):\n" - " pass\n" - "\n" - "@patch(TARGET)\n" - "def test_b(mock_x):\n" - " pass\n" - ) - scan = str(tmp_path / "test_foo.py") - mock_call.side_effect = [ - _ok( - { - "needs_rewrite": False, - "patch_renames": {"crispen.before.X": "crispen.after.X"}, - } - ), - _ok(_VERIFY_OK), - _ok({"needs_rewrite": True}), - LLMCallResult(tool_input=None, elapsed=0.0, input_tokens=0, output_tokens=0), - ] - result, changed, cross = _process_file_source( - src, - {"crispen.before.X"}, - "ctx", - MagicMock(), - _CFG, - 1, - scan_file=scan, - repo_root=str(tmp_path), - ) - # Cross-file ref → no same-file conflict → cross updated (test_a's rename wins). - helpers_abs = str(helpers.resolve()) - assert helpers_abs in cross - assert cross[helpers_abs] == {"crispen.before.X": "crispen.after.X"} - - -# --------------------------------------------------------------------------- -# _callgraph_update_file — BFS-ambiguous const-ref casts keep-old vote (line 3420) -# --------------------------------------------------------------------------- - - -def test_callgraph_const_ref_no_scan_file_skips_keep_old(tmp_path): - """scan_file=None → scan_file_abs="" (falsy) → the keep-old block is skipped - entirely (3420->3426 branch). BFS ambiguous functions don't cast any vote. - The string literal path is still updated normally. - """ - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", - "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", - }, - new_module_paths={ - "placement.py": "pkg.placement", - "conflict.py": "pkg.conflict", - }, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = ( - "from pkg.placement import helper\n" - "from pkg.conflict import resolve\n" - '@patch("pkg.orig.use_fn")\n' - "def test_f(m):\n" - " helper()\n" - " resolve()\n" - ) - # scan_file=None → scan_file_abs="" → keep-old block skipped; string literal - # unchanged because BFS is ambiguous (no resolved result). - result, changed, _unresolved = _callgraph_update_file( - test_src, - {"pkg.orig.use_fn"}, - [ctx], - scan_file=None, - index=None, - ) - assert not changed - - -def test_callgraph_const_ref_ambiguous_casts_keep_old_vote(tmp_path): - """When BFS finds multiple candidates for a const-backed path (ambiguous), - the function casts a keep-old vote so a shared constant isn't updated to a - value that is wrong for the ambiguous function. - - test_a: calls helper() → placement (single BFS candidate) → vote "placement". - test_b: calls helper() + resolve() → placement AND conflict (2 BFS candidates - for use_fn) → ambiguous → keep-old vote. - Proposals for _PATCH_USE: {"pkg.placement.use_fn", "pkg.orig.use_fn"} → conflict - → constant NOT updated; test_a gets its decorator inlined individually. - """ - ctx = _FLContext( - filepath="/proj/pkg/orig.py", - old_module="pkg.orig", - original_source="from external import use_fn\n", - modified_source="", - new_files={ - "placement.py": "from external import use_fn\ndef helper(): use_fn()\n", - "conflict.py": "from external import use_fn\ndef resolve(): use_fn()\n", - }, - new_module_paths={ - "placement.py": "pkg.placement", - "conflict.py": "pkg.conflict", - }, - entity_to_target={}, - forking_old_paths={"pkg.orig.use_fn"}, - ) - test_src = ( - "from pkg.placement import helper\n" - "from pkg.conflict import resolve\n" - '_PATCH_USE = "pkg.orig.use_fn"\n' - "@patch(_PATCH_USE)\n" - "def test_a(m):\n" - " helper()\n" - "\n" - "@patch(_PATCH_USE)\n" - "def test_b(m):\n" - " helper()\n" - " resolve()\n" - ) - scan = str(tmp_path / "test_foo.py") - index = _make_cuf_index(scan, test_src) - result, changed, _unresolved = _callgraph_update_file( - test_src, {"pkg.orig.use_fn"}, [ctx], scan_file=scan, index=index - ) - # test_b is ambiguous → keep-old vote → conflict with test_a's rename vote. - # Constant definition must NOT be updated. - assert '_PATCH_USE = "pkg.orig.use_fn"' in result - # test_a's decorator IS inlined (it had a resolved rename). - assert '@patch("pkg.placement.use_fn")' in result diff --git a/tests/test_runner.py b/tests/test_runner.py index 6e3b3f9..9d48db4 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,2073 +1 @@ -"""Tests for file_limiter.runner — 100% branch coverage.""" - from __future__ import annotations - -from unittest.mock import patch - -from crispen.config import CrispenConfig -from crispen.file_limiter.advisor import FileLimiterPlan, GroupPlacement -from crispen.file_limiter.classifier import ClassifiedEntities -from crispen.file_limiter.code_gen import SplitResult -from crispen.file_limiter.entity_parser import Entity, EntityKind -from crispen.file_limiter.runner import ( - _MAIN_SUBDIR_SUFFIXES, - _detect_naming_conflicts, - _has_main_block, - _is_whole_file_diff, - _strip_imports_by_line, - _verify_preservation, - run_file_limiter, -) - -_CONFIG = CrispenConfig() -# Zero-retry config for tests that exercise a single-attempt failure path. -_CONFIG_NO_RETRY = CrispenConfig(file_limiter_retries=0) -_PATCH_CLASSIFY = "crispen.file_limiter.runner.classify_entities" -_PATCH_ADVISE = "crispen.file_limiter.runner.advise_file_limiter" -_PATCH_GEN = "crispen.file_limiter.runner.generate_file_splits" -_PATCH_RESOLVE = "crispen.file_limiter.runner.resolve_naming_conflicts" - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_entity(name: str, start: int, end: int) -> Entity: - return Entity(EntityKind.FUNCTION, name, start, end, [name]) - - -def _make_classified(entities=None) -> ClassifiedEntities: - return ClassifiedEntities( - entities=entities or [], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[], - set_3_groups=[], - abort=False, - ) - - -def _abort_plan() -> FileLimiterPlan: - return FileLimiterPlan(set3_migrate=[], placements=[], abort=True) - - -def _empty_plan() -> FileLimiterPlan: - return FileLimiterPlan(set3_migrate=[], placements=[], abort=False) - - -def _plan_with(group: list, target: str) -> FileLimiterPlan: - return FileLimiterPlan( - set3_migrate=[], - placements=[GroupPlacement(group=group, target_file=target)], - abort=False, - ) - - -def _classified_with_groups(entities=None) -> ClassifiedEntities: - """Classified result with non-empty set_3_groups (triggers LLM advise).""" - ents = entities or [_make_entity("foo", 1, 2)] - return ClassifiedEntities( - entities=ents, - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[], - set_3_groups=[[e.name for e in ents]], - abort=False, - ) - - -def _good_split(entity_name: str = "foo", target: str = "utils.py") -> SplitResult: - return SplitResult( - new_files={target: f"def {entity_name}():\n pass"}, - original_source="# original updated\n", - abort=False, - ) - - -# --------------------------------------------------------------------------- -# _verify_preservation -# --------------------------------------------------------------------------- - - -def test_verify_entity_source_in_original(): - # Entity that stayed in the original file — passes verification but is not - # counted (it wasn't a FileLimiter edit). - post_source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - split = SplitResult( - new_files={}, - original_source="def foo():\n pass\n", - abort=False, - ) - vr = _verify_preservation([entity], split, post_source, []) - assert vr.failures == [] - assert vr.verified_functions == 0 - assert vr.verified_lines == 0 - - -def test_verify_entity_source_in_new_file(): - # Entity that was migrated — passes verification and is counted. - post_source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - split = SplitResult( - new_files={"utils.py": "def foo():\n pass"}, - original_source="# original\n", - abort=False, - ) - placements = [GroupPlacement(group=["foo"], target_file="utils.py")] - vr = _verify_preservation([entity], split, post_source, placements) - assert vr.failures == [] - assert vr.verified_functions == 1 - assert vr.verified_lines == 2 # "def foo():\n pass" → 2 lines matched - - -def test_verify_entity_source_missing(): - post_source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - split = SplitResult( - new_files={}, - original_source="# nothing relevant\n", - abort=False, - ) - vr = _verify_preservation([entity], split, post_source, []) - assert len(vr.failures) == 1 - assert "'foo'" in vr.failures[0] - assert "1" in vr.failures[0] # start line - assert "2" in vr.failures[0] # end line - assert vr.verified_lines == 0 - - -def test_verify_entity_source_missing_long(): - # Entity with more than 3 lines → preview includes trailing "..." - post_source = "def foo():\n a = 1\n b = 2\n c = 3\n pass\n" - entity = _make_entity("foo", 1, 5) - split = SplitResult( - new_files={}, - original_source="# nothing relevant\n", - abort=False, - ) - vr = _verify_preservation([entity], split, post_source, []) - assert len(vr.failures) == 1 - assert "..." in vr.failures[0] - - -def test_verify_empty_entity_source_skipped(): - # Entity spanning only a blank line → rstrip → "" → falsy → skipped. - post_source = "\n" - entity = _make_entity("_block_1", 1, 1) - split = SplitResult( - new_files={}, - original_source="# completely different", - abort=False, - ) - vr = _verify_preservation([entity], split, post_source, []) - assert vr.failures == [] - assert vr.verified_lines == 0 - - -def test_verify_top_level_entity_skipped(): - # TOP_LEVEL entities (import/docstring blocks) are always skipped — - # they are intentionally restructured when the file is split. - post_source = "from __future__ import annotations\nimport os\n" - entity = Entity(EntityKind.TOP_LEVEL, "_block_1", 1, 2, ["annotations", "os"]) - split = SplitResult( - new_files={}, - original_source="# completely different", - abort=False, - ) - vr = _verify_preservation([entity], split, post_source, []) - assert vr.failures == [] - assert vr.verified_lines == 0 - - -def test_verify_annotation_migrated(): - # Failure for an entity that was in the plan → annotated "migrated → target". - post_source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - split = SplitResult( - new_files={"utils.py": "# empty"}, - original_source="# empty", - abort=False, - ) - placements = [GroupPlacement(group=["foo"], target_file="utils.py")] - vr = _verify_preservation([entity], split, post_source, placements) - assert len(vr.failures) == 1 - assert "migrated" in vr.failures[0] - assert "utils.py" in vr.failures[0] - - -def test_verify_annotation_stayed(): - # Failure for an entity not in any placement → annotated "stayed in original". - post_source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - split = SplitResult( - new_files={}, - original_source="# empty", - abort=False, - ) - vr = _verify_preservation([entity], split, post_source, []) - assert len(vr.failures) == 1 - assert "stayed in original" in vr.failures[0] - - -def test_verify_pruned_inline_import_passes(): - # Entity has an inline import; the new file has it pruned to a top-level one. - # Both sides are stripped before comparison, so the match succeeds. - # verified_lines counts only the non-import lines of the migrated entity. - post_source = "def foo():\n import os\n return os.getcwd()\n" - entity = _make_entity("foo", 1, 3) - split = SplitResult( - new_files={"utils.py": "import os\n\ndef foo():\n return os.getcwd()"}, - original_source="# original\n", - abort=False, - ) - placements = [GroupPlacement(group=["foo"], target_file="utils.py")] - vr = _verify_preservation([entity], split, post_source, placements) - assert vr.failures == [] - # "def foo():\n return os.getcwd()" → 2 lines (import stripped) - assert vr.verified_lines == 2 - - -def test_verify_inline_import_not_pruned_also_passes(): - # Import was NOT pruned — it appears on both sides. Stripping both sides - # still produces a match. - post_source = "def foo():\n import os\n return os.getcwd()\n" - entity = _make_entity("foo", 1, 3) - split = SplitResult( - new_files={"utils.py": "def foo():\n import os\n return os.getcwd()"}, - original_source="# original\n", - abort=False, - ) - placements = [GroupPlacement(group=["foo"], target_file="utils.py")] - vr = _verify_preservation([entity], split, post_source, placements) - assert vr.failures == [] - assert vr.verified_lines == 2 - - -def test_verify_multiline_import_stripped(): - # Multi-line imports are removed correctly using AST line spans. - post_source = ( - "def foo():\n" - " from os import (\n" - " path,\n" - " getcwd,\n" - " )\n" - " return getcwd()\n" - ) - entity = _make_entity("foo", 1, 6) - # New file has the multi-line import removed (3 lines gone). - split = SplitResult( - new_files={ - "utils.py": "from os import path, getcwd\n\ndef foo():\n return getcwd()" - }, - original_source="# original\n", - abort=False, - ) - placements = [GroupPlacement(group=["foo"], target_file="utils.py")] - vr = _verify_preservation([entity], split, post_source, placements) - assert vr.failures == [] - # "def foo():\n return getcwd()" → 2 lines (4-line import stripped) - assert vr.verified_lines == 2 - - -def test_verify_async_def_entity_passes(): - # Async functions are found after import stripping (no imports involved). - post_source = "async def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - split = SplitResult( - new_files={"utils.py": "async def foo():\n pass"}, - original_source="# original\n", - abort=False, - ) - placements = [GroupPlacement(group=["foo"], target_file="utils.py")] - vr = _verify_preservation([entity], split, post_source, placements) - assert vr.failures == [] - assert vr.verified_lines == 2 - - -def test_verify_blank_line_collapse_after_pruning_passes(): - # Regression: when multiple consecutive inline imports are all pruned to - # top-level, the resulting consecutive blank lines (3+ newlines before - # indented content) are collapsed by _normalize_blank_lines in code_gen. - # Verification must apply the same normalization to entity_no_imports so - # the substring match doesn't fail due to a blank-line count mismatch. - post_source = ( - "def test_seq():\n" - ' """Docstring."""\n' - " import libcst as cst\n" - "\n" - " from libcst.metadata import MetadataWrapper\n" - "\n" - " from foo import Bar\n" - "\n" - " x = cst.parse_module('')\n" - " w = MetadataWrapper(x)\n" - " b = Bar()\n" - ) - entity = _make_entity("test_seq", 1, 13) - # New file has all 3 inline imports hoisted to top-level and pruned from - # the function body; _normalize_blank_lines collapsed the 3+ consecutive - # blank lines down to 1. - new_file_src = ( - "import libcst as cst\n" - "from libcst.metadata import MetadataWrapper\n" - "from foo import Bar\n" - "\n" - "def test_seq():\n" - ' """Docstring."""\n' - "\n" - " x = cst.parse_module('')\n" - " w = MetadataWrapper(x)\n" - " b = Bar()\n" - ) - split = SplitResult( - new_files={"test_collectors.py": new_file_src}, - original_source="# original\n", - abort=False, - ) - placements = [GroupPlacement(group=["test_seq"], target_file="test_collectors.py")] - vr = _verify_preservation([entity], split, post_source, placements) - assert vr.failures == [] - assert vr.verified_functions == 1 - - -def test_verify_inline_import_not_pruned_with_surrounding_blanks_passes(): - # Regression: entity has an inline import surrounded by blank lines that is - # NOT pruned to a top-level import in the new file. After - # _strip_imports_by_line removes the import from the new file's content, the - # two surrounding blank lines merge into 3+ consecutive newlines before - # indented code — which _normalize_blank_lines in the new file did NOT - # collapse (it only runs before the import was stripped in verification). - # Verification must apply the same _EXCESS_BLANK_BODY_RE normalization to - # combined_no_imports so the blank-line count matches entity_no_imports. - post_source = ( - "def test_foo():\n" - " x = 1\n" - "\n" - " import pathlib\n" - "\n" - " y = pathlib.Path('.')\n" - " return y\n" - ) - entity = _make_entity("test_foo", 1, 8) - # New file keeps the inline import (not pruned — no module-level pathlib). - new_file_src = ( - "def test_foo():\n" - " x = 1\n" - "\n" - " import pathlib\n" - "\n" - " y = pathlib.Path('.')\n" - " return y\n" - ) - split = SplitResult( - new_files={"test_patch.py": new_file_src}, - original_source="# original\n", - abort=False, - ) - placements = [GroupPlacement(group=["test_foo"], target_file="test_patch.py")] - vr = _verify_preservation([entity], split, post_source, placements) - assert vr.failures == [] - assert vr.verified_functions == 1 - - -def test_verify_entity_with_name_rewrites_passes(): - # The original entity references SAFE_MODE; after splitting it becomes - # conversion.SAFE_MODE in the new file. Verification must apply the - # name_rewrites before the substring check so it passes rather than - # reporting a false failure. - post_source = ( - "def create_runtime(safe_mode=None):\n" - " if safe_mode is None:\n" - " safe_mode = SAFE_MODE\n" - ) - entity = _make_entity("create_runtime", 1, 3) - new_file_src = ( - "def create_runtime(safe_mode=None):\n" - " if safe_mode is None:\n" - " safe_mode = conversion.SAFE_MODE\n" - ) - split = SplitResult( - new_files={"runtime.py": new_file_src}, - original_source="# re-exports\n", - abort=False, - entity_name_rewrites={"create_runtime": {"SAFE_MODE": "conversion.SAFE_MODE"}}, - ) - placements = [GroupPlacement(group=["create_runtime"], target_file="runtime.py")] - vr = _verify_preservation([entity], split, post_source, placements) - assert vr.failures == [] - # The function passes verification. Only the 1 rewritten line is excluded; - # the other 2 unchanged lines are credited. - assert vr.verified_functions == 1 - assert vr.verified_lines == 2 - - -# --------------------------------------------------------------------------- -# _strip_imports_by_line -# --------------------------------------------------------------------------- - - -def test_strip_imports_no_imports(): - src = "def foo():\n return 1\n" - assert _strip_imports_by_line(src) == src - - -def test_strip_imports_single_line(): - src = "import os\nx = 1\n" - assert _strip_imports_by_line(src) == "x = 1\n" - - -def test_strip_imports_multiline(): - src = "from os import (\n path,\n getcwd,\n)\nx = 1\n" - assert _strip_imports_by_line(src) == "x = 1\n" - - -def test_strip_imports_inner_import(): - # Imports inside a function body are also stripped. - src = "def foo():\n import os\n return os.getcwd()\n" - assert _strip_imports_by_line(src) == "def foo():\n return os.getcwd()\n" - - -def test_strip_imports_syntax_error_returns_unchanged(): - src = "def foo(:\n pass\n" - assert _strip_imports_by_line(src) == src - - -# --------------------------------------------------------------------------- -# run_file_limiter — dashed parent directory early-return -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CLASSIFY) -def test_runner_dashed_parent_dir_skips(mock_classify): - # A filepath whose parent contains a dash must be skipped immediately, - # before classify_entities is ever called. - result = run_file_limiter( - "tests/cross-engine/test_lever.py", "", "x = 1\n", [(1, 1)], _CONFIG - ) - assert result.abort is True - assert "cross-engine" in result.messages[0] - assert "dash" in result.messages[0] - mock_classify.assert_not_called() - - -@patch(_PATCH_CLASSIFY) -def test_runner_dashed_parent_dir_deep_skips(mock_classify): - # Dash anywhere in the ancestor chain (not just the immediate parent). - result = run_file_limiter( - "src/my-pkg/sub/module.py", "", "x = 1\n", [(1, 1)], _CONFIG - ) - assert result.abort is True - assert "my-pkg" in result.messages[0] - mock_classify.assert_not_called() - - -# --------------------------------------------------------------------------- -# run_file_limiter — classified.abort early-return (no retry) -# --------------------------------------------------------------------------- - - -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_nonexistent_parent_dir_existing_dirs_empty(mock_classify, mock_advise): - """When source dir doesn't exist, iterdir raises FileNotFoundError → empty set.""" - mock_classify.return_value = ClassifiedEntities( - entities=[], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[], - set_3_groups=[], - abort=True, - ) - # /nonexistent/parent doesn't exist; iterdir() will raise FileNotFoundError. - result = run_file_limiter( - "/nonexistent/parent/module.py", - "", - "def foo(): pass\n", - [(1, 1)], - _CONFIG, - ) - # Abort from classifier, but the FileNotFoundError branch was hit. - assert result.abort is True - - -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_classifier_abort(mock_classify, mock_advise): - # classified.abort=True → early return before LLM; advise never called. - mock_classify.return_value = ClassifiedEntities( - entities=[_make_entity("a", 1, 2), _make_entity("b", 3, 4)], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[], - set_3_groups=[], - abort=True, - abort_reason="", - ) - - result = run_file_limiter("big.py", "", "def a(): b()\ndef b(): a()\n", [], _CONFIG) - - assert result.abort is True - mock_advise.assert_not_called() - assert any("cannot be split" in m for m in result.messages) - - -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_classifier_abort_with_reason(mock_classify, mock_advise): - mock_classify.return_value = ClassifiedEntities( - entities=[_make_entity("a", 1, 2), _make_entity("b", 3, 4)], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[], - set_3_groups=[], - abort=True, - abort_reason="all 2 top-level entities form one dependency cycle", - ) - - result = run_file_limiter("big.py", "", "def a(): b()\ndef b(): a()\n", [], _CONFIG) - - assert result.abort is True - mock_advise.assert_not_called() - assert any("dependency cycle" in m for m in result.messages) - - -# --------------------------------------------------------------------------- -# run_file_limiter — plan.abort path -# --------------------------------------------------------------------------- - - -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_plan_abort(mock_classify, mock_advise): - mock_classify.return_value = _make_classified() - mock_advise.return_value = _abort_plan() - - result = run_file_limiter( - "big.py", "", "def foo():\n pass\n", [], _CONFIG_NO_RETRY - ) - - assert result.abort is True - assert result.new_files == {} - assert any("cannot be split" in m for m in result.messages) - - -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_plan_abort_with_reason(mock_classify, mock_advise): - mock_classify.return_value = _make_classified() - mock_advise.return_value = FileLimiterPlan( - set3_migrate=[], placements=[], abort=True, abort_reason="all 3 entities cycle" - ) - - result = run_file_limiter( - "big.py", "", "def foo():\n pass\n", [], _CONFIG_NO_RETRY - ) - - assert result.abort is True - assert any("all 3 entities cycle" in m for m in result.messages) - - -# --------------------------------------------------------------------------- -# run_file_limiter — plan.abort retry paths -# --------------------------------------------------------------------------- - - -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_plan_abort_retries_and_fails(mock_classify, mock_advise): - # retries=1: both attempts produce plan.abort (set-3 failure) → 2 SKIP messages. - mock_classify.return_value = _make_classified() - mock_advise.return_value = FileLimiterPlan( - set3_migrate=[], - placements=[], - abort=True, - abort_reason="LLM failed to plan set-3 groups", - ) - cfg = CrispenConfig(file_limiter_retries=1) - - result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) - - assert result.abort is True - assert mock_advise.call_count == 2 - assert sum(1 for m in result.messages if "cannot be split" in m) == 2 - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_plan_abort_retries_and_succeeds(mock_classify, mock_advise, mock_gen): - # retries=1: first plan.abort (placement failure), second succeeds. - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.side_effect = [ - FileLimiterPlan( - set3_migrate=[], - placements=[], - abort=True, - abort_reason="LLM failed to assign file placements", - ), - _plan_with(["foo"], "utils.py"), - ] - mock_gen.return_value = _good_split() - cfg = CrispenConfig(file_limiter_retries=1) - - result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) - - assert result.abort is False - assert mock_advise.call_count == 2 - # Failed attempt message is preserved alongside the success message. - assert any("cannot be split" in m for m in result.messages) - assert any("FileLimiter: moved" in m for m in result.messages) - # Feedback was forwarded on the second call. - assert mock_advise.call_args_list[1].kwargs["prev_placement_failure"] != "" - - -# --------------------------------------------------------------------------- -# run_file_limiter — no placements path -# --------------------------------------------------------------------------- - - -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_no_placements(mock_classify, mock_advise): - mock_classify.return_value = _make_classified() - mock_advise.return_value = _empty_plan() - - source = "def foo():\n pass\n" - result = run_file_limiter("big.py", "", source, [], _CONFIG) - - assert result.abort is False - assert result.new_files == {} - assert result.original_source == source - assert result.messages == [] - - -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_no_placements_with_groups(mock_classify, mock_advise): - # When set_3_groups is non-empty but the LLM selects nothing to migrate, - # runner should emit a SKIP message so the user knows the file was examined. - mock_classify.return_value = _classified_with_groups() - mock_advise.return_value = _empty_plan() - - source = "def foo():\n pass\n" - result = run_file_limiter("big.py", "", source, [], _CONFIG_NO_RETRY) - - assert result.abort is False - assert result.new_files == {} - assert any("no entities selected for migration" in m for m in result.messages) - - -# --------------------------------------------------------------------------- -# run_file_limiter — no-migration retry paths -# --------------------------------------------------------------------------- - - -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_no_migration_retries_and_fails(mock_classify, mock_advise): - # retries=1: both attempts → no entities selected → 2 SKIP msgs, abort=False. - mock_classify.return_value = _classified_with_groups() - mock_advise.return_value = _empty_plan() - cfg = CrispenConfig(file_limiter_retries=1) - - result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) - - assert result.abort is False - assert mock_advise.call_count == 2 - assert sum(1 for m in result.messages if "no entities selected" in m) == 2 - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_no_migration_retries_and_succeeds(mock_classify, mock_advise, mock_gen): - # retries=1: first attempt → no migration; second → placements and success. - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _classified_with_groups(entities=[entity]) - mock_advise.side_effect = [_empty_plan(), _plan_with(["foo"], "utils.py")] - mock_gen.return_value = _good_split() - cfg = CrispenConfig(file_limiter_retries=1) - - result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) - - assert result.abort is False - assert mock_advise.call_count == 2 - # Failed attempt message is preserved alongside the success message. - assert any("no entities selected" in m for m in result.messages) - assert any("FileLimiter: moved" in m for m in result.messages) - # Feedback about all-stay was forwarded on the second call. - assert mock_advise.call_args_list[1].kwargs["prev_set3_failure"] != "" - - -# --------------------------------------------------------------------------- -# run_file_limiter — single-file placement guard -# --------------------------------------------------------------------------- - - -# --------------------------------------------------------------------------- -# Helpers for single-file guard tests (subdir-split mode) -# --------------------------------------------------------------------------- - -# A two-line source whose diff_ranges covers the whole file, triggering subdir -# split for "big.py" → subdir_name="big". Path("big") must not exist on disk. -_SUBDIR_SRC = "x = 1\ny = 2\n" -_SUBDIR_RANGES = [(1, 2)] - - -def _plan_two_same_target() -> FileLimiterPlan: - """Two groups, both assigned to the same target file.""" - return FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="utils.py"), - ], - abort=False, - ) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_all_in_one_file_subdir_retries_and_fails( - mock_classify, mock_advise, mock_gen -): - # Subdir split + all groups → same file → guard triggers every attempt. - # Two groups required so the n_groups > 1 pre-loop check doesn't fire first. - mock_classify.return_value = ClassifiedEntities( - entities=[_make_entity("foo", 1, 1), _make_entity("bar", 2, 2)], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[["foo"], ["bar"]], - set_3_groups=[], - abort=False, - ) - mock_advise.return_value = _plan_two_same_target() - cfg = CrispenConfig(file_limiter_retries=0) - - result = run_file_limiter("big.py", "", _SUBDIR_SRC, _SUBDIR_RANGES, cfg) - - assert result.abort is False - assert any("single file" in m for m in result.messages) - mock_gen.assert_not_called() - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_all_in_one_file_subdir_retries_and_succeeds( - mock_classify, mock_advise, mock_gen -): - # Subdir split: first attempt all in one file, second splits into two. - entity1 = _make_entity("foo", 1, 1) - entity2 = _make_entity("bar", 2, 2) - # Two groups required so the n_groups > 1 pre-loop check doesn't fire first. - mock_classify.return_value = ClassifiedEntities( - entities=[entity1, entity2], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[["foo"], ["bar"]], - set_3_groups=[], - abort=False, - ) - mock_advise.side_effect = [ - _plan_two_same_target(), - FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="helpers.py"), - ], - abort=False, - ), - ] - mock_gen.return_value = SplitResult( - new_files={ - "big/utils.py": "x = 1", - "big/helpers.py": "y = 2", - }, - original_source=_SUBDIR_SRC, - abort=False, - ) - cfg = CrispenConfig(file_limiter_retries=1) - - result = run_file_limiter("big.py", "", _SUBDIR_SRC, _SUBDIR_RANGES, cfg) - - assert result.abort is False - assert any("single file" in m for m in result.messages) - assert any("FileLimiter: moved" in m for m in result.messages) - assert mock_advise.call_args_list[1].kwargs["prev_placement_failure"] != "" - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_all_in_one_file_non_subdir_allowed( - mock_classify, mock_advise, mock_gen -): - # Non-subdir split: all groups → same file is always fine. - entity1 = _make_entity("foo", 1, 2) - entity2 = _make_entity("bar", 3, 4) - mock_classify.return_value = _make_classified(entities=[entity1, entity2]) - mock_advise.return_value = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="utils.py"), - ], - abort=False, - ) - mock_gen.return_value = SplitResult( - new_files={"utils.py": "def foo():\n pass\ndef bar():\n pass"}, - original_source="# reduced\n", - abort=False, - ) - - # diff_ranges=[] → not a whole-file diff → subdir_name=None → guard inactive. - result = run_file_limiter( - "big.py", - "", - "def foo():\n pass\ndef bar():\n pass\n", - [], - _CONFIG_NO_RETRY, - ) - - assert result.abort is False - assert not any("single file" in m for m in result.messages) - mock_gen.assert_called_once() - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_single_group_subdir_aborts_silently( - mock_classify, mock_advise, mock_gen -): - # Subdir split with only 1 group: moving it would just rename the file, - # not split it, causing infinite subdirectory nesting across runs. - # Abort immediately without calling the LLM. - entity = _make_entity("foo", 1, 1) - mock_classify.return_value = ClassifiedEntities( - entities=[entity], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[["foo"]], - set_3_groups=[], - abort=False, - ) - cfg = CrispenConfig(file_limiter_retries=0) - - result = run_file_limiter("big.py", "", _SUBDIR_SRC, _SUBDIR_RANGES, cfg) - - assert result.abort is True - assert result.messages == [] - mock_advise.assert_not_called() - mock_gen.assert_not_called() - - -# --------------------------------------------------------------------------- -# run_file_limiter — verification fails path -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_verification_fails(mock_classify, mock_advise, mock_gen): - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["foo"], "utils.py") - # Return a split where foo's source is NOT present anywhere. - mock_gen.return_value = SplitResult( - new_files={"utils.py": "# empty placeholder"}, - original_source="# empty original", - abort=False, - ) - - result = run_file_limiter("big.py", "", source, [], _CONFIG) - - assert result.abort is True - assert result.original_source == source - assert any("verification failed" in m for m in result.messages) - - -# --------------------------------------------------------------------------- -# run_file_limiter — success path -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_success(mock_classify, mock_advise, mock_gen): - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["foo"], "utils.py") - mock_gen.return_value = SplitResult( - new_files={"utils.py": "def foo():\n pass"}, - original_source="# original updated\n", - abort=False, - ) - - result = run_file_limiter("big.py", "", source, [], _CONFIG) - - assert result.abort is False - assert "utils.py" in result.new_files - assert result.original_source == "# original updated\n" - assert any("FileLimiter: moved" in m for m in result.messages) - assert any("foo" in m for m in result.messages) - assert any("utils.py" in m for m in result.messages) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_passes_pytest_conftest_to_generate( - mock_classify, mock_advise, mock_gen -): - # Verify config.file_limiter_pytest_conftest is forwarded to generate_file_splits. - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["foo"], "utils.py") - mock_gen.return_value = _good_split() - - config_false = CrispenConfig(file_limiter_pytest_conftest=False) - run_file_limiter("big.py", "", source, [], config_false) - - _, call_kwargs = mock_gen.call_args - assert call_kwargs.get("pytest_conftest") is False - - -# --------------------------------------------------------------------------- -# run_file_limiter — cycle abort path (split.abort=True) -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_split_aborts_on_cycle(mock_classify, mock_advise, mock_gen): - # generate_file_splits detects a cycle and returns abort=True with no - # new_files. run_file_limiter must emit a SKIP message (not bogus "moved" - # messages) and return abort=True so the engine skips the file. - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["foo"], "utils.py") - mock_gen.return_value = SplitResult( - new_files={}, - original_source=source, - abort=True, - ) - - result = run_file_limiter("big.py", "", source, [], _CONFIG_NO_RETRY) - - assert result.abort is True - assert result.new_files == {} - assert result.original_source == source - # Must not claim to have moved anything. - assert not any("FileLimiter: moved" in m for m in result.messages) - assert any("cannot be split" in m for m in result.messages) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_split_aborts_with_reason(mock_classify, mock_advise, mock_gen): - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["foo"], "utils.py") - mock_gen.return_value = SplitResult( - new_files={}, - original_source=source, - abort=True, - abort_reason="proposed split would create circular file imports", - ) - - result = run_file_limiter("big.py", "", source, [], _CONFIG_NO_RETRY) - - assert result.abort is True - assert any("circular file imports" in m for m in result.messages) - - -# --------------------------------------------------------------------------- -# run_file_limiter — split.abort retry paths -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_split_abort_retries_and_fails(mock_classify, mock_advise, mock_gen): - # retries=1: both attempts produce split.abort → 2 SKIP messages, abort=True. - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["foo"], "utils.py") - mock_gen.return_value = SplitResult( - new_files={}, - original_source=source, - abort=True, - abort_reason="proposed split would create circular file imports", - ) - cfg = CrispenConfig(file_limiter_retries=1) - - result = run_file_limiter("big.py", "", source, [], cfg) - - assert result.abort is True - assert mock_advise.call_count == 2 - assert mock_gen.call_count == 2 - assert sum(1 for m in result.messages if "cannot be split" in m) == 2 - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_split_abort_retries_and_succeeds(mock_classify, mock_advise, mock_gen): - # retries=1: first split.abort, second succeeds → only "moved" message, no SKIP. - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["foo"], "utils.py") - mock_gen.side_effect = [ - SplitResult( - new_files={}, - original_source=source, - abort=True, - abort_reason="circular imports", - ), - _good_split(), - ] - cfg = CrispenConfig(file_limiter_retries=1) - - result = run_file_limiter("big.py", "", source, [], cfg) - - assert result.abort is False - assert mock_advise.call_count == 2 - assert mock_gen.call_count == 2 - # Failed attempt message is preserved alongside the success message. - assert any("cannot be split" in m for m in result.messages) - assert any("FileLimiter: moved" in m for m in result.messages) - # Circular-import feedback was forwarded on the second call. - prev_pf = mock_advise.call_args_list[1].kwargs["prev_placement_failure"] - assert "circular" in prev_pf - - -# --------------------------------------------------------------------------- -# run_file_limiter — test_ prefix normalisation -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_adds_test_prefix_to_new_files(mock_classify, mock_advise, mock_gen): - # When the source file is test_*.py, target files in the same directory - # must also have the test_ prefix so pytest can discover the moved tests. - source = "def test_foo():\n pass\n" - entity = _make_entity("test_foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["test_foo"], "helpers.py") - mock_gen.return_value = SplitResult( - new_files={"test_helpers.py": "def test_foo():\n pass"}, - original_source="# original\n", - abort=False, - ) - - result = run_file_limiter("tests/test_big.py", "", source, [], _CONFIG) - - assert result.abort is False - # The placement target passed to generate_file_splits must have been - # normalised — verify via the success message. - assert any("test_helpers.py" in m for m in result.messages) - assert not any( - "helpers.py" in m and "test_helpers.py" not in m for m in result.messages - ) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_test_prefix_already_present(mock_classify, mock_advise, mock_gen): - # Target file already starts with test_ → name is left unchanged. - source = "def test_foo():\n pass\n" - entity = _make_entity("test_foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["test_foo"], "test_helpers.py") - mock_gen.return_value = SplitResult( - new_files={"test_helpers.py": "def test_foo():\n pass"}, - original_source="# original\n", - abort=False, - ) - - result = run_file_limiter("tests/test_big.py", "", source, [], _CONFIG) - - assert result.abort is False - assert any("test_helpers.py" in m for m in result.messages) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_no_test_prefix_for_helper_only_group( - mock_classify, mock_advise, mock_gen -): - # Source is test_*.py but the group contains only helper functions (no - # test_/Test* names) — the target file must NOT get a test_ prefix so - # pytest does not try to collect it. - source = "def _helper():\n pass\n" - entity = _make_entity("_helper", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["_helper"], "helpers.py") - mock_gen.return_value = SplitResult( - new_files={"helpers.py": "def _helper():\n pass"}, - original_source="# original\n", - abort=False, - ) - - result = run_file_limiter("tests/test_big.py", "", source, [], _CONFIG) - - assert result.abort is False - assert any("helpers.py" in m for m in result.messages) - assert not any("test_helpers.py" in m for m in result.messages) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_init_not_renamed_by_test_prefix_logic( - mock_classify, mock_advise, mock_gen -): - # Defence-in-depth: __init__.py placements must not get the test_ prefix. - source = "def test_foo():\n pass\n\ndef _setup():\n pass\n" - e1 = _make_entity("test_foo", 1, 2) - e2 = _make_entity("_setup", 4, 5) - mock_classify.return_value = _make_classified(entities=[e1, e2]) - mock_advise.return_value = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["test_foo"], target_file="cases.py"), - GroupPlacement(group=["_setup"], target_file="__init__.py"), - ], - abort=False, - ) - mock_gen.return_value = SplitResult( - new_files={ - "test_cases.py": "def test_foo():\n pass", - "__init__.py": "def _setup():\n pass", - }, - original_source="# original\n", - abort=False, - ) - - # tests/runner/ has no __init__.py so it won't appear in existing_files. - result = run_file_limiter("tests/runner/test_big.py", "", source, [], _CONFIG) - - assert result.abort is False - # cases.py → test_cases.py (has test_foo in group) - assert any("test_cases.py" in m for m in result.messages) - # __init__.py untouched - assert any("__init__.py" in m for m in result.messages) - assert not any("test___init__.py" in m for m in result.messages) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_no_test_prefix_for_non_test_file(mock_classify, mock_advise, mock_gen): - # Source file is NOT a test module — target file names are left as-is. - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["foo"], "helpers.py") - mock_gen.return_value = SplitResult( - new_files={"helpers.py": "def foo():\n pass"}, - original_source="# original\n", - abort=False, - ) - - result = run_file_limiter("big.py", "", source, [], _CONFIG) - - assert result.abort is False - assert any("helpers.py" in m for m in result.messages) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_strips_tests_suffix_and_adds_prefix( - mock_classify, mock_advise, mock_gen -): - # LLM returns a filename ending with _tests.py — strip the suffix and add - # the test_ prefix so pytest discovers the file. - source = "class TestFoo:\n def test_bar(self):\n pass\n" - entity = _make_entity("TestFoo", 1, 3) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["TestFoo"], "foo_tests.py") - mock_gen.return_value = SplitResult( - new_files={ - "test_foo.py": "class TestFoo:\n def test_bar(self):\n pass" - }, - original_source="# original\n", - abort=False, - ) - - result = run_file_limiter("tests/test_big.py", "", source, [], _CONFIG) - - assert result.abort is False - assert any("test_foo.py" in m for m in result.messages) - assert not any("foo_tests.py" in m for m in result.messages) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_adds_prefix_for_test_class_group(mock_classify, mock_advise, mock_gen): - # Group contains a Test-prefixed class (not test_ function) — must still - # get the test_ file prefix. - source = "class TestFoo:\n def test_bar(self):\n pass\n" - entity = _make_entity("TestFoo", 1, 3) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["TestFoo"], "foo_cases.py") - mock_gen.return_value = SplitResult( - new_files={ - "test_foo_cases.py": "class TestFoo:\n def test_bar(self):\n pass" - }, - original_source="# original\n", - abort=False, - ) - - result = run_file_limiter("tests/test_big.py", "", source, [], _CONFIG) - - assert result.abort is False - assert any("test_foo_cases.py" in m for m in result.messages) - assert not any( - "foo_cases.py" in m and "test_foo_cases.py" not in m for m in result.messages - ) - - -# --------------------------------------------------------------------------- -# _detect_naming_conflicts — unit tests -# --------------------------------------------------------------------------- - - -def test_detect_conflicts_no_conflicts(): - placements = [ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="helpers.py"), - ] - assert _detect_naming_conflicts(placements, frozenset(), frozenset()) == [] - - -def test_detect_conflicts_plan_vs_plan(): - # Plan contains both 'utils.py' and 'utils/io.py' → conflict on stem 'utils'. - placements = [ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="utils/io.py"), - ] - conflicts = _detect_naming_conflicts(placements, frozenset(), frozenset()) - assert len(conflicts) == 1 - assert "'utils.py'" in conflicts[0] - assert "'utils/'" in conflicts[0] - - -def test_detect_conflicts_plan_file_vs_existing_dir(): - # Plan proposes 'models.py' but 'models' directory already exists on disk. - placements = [GroupPlacement(group=["foo"], target_file="models.py")] - conflicts = _detect_naming_conflicts(placements, frozenset(), frozenset({"models"})) - assert len(conflicts) == 1 - assert "'models.py'" in conflicts[0] - assert "'models/'" in conflicts[0] - - -def test_detect_conflicts_plan_dir_vs_existing_file(): - # Plan proposes 'helpers/io.py' but 'helpers.py' already exists on disk. - placements = [GroupPlacement(group=["bar"], target_file="helpers/io.py")] - conflicts = _detect_naming_conflicts( - placements, frozenset({"helpers.py"}), frozenset() - ) - assert len(conflicts) == 1 - assert "'helpers/'" in conflicts[0] - assert "'helpers.py'" in conflicts[0] - - -def test_detect_conflicts_no_filesystem_conflict(): - # Proposed 'utils.py'; existing dir named 'other' — no overlap. - placements = [GroupPlacement(group=["foo"], target_file="utils.py")] - assert _detect_naming_conflicts(placements, frozenset(), frozenset({"other"})) == [] - - -def test_detect_conflicts_multiple_conflicts(): - # Three separate conflicts in one plan. - placements = [ - GroupPlacement(group=["a"], target_file="alpha.py"), # vs alpha/ dir on disk - GroupPlacement(group=["b"], target_file="beta/x.py"), # vs beta.py on disk - GroupPlacement(group=["c"], target_file="gamma.py"), # vs gamma/ in plan - GroupPlacement(group=["d"], target_file="gamma/y.py"), # vs gamma.py in plan - ] - conflicts = _detect_naming_conflicts( - placements, frozenset({"beta.py"}), frozenset({"alpha"}) - ) - assert len(conflicts) == 3 # alpha (disk dir), beta (disk file), gamma (plan) - - -def test_detect_conflicts_subdir_only_no_conflict(): - # All targets are in different subdirectories — no stem overlap. - placements = [ - GroupPlacement(group=["a"], target_file="pkg/models.py"), - GroupPlacement(group=["b"], target_file="pkg/helpers.py"), - ] - # Both land in 'pkg/' — that's fine; only 'pkg.py' vs 'pkg/' would conflict. - assert _detect_naming_conflicts(placements, frozenset(), frozenset()) == [] - - -def test_detect_conflicts_flat_target_in_existing_files(): - # Flat target whose filename is in existing_files (e.g. conftest.py) → conflict. - placements = [GroupPlacement(group=["fix"], target_file="conftest.py")] - conflicts = _detect_naming_conflicts( - placements, frozenset({"conftest.py"}), frozenset() - ) - assert len(conflicts) == 1 - assert "conftest.py" in conflicts[0] - - -# --------------------------------------------------------------------------- -# run_file_limiter — naming conflict retry paths -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_RESOLVE) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_naming_conflict_resolve_succeeds( - mock_classify, mock_advise, mock_resolve, mock_gen -): - # Conflict → resolve returns updated placements → generate called once, advise once. - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - conflicting_plan = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="utils/helpers.py"), # conflict! - ], - abort=False, - ) - resolved_placements = [GroupPlacement(group=["foo"], target_file="models.py")] - mock_advise.return_value = conflicting_plan - mock_resolve.return_value = resolved_placements - mock_gen.return_value = _good_split(entity_name="foo", target="models.py") - - result = run_file_limiter( - "big.py", "", "def foo():\n pass\n", [], _CONFIG_NO_RETRY - ) - - assert result.abort is False - assert mock_advise.call_count == 1 # no outer retry needed - assert mock_resolve.call_count == 1 - assert mock_gen.call_count == 1 - # No SKIP message — resolve handled the conflict without retrying advise. - assert not any("naming conflicts" in m for m in result.messages) - assert any("FileLimiter: moved" in m for m in result.messages) - - -@patch(_PATCH_GEN) -@patch(_PATCH_RESOLVE) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_naming_conflict_resolve_fails_then_outer_retry( - mock_classify, mock_advise, mock_resolve, mock_gen -): - # resolve returns None → outer retry → second advise succeeds. - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - conflicting_plan = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["foo"], target_file="pkg.py"), - GroupPlacement(group=["bar"], target_file="pkg/mod.py"), # conflict! - ], - abort=False, - ) - mock_advise.side_effect = [conflicting_plan, _plan_with(["foo"], "models.py")] - mock_resolve.return_value = None # targeted rename fails - mock_gen.return_value = _good_split(entity_name="foo", target="models.py") - cfg = CrispenConfig(file_limiter_retries=1) - - result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) - - assert result.abort is False - assert mock_advise.call_count == 2 - assert mock_resolve.call_count == 1 - assert any("naming conflicts" in m for m in result.messages) - assert any("FileLimiter: moved" in m for m in result.messages) - # Conflict description was forwarded as feedback for the second advise call. - prev_pf = mock_advise.call_args_list[1].kwargs["prev_placement_failure"] - assert "naming conflicts" in prev_pf - - -@patch(_PATCH_RESOLVE) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_naming_conflict_exhausts_all(mock_classify, mock_advise, mock_resolve): - # resolve always fails, all retries exhausted → abort=True, 2 SKIP messages. - mock_classify.return_value = _make_classified() - conflicting_plan = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["foo"], target_file="pkg.py"), - GroupPlacement(group=["bar"], target_file="pkg/mod.py"), # conflict! - ], - abort=False, - ) - mock_advise.return_value = conflicting_plan - mock_resolve.return_value = None # always fails - cfg = CrispenConfig(file_limiter_retries=1) - - result = run_file_limiter("big.py", "", "def foo():\n pass\n", [], cfg) - - assert result.abort is True - assert mock_advise.call_count == 2 - assert mock_resolve.call_count == 2 - assert sum(1 for m in result.messages if "naming conflicts" in m) == 2 - - -# --------------------------------------------------------------------------- -# _is_whole_file_diff -# --------------------------------------------------------------------------- - - -def test_is_whole_file_diff_empty_ranges(): - assert _is_whole_file_diff([], 5) is False - - -def test_is_whole_file_diff_zero_lines(): - assert _is_whole_file_diff([(1, 3)], 0) is False - - -def test_is_whole_file_diff_gap(): - # Lines 1-2 and 4-5 — line 3 is missing. - assert _is_whole_file_diff([(1, 2), (4, 5)], 5) is False - - -def test_is_whole_file_diff_doesnt_start_at_one(): - # Range starts at line 2 — line 1 is not covered. - assert _is_whole_file_diff([(2, 5)], 5) is False - - -def test_is_whole_file_diff_partial_coverage(): - # Covers lines 1-3 but file has 5 lines. - assert _is_whole_file_diff([(1, 3)], 5) is False - - -def test_is_whole_file_diff_exact_coverage(): - assert _is_whole_file_diff([(1, 5)], 5) is True - - -def test_is_whole_file_diff_multi_range_contiguous(): - # Two adjacent ranges that together cover 1..5. - assert _is_whole_file_diff([(1, 3), (4, 5)], 5) is True - - -def test_is_whole_file_diff_overshoots(): - # Ranges cover more lines than n_lines — still counts as whole-file. - assert _is_whole_file_diff([(1, 10)], 5) is True - - -# --------------------------------------------------------------------------- -# _has_main_block -# --------------------------------------------------------------------------- - - -def test_has_main_block_detects_dunder_main(): - src = "def foo():\n pass\n\nif __name__ == '__main__':\n foo()\n" - assert _has_main_block(src) is True - - -def test_has_main_block_no_main(): - assert _has_main_block("def foo():\n pass\n") is False - - -def test_has_main_block_syntax_error(): - assert _has_main_block("def (:\n") is False - - -# --------------------------------------------------------------------------- -# run_file_limiter — subdir-split: directory already exists -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CLASSIFY) -def test_runner_subdir_split_dir_exists_aborts(mock_classify, tmp_path): - mock_classify.return_value = _make_classified() - # Create a directory named 'service' alongside the source file. - service_dir = tmp_path / "service" - service_dir.mkdir() - filepath = str(tmp_path / "service.py") - - source = "def foo():\n pass\n" - # Whole-file diff: ranges cover all 2 lines. - cfg = CrispenConfig(file_limiter_subdir_split=True) - result = run_file_limiter(filepath, source, source, [(1, 2)], cfg) - - assert result.abort is True - assert result.new_files == {} - assert any("already exists" in m for m in result.messages) - assert any("service/" in m for m in result.messages) - - -@patch(_PATCH_CLASSIFY) -def test_runner_subdir_split_sibling_py_aborts(mock_classify, tmp_path): - mock_classify.return_value = _make_classified() - # Create a sibling 'service.py' alongside the source file — the intended - # subdirectory 'service/' would shadow it. - (tmp_path / "service.py").write_text("# helper\n") - filepath = str(tmp_path / "test_service.py") - - source = "def test_foo():\n pass\n" - # Whole-file diff: ranges cover all 2 lines. - cfg = CrispenConfig(file_limiter_subdir_split=True) - result = run_file_limiter(filepath, source, source, [(1, 2)], cfg) - - assert result.abort is True - assert result.new_files == {} - assert any("shadow" in m for m in result.messages) - assert any("service/" in m for m in result.messages) - - -@patch(_PATCH_CLASSIFY) -def test_runner_subdir_split_disabled(mock_classify, tmp_path): - # file_limiter_subdir_split=False — subdir detection is skipped entirely. - mock_classify.return_value = ClassifiedEntities( - entities=[], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[], - set_3_groups=[], - abort=True, # force early abort so advise is not called - ) - filepath = str(tmp_path / "service.py") - source = "def foo():\n pass\n" - cfg = CrispenConfig(file_limiter_subdir_split=False) - result = run_file_limiter(filepath, source, source, [(1, 2)], cfg) - - # abort comes from classifier, not from subdir detection - assert result.abort is True - assert "already exists" not in " ".join(result.messages) - - -# --------------------------------------------------------------------------- -# run_file_limiter — subdir-split: non-test success (redirects __init__.py) -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_subdir_split_non_test_success(mock_classify, mock_advise, mock_gen): - # Whole-file diff on a non-test file → placements get subdir prefix, - # original_source is unchanged, and __init__.py carries the split content. - source = "def foo():\n pass\ndef bar():\n pass\n" - entity1 = _make_entity("foo", 1, 2) - entity2 = _make_entity("bar", 3, 4) - # Two groups required so the n_groups > 1 subdir guard doesn't fire. - mock_classify.return_value = ClassifiedEntities( - entities=[entity1, entity2], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[["foo"], ["bar"]], - set_3_groups=[], - abort=False, - ) - # LLM returns flat filenames (no subdir prefix yet). - mock_advise.return_value = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="helpers.py"), - ], - abort=False, - ) - mock_gen.return_value = SplitResult( - new_files={ - "service/utils.py": "def foo():\n pass", - "service/helpers.py": "def bar():\n pass", - }, - original_source="# init content\n", - abort=False, - ) - - cfg = CrispenConfig(file_limiter_subdir_split=True) - result = run_file_limiter("service.py", source, source, [(1, 4)], cfg) - - assert result.abort is False - # service/__init__.py carries the post-split original source. - assert "service/__init__.py" in result.new_files - assert result.new_files["service/__init__.py"] == "# init content\n" - # original_source is reset to the input (so service.py is not modified). - assert result.original_source == source - assert result.subdir_name == "service" - # The moved-message includes the prefixed target file. - assert any("service/utils.py" in m for m in result.messages) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_subdir_split_test_file_keeps_original( - mock_classify, mock_advise, mock_gen -): - # Whole-file diff on a test file → placements get subdir prefix but - # original_source (re-export stubs in test_service.py) is written back. - source = "def test_foo():\n pass\ndef test_bar():\n pass\n" - entity1 = _make_entity("test_foo", 1, 2) - entity2 = _make_entity("test_bar", 3, 4) - # Two groups required so the n_groups > 1 subdir guard doesn't fire. - mock_classify.return_value = ClassifiedEntities( - entities=[entity1, entity2], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[["test_foo"], ["test_bar"]], - set_3_groups=[], - abort=False, - ) - mock_advise.return_value = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["test_foo"], target_file="helpers.py"), - GroupPlacement(group=["test_bar"], target_file="extras.py"), - ], - abort=False, - ) - mock_gen.return_value = SplitResult( - new_files={ - "service/test_helpers.py": "def test_foo():\n pass", - "service/test_extras.py": "def test_bar():\n pass", - }, - original_source="# re-export stubs\n", - abort=False, - ) - - cfg = CrispenConfig(file_limiter_subdir_split=True) - result = run_file_limiter("tests/test_service.py", source, source, [(1, 4)], cfg) - - assert result.abort is False - # No __init__.py injected for test files. - assert "service/__init__.py" not in result.new_files - # original_source has the re-export stubs (NOT reset to input). - assert result.original_source == "# re-export stubs\n" - assert result.subdir_name == "service" - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_subdir_split_strips_test_prefix_from_stem( - mock_classify, mock_advise, mock_gen -): - # test_big.py → subdir "big/" (strip "test_" prefix from stem). - source = "def test_foo():\n pass\ndef test_bar():\n pass\n" - entity1 = _make_entity("test_foo", 1, 2) - entity2 = _make_entity("test_bar", 3, 4) - # Two groups required so the n_groups > 1 subdir guard doesn't fire. - mock_classify.return_value = ClassifiedEntities( - entities=[entity1, entity2], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[["test_foo"], ["test_bar"]], - set_3_groups=[], - abort=False, - ) - mock_advise.return_value = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["test_foo"], target_file="helpers.py"), - GroupPlacement(group=["test_bar"], target_file="extras.py"), - ], - abort=False, - ) - mock_gen.return_value = SplitResult( - new_files={ - "big/test_helpers.py": "def test_foo():\n pass", - "big/test_extras.py": "def test_bar():\n pass", - }, - original_source="# stubs\n", - abort=False, - ) - - cfg = CrispenConfig(file_limiter_subdir_split=True) - result = run_file_limiter("tests/test_big.py", source, source, [(1, 4)], cfg) - - assert result.abort is False - assert result.subdir_name == "big" - # "helpers.py" → test_ prefix → "test_helpers.py" → "big/test_helpers.py". - assert any("big/test_helpers.py" in m for m in result.messages) - - -# --------------------------------------------------------------------------- -# run_file_limiter — subdir-split: has_main keeps original and uses _lib suffix -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_subdir_split_has_main_uses_lib_suffix( - mock_classify, mock_advise, mock_gen, tmp_path -): - # Non-test file with __main__: subdir uses "_lib" suffix, original_source - # is the split content (re-export stubs + __main__), and has_main=True. - # No blank lines between entities so entity ranges don't pick up leading \n. - source = ( - "def foo():\n pass\n" - "def bar():\n pass\n" - "if __name__ == '__main__':\n foo()\n" - ) - entity1 = _make_entity("foo", 1, 2) - entity2 = _make_entity("bar", 3, 4) - mock_classify.return_value = ClassifiedEntities( - entities=[entity1, entity2], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[["foo"], ["bar"]], - set_3_groups=[], - abort=False, - ) - mock_advise.return_value = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="helpers.py"), - ], - abort=False, - ) - mock_gen.return_value = SplitResult( - new_files={ - "service_lib/utils.py": "def foo():\n pass", - "service_lib/helpers.py": "def bar():\n pass", - }, - original_source=( - "from service_lib.utils import foo\n\n" - "if __name__ == '__main__':\n foo()\n" - ), - abort=False, - ) - - cfg = CrispenConfig(file_limiter_subdir_split=True) - filepath = str(tmp_path / "service.py") - result = run_file_limiter(filepath, source, source, [(1, 6)], cfg) - - assert result.abort is False - assert result.has_main is True - assert result.subdir_name == "service_lib" - # original_source keeps the split content (re-exports + __main__), not reset. - assert "__main__" in result.original_source - # No __init__.py injected: original file stays as the entry point. - assert "service_lib/__init__.py" not in result.new_files - assert any("service_lib/utils.py" in m for m in result.messages) - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_subdir_split_has_main_fallback_suffix( - mock_classify, mock_advise, mock_gen, tmp_path -): - # When service_lib/ already exists, fall back to the next suffix (_helpers). - source = ( - "def foo():\n pass\n" - "def bar():\n pass\n" - "if __name__ == '__main__':\n foo()\n" - ) - (tmp_path / "service_lib").mkdir() - entity1 = _make_entity("foo", 1, 2) - entity2 = _make_entity("bar", 3, 4) - mock_classify.return_value = ClassifiedEntities( - entities=[entity1, entity2], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[["foo"], ["bar"]], - set_3_groups=[], - abort=False, - ) - mock_advise.return_value = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="helpers.py"), - ], - abort=False, - ) - mock_gen.return_value = SplitResult( - new_files={ - "service_helpers/utils.py": "def foo():\n pass", - "service_helpers/helpers.py": "def bar():\n pass", - }, - original_source="# stubs\n", - abort=False, - ) - - cfg = CrispenConfig(file_limiter_subdir_split=True) - filepath = str(tmp_path / "service.py") - result = run_file_limiter(filepath, source, source, [(1, 6)], cfg) - - assert result.abort is False - assert result.subdir_name == "service_helpers" - assert result.has_main is True - - -@patch(_PATCH_CLASSIFY) -def test_runner_subdir_split_has_main_all_suffixes_conflict_aborts( - mock_classify, tmp_path -): - # All _lib/_helpers/etc. directories already exist → abort with a clear message. - source = "def foo():\n pass\n\nif __name__ == '__main__':\n foo()\n" - for suffix in _MAIN_SUBDIR_SUFFIXES: - (tmp_path / f"service{suffix}").mkdir() - mock_classify.return_value = _make_classified() - - cfg = CrispenConfig(file_limiter_subdir_split=True) - filepath = str(tmp_path / "service.py") - result = run_file_limiter(filepath, source, source, [(1, 5)], cfg) - - assert result.abort is True - assert result.new_files == {} - assert any("__main__" in m for m in result.messages) - assert any("conflict" in m for m in result.messages) - - -# --------------------------------------------------------------------------- -# verbose=True paths -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_verbose_prints_to_stderr(mock_classify, mock_advise, mock_gen, capsys): - """verbose=True prints analysis/verification messages to stderr.""" - source = "def foo():\n pass\n" - entity = _make_entity("foo", 1, 2) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["foo"], "utils.py") - mock_gen.return_value = SplitResult( - new_files={"utils.py": "def foo():\n pass"}, - original_source="# original updated\n", - abort=False, - ) - - result = run_file_limiter("big.py", "", source, [], _CONFIG, verbose=True) - - assert result.abort is False - err = capsys.readouterr().err - assert "FileLimiter" in err - assert "big.py" in err - - -# --------------------------------------------------------------------------- -# Verification entity counting — TOP_LEVEL, CLASS, and empty-source branches -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_success_with_class_entity(mock_classify, mock_advise, mock_gen): - """Verification loop increments verified_classes for CLASS entities.""" - source = "class Foo:\n pass\n" - entity = Entity(EntityKind.CLASS, "Foo", 1, 2, ["Foo"]) - mock_classify.return_value = _make_classified(entities=[entity]) - mock_advise.return_value = _plan_with(["Foo"], "models.py") - mock_gen.return_value = SplitResult( - new_files={"models.py": "class Foo:\n pass"}, - original_source="# original\n", - abort=False, - ) - - result = run_file_limiter("big.py", "", source, [], _CONFIG) - - assert result.abort is False - assert result.verified_classes == 1 - assert result.verified_functions == 0 - assert result.verified_lines == 2 - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_success_with_top_level_entity(mock_classify, mock_advise, mock_gen): - """TOP_LEVEL entities are skipped in the verification count loop.""" - source = "import os\ndef foo():\n pass\n" - import_entity = Entity(EntityKind.TOP_LEVEL, "_block_0", 1, 1, ["os"]) - func_entity = _make_entity("foo", 2, 3) - mock_classify.return_value = _make_classified(entities=[import_entity, func_entity]) - mock_advise.return_value = _plan_with(["foo"], "utils.py") - mock_gen.return_value = SplitResult( - new_files={"utils.py": "def foo():\n pass"}, - original_source="import os\n", - abort=False, - ) - - result = run_file_limiter("big.py", "", source, [], _CONFIG) - - assert result.abort is False - # Only the function counts; TOP_LEVEL is skipped. - assert result.verified_functions == 1 - assert result.verified_classes == 0 - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_runner_success_with_empty_entity_source(mock_classify, mock_advise, mock_gen): - """Entities whose source is blank after rstrip are skipped in the count. - - Also covers verification of an entity that stays in the original file - (stays_entity is verified and counted alongside migrated entities). - """ - source = "def foo():\n pass\n\ndef bar():\n pass\n" - # blank_entity has empty source → skipped. foo migrated; bar stays in original. - blank_entity = _make_entity("_block_1", 3, 3) - func_entity = _make_entity("foo", 1, 2) - stays_entity = _make_entity("bar", 4, 5) - mock_classify.return_value = _make_classified( - entities=[func_entity, blank_entity, stays_entity] - ) - mock_advise.return_value = _plan_with(["foo"], "utils.py") - mock_gen.return_value = SplitResult( - new_files={"utils.py": "def foo():\n pass"}, - original_source="def bar():\n pass\n", - abort=False, - ) - - result = run_file_limiter("big.py", "", source, [], _CONFIG) - - assert result.abort is False - # blank_entity: empty source → skipped. bar: stays in original → not counted. - # Only foo (migrated) counts. - assert result.verified_functions == 1 - assert result.verified_lines == 2 - - -# --------------------------------------------------------------------------- -# run_file_limiter — __init__.py never gets a subdir split -# --------------------------------------------------------------------------- - - -@patch(_PATCH_CLASSIFY) -def test_runner_init_py_skips_subdir_split(mock_classify, tmp_path): - """__init__.py with a whole-file diff must not trigger subdir-split detection. - - A subdir split for __init__.py would create an ``__init__/`` subdirectory, - which is nonsensical. Instead it should fall through to the normal in-place - split (siblings in the same package directory). - """ - # Classify returns abort so the LLM path is skipped; we only care that - # subdir_name is NOT set on the result. - mock_classify.return_value = ClassifiedEntities( - entities=[], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[], - set_3_groups=[], - abort=True, - ) - filepath = str(tmp_path / "__init__.py") - # Make the source long enough to be a "whole-file diff". - source = "".join(f"def func_{i}():\n pass\n\n" for i in range(10)) - cfg = CrispenConfig(file_limiter_subdir_split=True) - result = run_file_limiter( - filepath, source, source, [(1, len(source.splitlines()))], cfg - ) - - # Abort comes from the classifier — subdir conflict detection was bypassed. - assert result.abort is True - assert result.subdir_name is None - assert "already exists" not in " ".join(result.messages) - - -# --------------------------------------------------------------------------- -# run_file_limiter — entity_to_target -# --------------------------------------------------------------------------- - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_entity_to_target_populated_on_success(mock_classify, mock_advise, mock_gen): - """On a successful run, entity_to_target maps entity names to target files.""" - source = "def foo():\n pass\ndef bar():\n pass\n" - entity_foo = _make_entity("foo", 1, 2) - entity_bar = _make_entity("bar", 3, 4) - mock_classify.return_value = _make_classified(entities=[entity_foo, entity_bar]) - # Plan: foo → utils.py, bar → helpers.py - from crispen.file_limiter.advisor import FileLimiterPlan, GroupPlacement - - mock_advise.return_value = FileLimiterPlan( - set3_migrate=[], - placements=[ - GroupPlacement(group=["foo"], target_file="utils.py"), - GroupPlacement(group=["bar"], target_file="helpers.py"), - ], - abort=False, - ) - mock_gen.return_value = SplitResult( - new_files={ - "utils.py": "def foo():\n pass", - "helpers.py": "def bar():\n pass", - }, - original_source="# original updated\n", - abort=False, - ) - - result = run_file_limiter("big.py", "", source, [], _CONFIG) - - assert result.abort is False - assert result.entity_to_target == { - "foo": "utils.py", - "bar": "helpers.py", - } - - -@patch(_PATCH_GEN) -@patch(_PATCH_ADVISE) -@patch(_PATCH_CLASSIFY) -def test_entity_to_target_empty_on_abort(mock_classify, mock_advise, mock_gen): - """Abort result has empty entity_to_target.""" - mock_classify.return_value = ClassifiedEntities( - entities=[], - entity_class={}, - graph={}, - set_1=[], - set_2_groups=[], - set_3_groups=[], - abort=True, - ) - - result = run_file_limiter("big.py", "", "x = 1\n", [], _CONFIG_NO_RETRY) - - assert result.abort is True - assert result.entity_to_target == {}