From 403eb7d5f8db1bf1dc44c5aec897a55ad2f7f59e Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Wed, 22 Jul 2026 15:51:42 +0700 Subject: [PATCH] automatically apply gitignore rules --- .gitignore | 3 +- README.md | 10 + codewiki/cli/adapters/doc_generator.py | 3 +- codewiki/cli/commands/config.py | 26 +- codewiki/cli/commands/generate.py | 15 +- codewiki/cli/config_manager.py | 7 +- codewiki/cli/models/config.py | 8 +- codewiki/mcp/server.py | 11 + codewiki/mcp/tools/analysis.py | 1 + .../analysis/analysis_service.py | 36 ++- .../analysis/repo_analyzer.py | 178 ++++++++++++- .../src/be/dependency_analyzer/ast_parser.py | 13 +- .../dependency_graphs_builder.py | 5 +- codewiki/src/config.py | 14 +- pyproject.toml | 2 +- tests/test_gitignore_filtering.py | 235 ++++++++++++++++++ 16 files changed, 531 insertions(+), 36 deletions(-) create mode 100644 tests/test_gitignore_filtering.py diff --git a/.gitignore b/.gitignore index 8529c8f5..8f5d45bf 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,8 @@ MANIFEST htmlcov/ .tox/ .hypothesis/ -tests/ +tests/* +!tests/test_gitignore_filtering.py # Jupyter *.ipynb diff --git a/README.md b/README.md index 6e6699db..60a9dc84 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,9 @@ codewiki generate --focus "src/core,src/api" --doc-type architecture # Add custom instructions for the AI agent codewiki generate --instructions "Focus on public APIs and include usage examples" + +# Analyze files ignored by Git (Git ignore filtering is enabled by default) +codewiki generate --no-gitignore ``` #### Pattern Behavior (Important!) @@ -238,6 +241,12 @@ codewiki generate --instructions "Focus on public APIs and include usage example - Glob patterns: `*.test.js`, `*_test.py`, `*.min.*` - Directory patterns: `build/`, `dist/`, `coverage/` +- **`--use-gitignore/--no-gitignore`**: Git ignore rules are applied by default + - Root and nested `.gitignore` files are respected before call-graph analysis + - Tracked files remain included, matching Git behavior + - Built-in and explicit `--exclude` patterns still apply when Git includes a path + - Use `codewiki config set --no-gitignore` to persistently disable this behavior + #### Setting Persistent Defaults Save your preferred settings as defaults: @@ -266,6 +275,7 @@ codewiki config agent --clear |--------|-------------|----------|---------| | `--include` | File patterns to include | **Replaces** defaults | `*.cs`, `*.py`, `src/**/*.ts` | | `--exclude` | Patterns to exclude | **Merges** with defaults | `Tests,Specs`, `*.test.js`, `build/` | +| `--use-gitignore/--no-gitignore` | Apply Git ignore rules | Enabled by default | `--no-gitignore` | | `--focus` | Modules to document in detail | Standalone option | `src/core,src/api` | | `--doc-type` | Documentation style | Standalone option | `api`, `architecture`, `user-guide`, `developer` | | `--instructions` | Custom agent instructions | Standalone option | Free-form text | diff --git a/codewiki/cli/adapters/doc_generator.py b/codewiki/cli/adapters/doc_generator.py index 05d605be..80790b3f 100644 --- a/codewiki/cli/adapters/doc_generator.py +++ b/codewiki/cli/adapters/doc_generator.py @@ -146,7 +146,8 @@ def generate(self) -> DocumentationJob: max_token_per_module=self.config.get('max_token_per_module', 36369), max_token_per_leaf_module=self.config.get('max_token_per_leaf_module', 16000), max_depth=self.config.get('max_depth', 2), - agent_instructions=self.config.get('agent_instructions') + agent_instructions=self.config.get('agent_instructions'), + use_gitignore=self.config.get('use_gitignore', True), ) # Run backend documentation generation diff --git a/codewiki/cli/commands/config.py b/codewiki/cli/commands/config.py index 91b47ffa..1903379b 100644 --- a/codewiki/cli/commands/config.py +++ b/codewiki/cli/commands/config.py @@ -111,6 +111,11 @@ def config_group(): type=str, help="Azure OpenAI deployment name" ) +@click.option( + "--use-gitignore/--no-gitignore", + default=None, + help="Apply Git ignore rules during repository analysis (default: enabled)", +) def config_set( api_key: Optional[str], base_url: Optional[str], @@ -124,7 +129,8 @@ def config_set( provider: Optional[str] = None, aws_region: Optional[str] = None, api_version: Optional[str] = None, - azure_deployment: Optional[str] = None + azure_deployment: Optional[str] = None, + use_gitignore: Optional[bool] = None, ): """ Set configuration values for CodeWiki. @@ -170,10 +176,14 @@ def config_set( \b # Set max depth for hierarchical decomposition $ codewiki config set --max-depth 3 + + \b + # Persistently disable Git ignore filtering + $ codewiki config set --no-gitignore """ try: # Check if at least one option is provided - if not any([api_key, base_url, main_model, cluster_model, fallback_model, max_tokens, max_token_per_module, max_token_per_leaf_module, max_depth, provider, aws_region, api_version, azure_deployment]): + if not any([api_key, base_url, main_model, cluster_model, fallback_model, max_tokens, max_token_per_module, max_token_per_leaf_module, max_depth, provider, aws_region, api_version, azure_deployment, use_gitignore is not None]): click.echo("No options provided. Use --help for usage information.") sys.exit(EXIT_CONFIG_ERROR) @@ -237,6 +247,9 @@ def config_set( if azure_deployment is not None: validated_data['azure_deployment'] = azure_deployment + if use_gitignore is not None: + validated_data['use_gitignore'] = use_gitignore + # Create config manager and save manager = ConfigManager() manager.load() # Load existing config if present @@ -254,7 +267,8 @@ def config_set( provider=validated_data.get('provider'), aws_region=validated_data.get('aws_region'), api_version=validated_data.get('api_version'), - azure_deployment=validated_data.get('azure_deployment') + azure_deployment=validated_data.get('azure_deployment'), + use_gitignore=validated_data.get('use_gitignore'), ) # Display success messages @@ -315,6 +329,9 @@ def config_set( if azure_deployment: click.secho(f"✓ Azure Deployment: {azure_deployment}", fg="green") + if use_gitignore is not None: + click.secho(f"✓ Use gitignore: {use_gitignore}", fg="green") + click.echo("\n" + click.style("Configuration updated successfully.", fg="green", bold=True)) except ConfigurationError as e: @@ -375,6 +392,7 @@ def config_show(output_json: bool): "max_token_per_module": config.max_token_per_module if config else 36369, "max_token_per_leaf_module": config.max_token_per_leaf_module if config else 16000, "max_depth": config.max_depth if config else 2, + "use_gitignore": config.use_gitignore if config else True, "agent_instructions": config.agent_instructions.to_dict() if config and config.agent_instructions else {}, "config_file": str(manager.config_file_path) } @@ -435,6 +453,7 @@ def config_show(output_json: bool): click.secho("Decomposition Settings", fg="cyan", bold=True) if config: click.echo(f" Max Depth: {config.max_depth}") + click.echo(f" Use Gitignore: {config.use_gitignore}") click.echo() click.secho("Agent Instructions", fg="cyan", bold=True) @@ -859,4 +878,3 @@ def config_agent( sys.exit(e.exit_code) except Exception as e: sys.exit(handle_error(e)) - diff --git a/codewiki/cli/commands/generate.py b/codewiki/cli/commands/generate.py index 540ab310..ccc75f86 100644 --- a/codewiki/cli/commands/generate.py +++ b/codewiki/cli/commands/generate.py @@ -266,6 +266,11 @@ def _find_affected(tree, parent_names=None): default=None, help="Custom instructions for the documentation agent", ) +@click.option( + "--use-gitignore/--no-gitignore", + default=None, + help="Apply Git ignore rules during analysis (default: enabled)", +) @click.option( "--verbose", "-v", @@ -319,6 +324,7 @@ def generate_command( focus: Optional[str], doc_type: Optional[str], instructions: Optional[str], + use_gitignore: Optional[bool], verbose: bool, max_tokens: Optional[int], max_token_per_module: Optional[int], @@ -346,6 +352,10 @@ def generate_command( \b # Force full regeneration $ codewiki generate --no-cache + + \b + # Analyze ignored files as well + $ codewiki generate --no-gitignore \b # C# project: only .cs files, exclude tests @@ -521,10 +531,12 @@ def generate_command( effective_max_token_per_module = max_token_per_module if max_token_per_module is not None else config.max_token_per_module effective_max_token_per_leaf = max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module effective_max_depth = max_depth if max_depth is not None else config.max_depth + effective_use_gitignore = use_gitignore if use_gitignore is not None else config.use_gitignore logger.debug(f"Max tokens: {effective_max_tokens}") logger.debug(f"Max token/module: {effective_max_token_per_module}") logger.debug(f"Max token/leaf module: {effective_max_token_per_leaf}") logger.debug(f"Max depth: {effective_max_depth}") + logger.debug(f"Use gitignore: {effective_use_gitignore}") # Get agent instructions (merge runtime with persistent) agent_instructions_dict = None @@ -562,6 +574,8 @@ def generate_command( 'max_token_per_leaf_module': max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module, # Max depth setting (runtime override takes precedence) 'max_depth': max_depth if max_depth is not None else config.max_depth, + # Gitignore setting (runtime override takes precedence) + 'use_gitignore': use_gitignore if use_gitignore is not None else config.use_gitignore, }, verbose=verbose, generate_html=github_pages, @@ -629,4 +643,3 @@ def generate_command( sys.exit(130) except Exception as e: sys.exit(handle_error(e, verbose=verbose)) - diff --git a/codewiki/cli/config_manager.py b/codewiki/cli/config_manager.py index c858a4e3..5b62a5bb 100644 --- a/codewiki/cli/config_manager.py +++ b/codewiki/cli/config_manager.py @@ -135,7 +135,8 @@ def save( provider: Optional[str] = None, aws_region: Optional[str] = None, api_version: Optional[str] = None, - azure_deployment: Optional[str] = None + azure_deployment: Optional[str] = None, + use_gitignore: Optional[bool] = None, ): """ Save configuration to file and keyring. @@ -155,6 +156,7 @@ def save( aws_region: AWS region for Bedrock provider api_version: Azure OpenAI API version azure_deployment: Azure OpenAI deployment name + use_gitignore: Apply Git ignore rules during repository analysis """ # Ensure config directory exists try: @@ -204,6 +206,8 @@ def save( self._config.api_version = api_version if azure_deployment is not None: self._config.azure_deployment = azure_deployment + if use_gitignore is not None: + self._config.use_gitignore = use_gitignore # Validate configuration whenever the minimum required fields are set. # Caw providers only need main_model; API providers need base_url + @@ -330,4 +334,3 @@ def keyring_available(self) -> bool: def config_file_path(self) -> Path: """Get configuration file path.""" return CONFIG_FILE - diff --git a/codewiki/cli/models/config.py b/codewiki/cli/models/config.py index 2aed7604..081260d1 100644 --- a/codewiki/cli/models/config.py +++ b/codewiki/cli/models/config.py @@ -121,6 +121,7 @@ class Configuration: max_token_per_module: Maximum tokens per module for clustering (default: 36369) max_token_per_leaf_module: Maximum tokens per leaf module (default: 16000) max_depth: Maximum depth for hierarchical decomposition (default: 2) + use_gitignore: Apply Git ignore rules during repository analysis agent_instructions: Custom agent instructions for documentation generation """ base_url: str @@ -136,6 +137,7 @@ class Configuration: max_token_per_module: int = 36369 max_token_per_leaf_module: int = 16000 max_depth: int = 2 + use_gitignore: bool = True agent_instructions: AgentInstructions = field(default_factory=AgentInstructions) def validate(self): @@ -172,6 +174,7 @@ def to_dict(self) -> dict: 'max_token_per_module': self.max_token_per_module, 'max_token_per_leaf_module': self.max_token_per_leaf_module, 'max_depth': self.max_depth, + 'use_gitignore': self.use_gitignore, 'fallback_model': self.fallback_model, } if self.agent_instructions and not self.agent_instructions.is_empty(): @@ -207,6 +210,7 @@ def from_dict(cls, data: dict) -> 'Configuration': max_token_per_module=data.get('max_token_per_module', 36369), max_token_per_leaf_module=data.get('max_token_per_leaf_module', 16000), max_depth=data.get('max_depth', 2), + use_gitignore=data.get('use_gitignore', True), agent_instructions=agent_instructions, ) @@ -273,6 +277,6 @@ def to_backend_config(self, repo_path: str, output_dir: str, api_key: str, runti max_token_per_module=self.max_token_per_module, max_token_per_leaf_module=self.max_token_per_leaf_module, max_depth=self.max_depth, - agent_instructions=final_instructions.to_dict() if final_instructions else None + agent_instructions=final_instructions.to_dict() if final_instructions else None, + use_gitignore=self.use_gitignore, ) - diff --git a/codewiki/mcp/server.py b/codewiki/mcp/server.py index a69355aa..5ae390ab 100644 --- a/codewiki/mcp/server.py +++ b/codewiki/mcp/server.py @@ -102,6 +102,11 @@ def _fine_grained_tools() -> list[Tool]: "type": "string", "description": "Comma-separated patterns to exclude (e.g., '*test*,*spec*')", }, + "use_gitignore": { + "type": "boolean", + "description": "Apply Git ignore rules before analysis (default: true)", + "default": True, + }, }, "required": ["repo_path"], }, @@ -320,6 +325,11 @@ def _legacy_tools() -> list[Tool]: "type": "string", "description": "Comma-separated patterns to exclude", }, + "use_gitignore": { + "type": "boolean", + "description": "Apply Git ignore rules before analysis (default: true)", + "default": True, + }, }, "required": ["repo_path"], }, @@ -489,6 +499,7 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]: aws_region=getattr(config, "aws_region", "us-east-1"), max_tokens=config.max_tokens, agent_instructions=agent_instructions or None, + use_gitignore=arguments.get("use_gitignore", True), ) from codewiki.cli.utils.repo_validator import get_git_commit_hash diff --git a/codewiki/mcp/tools/analysis.py b/codewiki/mcp/tools/analysis.py index 75613c57..70a1eb6b 100644 --- a/codewiki/mcp/tools/analysis.py +++ b/codewiki/mcp/tools/analysis.py @@ -304,6 +304,7 @@ def handle_analyze_repo( llm_api_key="not-needed", main_model="unused", cluster_model="unused", + use_gitignore=arguments.get("use_gitignore", True), ) # Apply optional include/exclude patterns diff --git a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py index c9cf5bb6..0e31db3d 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py +++ b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py @@ -42,7 +42,8 @@ def analyze_local_repository( self, repo_path: str, max_files: int = 100, - languages: Optional[List[str]] = None + languages: Optional[List[str]] = None, + use_gitignore: bool = True, ) -> Dict[str, Any]: """ Analyze a local repository folder. @@ -51,6 +52,7 @@ def analyze_local_repository( repo_path: Path to local repository folder max_files: Maximum number of files to analyze languages: List of languages to include (e.g., ['python', 'javascript']) + use_gitignore: Whether to apply Git ignore rules Returns: Dict with analysis results including nodes and relationships @@ -59,7 +61,7 @@ def analyze_local_repository( logger.debug(f"Analyzing local repository at {repo_path}") # Get repo analyzer to find files - repo_analyzer = RepoAnalyzer() + repo_analyzer = RepoAnalyzer(use_gitignore=use_gitignore) structure_result = repo_analyzer.analyze_repository_structure(repo_path) # Extract code files @@ -98,6 +100,7 @@ def analyze_repository_full( github_url: str, include_patterns: Optional[List[str]] = None, exclude_patterns: Optional[List[str]] = None, + use_gitignore: bool = True, ) -> AnalysisResult: """ Perform complete repository analysis including call graph generation. @@ -106,6 +109,7 @@ def analyze_repository_full( github_url: GitHub repository URL to analyze include_patterns: File patterns to include (e.g., ['*.py', '*.js']) exclude_patterns: Additional patterns to exclude + use_gitignore: Whether to apply Git ignore rules Returns: AnalysisResult: Complete analysis with functions, relationships, and visualization @@ -122,7 +126,9 @@ def analyze_repository_full( repo_info = self._parse_repository_info(github_url) logger.debug("Analyzing repository file structure...") - structure_result = self._analyze_structure(temp_dir, include_patterns, exclude_patterns) + structure_result = self._analyze_structure( + temp_dir, include_patterns, exclude_patterns, use_gitignore + ) logger.debug(f"Found {structure_result['summary']['total_files']} files to analyze.") logger.debug("Starting call graph analysis...") @@ -172,6 +178,7 @@ def analyze_repository_structure_only( github_url: str, include_patterns: Optional[List[str]] = None, exclude_patterns: Optional[List[str]] = None, + use_gitignore: bool = True, ) -> Dict[str, Any]: """ Perform lightweight structure-only analysis without call graph generation. @@ -180,6 +187,7 @@ def analyze_repository_structure_only( github_url: GitHub repository URL to analyze include_patterns: File patterns to include exclude_patterns: Additional patterns to exclude + use_gitignore: Whether to apply Git ignore rules Returns: Dict: Repository structure with file tree and summary statistics @@ -191,7 +199,9 @@ def analyze_repository_structure_only( temp_dir = self._clone_repository(github_url) repo_info = self._parse_repository_info(github_url) - structure_result = self._analyze_structure(temp_dir, include_patterns, exclude_patterns) + structure_result = self._analyze_structure( + temp_dir, include_patterns, exclude_patterns, use_gitignore + ) result = { "repository": repo_info, @@ -233,12 +243,16 @@ def _analyze_structure( repo_dir: str, include_patterns: Optional[List[str]], exclude_patterns: Optional[List[str]], + use_gitignore: bool = True, ) -> Dict[str, Any]: """Analyze repository file structure with filtering.""" logger.debug( - f"Initializing RepoAnalyzer with include: {include_patterns}, exclude: {exclude_patterns}" + "Initializing RepoAnalyzer with include: %s, exclude: %s, use_gitignore: %s", + include_patterns, + exclude_patterns, + use_gitignore, ) - repo_analyzer = RepoAnalyzer(include_patterns, exclude_patterns) + repo_analyzer = RepoAnalyzer(include_patterns, exclude_patterns, use_gitignore) return repo_analyzer.analyze_repository_structure(repo_dir) def _read_readme_file(self, repo_dir: str) -> Optional[str]: @@ -341,7 +355,7 @@ def __del__(self): def analyze_repository( - github_url: str, include_patterns=None, exclude_patterns=None + github_url: str, include_patterns=None, exclude_patterns=None, use_gitignore=True ) -> tuple[AnalysisResult, None]: """ Backward compatibility function. @@ -350,12 +364,14 @@ def analyze_repository( tuple: (AnalysisResult, None) - None instead of temp_dir since cleanup is handled internally """ service = AnalysisService() - result = service.analyze_repository_full(github_url, include_patterns, exclude_patterns) + result = service.analyze_repository_full( + github_url, include_patterns, exclude_patterns, use_gitignore + ) return result, None def analyze_repository_structure_only( - github_url: str, include_patterns=None, exclude_patterns=None + github_url: str, include_patterns=None, exclude_patterns=None, use_gitignore=True ) -> tuple[Dict, None]: """ Backward compatibility function. @@ -365,6 +381,6 @@ def analyze_repository_structure_only( """ service = AnalysisService() result = service.analyze_repository_structure_only( - github_url, include_patterns, exclude_patterns + github_url, include_patterns, exclude_patterns, use_gitignore ) return result, None diff --git a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py index 458ac14c..aa13c9c4 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py @@ -5,12 +5,172 @@ detailed file tree representations with filtering capabilities. """ -import os import fnmatch -import json +import logging +import shutil +import subprocess from pathlib import Path -from typing import Dict, List, Optional, Union -from codewiki.src.be.dependency_analyzer.utils.patterns import DEFAULT_IGNORE_PATTERNS, DEFAULT_INCLUDE_PATTERNS +from typing import Dict, List, Optional + +from pathspec import GitIgnoreSpec + +from codewiki.src.be.dependency_analyzer.utils.patterns import ( + DEFAULT_IGNORE_PATTERNS, + DEFAULT_INCLUDE_PATTERNS, +) + + +logger = logging.getLogger(__name__) + + +class GitIgnoreFilter: + """Evaluate Git ignore rules once per repository analysis. + + Git repositories use ``git ls-files`` so nested ignore files, repository + excludes, global excludes, and tracked-file semantics exactly match Git. + Plain directories fall back to pathspec-backed ``.gitignore`` evaluation. + """ + + def __init__(self, repo_dir: Path) -> None: + self.repo_dir = repo_dir.resolve() + self._ignored_files: set[str] = set() + self._ignored_dirs: set[str] = set() + self._ignore_all_untracked = False + self._fallback_specs: list[tuple[str, GitIgnoreSpec]] = [] + self._using_git = self._load_git_ignored_paths() + if not self._using_git: + self._load_fallback_specs() + + def _load_git_ignored_paths(self) -> bool: + git_path = shutil.which("git") + if not git_path: + logger.debug("Git is unavailable; falling back to direct .gitignore parsing") + return False + + try: + root_result = subprocess.run( + [git_path, "-C", str(self.repo_dir), "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + timeout=10, + ) + if root_result.returncode != 0: + logger.debug( + "%s is not in a Git worktree; using direct .gitignore parsing", + self.repo_dir, + ) + return False + worktree_root = Path(root_result.stdout.strip()).resolve() + scope = self.repo_dir.relative_to(worktree_root).as_posix() + scope = scope if scope != "." else "." + + ignored_result = subprocess.run( + [ + git_path, + "-C", + str(worktree_root), + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + "--directory", + "-z", + "--", + scope, + ], + check=True, + capture_output=True, + timeout=30, + ) + except (OSError, ValueError, subprocess.SubprocessError) as exc: + logger.warning( + "Could not query Git ignore rules for %s; falling back to direct parsing: %s", + self.repo_dir, + exc, + ) + return False + + scope_prefix = "" if scope == "." else f"{scope.rstrip('/')}/" + for raw_path in ignored_result.stdout.split(b"\0"): + if not raw_path: + continue + repo_relative = raw_path.decode("utf-8", errors="surrogateescape").replace("\\", "/") + if scope_prefix: + if not repo_relative.startswith(scope_prefix): + continue + relative = repo_relative[len(scope_prefix) :] + else: + relative = repo_relative + + if not relative and repo_relative.endswith("/"): + self._ignore_all_untracked = True + elif relative.endswith("/"): + self._ignored_dirs.add(relative.rstrip("/")) + else: + self._ignored_files.add(relative) + + return True + + def _load_fallback_specs(self) -> None: + try: + candidates = sorted( + ( + path + for path in self.repo_dir.rglob(".gitignore") + if path.is_file() and not path.is_symlink() + ), + key=lambda path: (len(path.relative_to(self.repo_dir).parts), path.as_posix()), + ) + except OSError as exc: + logger.warning("Could not discover .gitignore files under %s: %s", self.repo_dir, exc) + return + + for gitignore_path in candidates: + try: + lines = gitignore_path.read_text(encoding="utf-8").splitlines() + spec = GitIgnoreSpec.from_lines(lines) + base = gitignore_path.parent.relative_to(self.repo_dir).as_posix() + self._fallback_specs.append(("" if base == "." else base, spec)) + except (OSError, UnicodeError, ValueError) as exc: + logger.warning("Skipping unreadable .gitignore at %s: %s", gitignore_path, exc) + + def is_ignored(self, relative_path: str, is_dir: bool) -> bool: + """Return whether a repository-relative path should be ignored.""" + normalized = relative_path.replace("\\", "/") + if normalized.startswith("./"): + normalized = normalized[2:] + if normalized in ("", "."): + return False + + if self._using_git: + if self._ignore_all_untracked: + return True + if normalized in self._ignored_files or normalized in self._ignored_dirs: + return True + return any( + normalized.startswith(f"{ignored_dir}/") for ignored_dir in self._ignored_dirs + ) + + ignored = False + for base, spec in self._fallback_specs: + if base: + if normalized == base: + local_path = "" + elif normalized.startswith(f"{base}/"): + local_path = normalized[len(base) + 1 :] + else: + continue + else: + local_path = normalized + + if not local_path: + continue + candidate = f"{local_path}/" if is_dir else local_path + result = spec.check_file(candidate) + if result.include is not None: + ignored = bool(result.include) + + return ignored class RepoAnalyzer: @@ -18,6 +178,7 @@ def __init__( self, include_patterns: Optional[List[str]] = None, exclude_patterns: Optional[List[str]] = None, + use_gitignore: bool = True, ) -> None: # Include patterns: if specified, use ONLY those patterns (replaces defaults) self.include_patterns = ( @@ -29,8 +190,11 @@ def __init__( if exclude_patterns is not None else list(DEFAULT_IGNORE_PATTERNS) ) + self.use_gitignore = use_gitignore + self._gitignore_filter: Optional[GitIgnoreFilter] = None def analyze_repository_structure(self, repo_dir: str) -> Dict: + self._gitignore_filter = GitIgnoreFilter(Path(repo_dir)) if self.use_gitignore else None file_tree = self._build_file_tree(repo_dir) return { "file_tree": file_tree, @@ -57,7 +221,7 @@ def build_tree(path: Path, base_path: Path) -> Optional[Dict]: if not str(path.resolve()).startswith(str(base_path.resolve())): return None - if self._should_exclude_path(relative_path_str, path.name): + if self._should_exclude_path(relative_path_str, path.name, path.is_dir()): return None if path.is_file(): @@ -97,7 +261,7 @@ def build_tree(path: Path, base_path: Path) -> Optional[Dict]: return build_tree(Path(repo_dir), Path(repo_dir)) - def _should_exclude_path(self, path: str, filename: str) -> bool: + def _should_exclude_path(self, path: str, filename: str, is_dir: bool = False) -> bool: for pattern in self.exclude_patterns: if fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(filename, pattern): return True @@ -107,6 +271,8 @@ def _should_exclude_path(self, path: str, filename: str) -> bool: return True if pattern in path.split("/"): return True + if self._gitignore_filter and self._gitignore_filter.is_ignored(path, is_dir): + return True return False def _should_include_file(self, path: str, filename: str) -> bool: diff --git a/codewiki/src/be/dependency_analyzer/ast_parser.py b/codewiki/src/be/dependency_analyzer/ast_parser.py index 4c50ea59..1a402582 100644 --- a/codewiki/src/be/dependency_analyzer/ast_parser.py +++ b/codewiki/src/be/dependency_analyzer/ast_parser.py @@ -18,7 +18,13 @@ class DependencyParser: """Parser for extracting code components from multi-language repositories.""" - def __init__(self, repo_path: str, include_patterns: List[str] = None, exclude_patterns: List[str] = None): + def __init__( + self, + repo_path: str, + include_patterns: List[str] = None, + exclude_patterns: List[str] = None, + use_gitignore: bool = True, + ): """ Initialize the dependency parser. @@ -26,12 +32,14 @@ def __init__(self, repo_path: str, include_patterns: List[str] = None, exclude_p repo_path: Path to the repository include_patterns: File patterns to include (e.g., ["*.cs", "*.py"]) exclude_patterns: File/directory patterns to exclude (e.g., ["*Tests*"]) + use_gitignore: Whether to apply Git ignore rules """ self.repo_path = os.path.abspath(repo_path) self.components: Dict[str, Node] = {} self.modules: Set[str] = set() self.include_patterns = include_patterns self.exclude_patterns = exclude_patterns + self.use_gitignore = use_gitignore self.analysis_service = AnalysisService() @@ -47,7 +55,8 @@ def parse_repository(self, filtered_folders: List[str] = None) -> Dict[str, Node structure_result = self.analysis_service._analyze_structure( self.repo_path, include_patterns=self.include_patterns, - exclude_patterns=self.exclude_patterns + exclude_patterns=self.exclude_patterns, + use_gitignore=self.use_gitignore, ) call_graph_result = self.analysis_service._analyze_call_graph( diff --git a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py index 06990bd7..a1186e6e 100644 --- a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py +++ b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py @@ -45,7 +45,8 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: parser = DependencyParser( self.config.repo_path, include_patterns=include_patterns, - exclude_patterns=exclude_patterns + exclude_patterns=exclude_patterns, + use_gitignore=self.config.use_gitignore, ) filtered_folders = None @@ -76,4 +77,4 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: valid_types = compute_valid_leaf_types(components) keep_leaf_nodes = filter_leaf_nodes(leaf_nodes, components, valid_types) - return components, keep_leaf_nodes \ No newline at end of file + return components, keep_leaf_nodes diff --git a/codewiki/src/config.py b/codewiki/src/config.py index cd37cdc3..5bdc2398 100644 --- a/codewiki/src/config.py +++ b/codewiki/src/config.py @@ -72,6 +72,8 @@ class Config: max_token_per_leaf_module: int = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE # Agent instructions for customization agent_instructions: Optional[Dict[str, Any]] = None + # Apply Git ignore rules before dependency analysis + use_gitignore: bool = True @property def include_patterns(self) -> Optional[List[str]]: @@ -151,7 +153,8 @@ def from_args(cls, args: argparse.Namespace) -> 'Config': llm_api_key=LLM_API_KEY, main_model=MAIN_MODEL, cluster_model=CLUSTER_MODEL, - fallback_model=FALLBACK_MODEL_1 + fallback_model=FALLBACK_MODEL_1, + use_gitignore=getattr(args, "use_gitignore", True), ) @classmethod @@ -172,7 +175,8 @@ def from_cli( max_token_per_module: int = DEFAULT_MAX_TOKEN_PER_MODULE, max_token_per_leaf_module: int = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE, max_depth: int = MAX_DEPTH, - agent_instructions: Optional[Dict[str, Any]] = None + agent_instructions: Optional[Dict[str, Any]] = None, + use_gitignore: bool = True, ) -> 'Config': """ Create configuration for CLI context. @@ -194,6 +198,7 @@ def from_cli( max_token_per_leaf_module: Maximum tokens per leaf module max_depth: Maximum depth for hierarchical decomposition agent_instructions: Custom agent instructions dict + use_gitignore: Whether to apply Git ignore rules Returns: Config instance @@ -219,5 +224,6 @@ def from_cli( max_tokens=max_tokens, max_token_per_module=max_token_per_module, max_token_per_leaf_module=max_token_per_leaf_module, - agent_instructions=agent_instructions - ) \ No newline at end of file + agent_instructions=agent_instructions, + use_gitignore=use_gitignore, + ) diff --git a/pyproject.toml b/pyproject.toml index 9e000aab..25a0187c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dependencies = [ "pydantic-ai>=1.0.6", "requests>=2.32.4", "python-dotenv>=1.1.1", + "pathspec>=0.12.1", "rich>=14.1.0", "networkx>=3.5", "psutil>=7.0.0", @@ -132,4 +133,3 @@ python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] addopts = "-v --cov=codewiki --cov-report=term-missing" - diff --git a/tests/test_gitignore_filtering.py b/tests/test_gitignore_filtering.py new file mode 100644 index 00000000..120ba128 --- /dev/null +++ b/tests/test_gitignore_filtering.py @@ -0,0 +1,235 @@ +"""Tests for automatic Git ignore filtering (issue #71).""" + +import asyncio +import shutil +import subprocess +from pathlib import Path + +import pytest +from click.testing import CliRunner + +import codewiki.cli.config_manager as config_manager_module +from codewiki.cli.main import cli +from codewiki.cli.config_manager import ConfigManager +from codewiki.cli.models.config import Configuration +from codewiki.mcp.server import list_tools +from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser +from codewiki.src.be.dependency_analyzer.analysis.repo_analyzer import RepoAnalyzer + + +def _write_source(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("def example():\n return True\n", encoding="utf-8") + + +def _tree_paths(tree: dict) -> set[str]: + paths: set[str] = set() + + def visit(node: dict) -> None: + if node["type"] == "file": + paths.add(node["path"]) + for child in node.get("children", []): + visit(child) + + visit(tree) + return paths + + +def _analyzed_paths(repo: Path, **kwargs) -> set[str]: + result = RepoAnalyzer(include_patterns=["*.py"], **kwargs).analyze_repository_structure( + str(repo) + ) + return _tree_paths(result["file_tree"]) + + +def _git(repo: Path, *args: str) -> None: + subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + ) + + +@pytest.mark.skipif(shutil.which("git") is None, reason="Git is required") +def test_gitignore_is_default_and_keeps_tracked_matches(tmp_path: Path) -> None: + _git(tmp_path, "init", "-q") + (tmp_path / ".gitignore").write_text( + "ignored.py\nignored_dir/\n*.generated.py\n!keep.generated.py\ntracked.py\n", + encoding="utf-8", + ) + (tmp_path / "nested").mkdir() + (tmp_path / "nested" / ".gitignore").write_text( + "secret.py\n!visible.py\n", + encoding="utf-8", + ) + + for relative in ( + "ignored.py", + "ignored_dir/child.py", + "drop.generated.py", + "keep.generated.py", + "kept.py", + "nested/secret.py", + "nested/visible.py", + "tracked.py", + ): + _write_source(tmp_path / relative) + + _git(tmp_path, "add", "-f", "tracked.py") + + assert _analyzed_paths(tmp_path) == { + "keep.generated.py", + "kept.py", + "nested/visible.py", + "tracked.py", + } + + +@pytest.mark.skipif(shutil.which("git") is None, reason="Git is required") +def test_gitignore_can_be_disabled_without_disabling_explicit_excludes(tmp_path: Path) -> None: + _git(tmp_path, "init", "-q") + (tmp_path / ".gitignore").write_text("ignored.py\n", encoding="utf-8") + _write_source(tmp_path / "ignored.py") + _write_source(tmp_path / "explicit.py") + + assert _analyzed_paths( + tmp_path, + use_gitignore=False, + exclude_patterns=["explicit.py"], + ) == {"ignored.py"} + + +@pytest.mark.skipif(shutil.which("git") is None, reason="Git is required") +def test_monorepo_subdirectory_honors_parent_gitignore(tmp_path: Path) -> None: + _git(tmp_path, "init", "-q") + app = tmp_path / "packages" / "app" + _write_source(app / "ignored.py") + _write_source(app / "kept.py") + (tmp_path / ".gitignore").write_text( + "packages/app/ignored.py\n", + encoding="utf-8", + ) + + assert _analyzed_paths(app) == {"kept.py"} + + +@pytest.mark.skipif(shutil.which("git") is None, reason="Git is required") +def test_wholly_ignored_monorepo_subdirectory_is_empty(tmp_path: Path) -> None: + _git(tmp_path, "init", "-q") + app = tmp_path / "packages" / "app" + _write_source(app / "ignored.py") + (tmp_path / ".gitignore").write_text("packages/app/\n", encoding="utf-8") + + assert _analyzed_paths(app) == set() + + _write_source(app / "tracked.py") + _git(tmp_path, "add", "-f", "packages/app/tracked.py") + assert _analyzed_paths(app) == {"tracked.py"} + + +@pytest.mark.skipif(shutil.which("git") is None, reason="Git is required") +def test_ignored_sources_do_not_reach_dependency_graph(tmp_path: Path) -> None: + _git(tmp_path, "init", "-q") + (tmp_path / ".gitignore").write_text("ignored.py\n", encoding="utf-8") + _write_source(tmp_path / "ignored.py") + _write_source(tmp_path / "kept.py") + + components = DependencyParser(str(tmp_path)).parse_repository() + + assert {node.relative_path for node in components.values()} == {"kept.py"} + + +def test_non_git_fallback_supports_nested_rules_and_negation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "codewiki.src.be.dependency_analyzer.analysis.repo_analyzer.shutil.which", + lambda _name: None, + ) + (tmp_path / ".gitignore").write_text( + "ignored.py\n*.generated.py\n!keep.generated.py\n", + encoding="utf-8", + ) + (tmp_path / "nested").mkdir() + (tmp_path / "nested" / ".gitignore").write_text( + "secret.py\n!visible.py\n", + encoding="utf-8", + ) + for relative in ( + "ignored.py", + "drop.generated.py", + "keep.generated.py", + "kept.py", + "nested/secret.py", + "nested/visible.py", + ): + _write_source(tmp_path / relative) + + assert _analyzed_paths(tmp_path) == { + "keep.generated.py", + "kept.py", + "nested/visible.py", + } + + +def test_unreadable_fallback_gitignore_does_not_abort_analysis( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "codewiki.src.be.dependency_analyzer.analysis.repo_analyzer.shutil.which", + lambda _name: None, + ) + (tmp_path / ".gitignore").write_bytes(b"\xff\xfe") + _write_source(tmp_path / "kept.py") + + assert _analyzed_paths(tmp_path) == {"kept.py"} + + +def test_configuration_defaults_and_round_trips_gitignore_setting() -> None: + old_config = Configuration.from_dict( + {"base_url": "https://example.com", "main_model": "main", "cluster_model": "cluster"} + ) + assert old_config.use_gitignore is True + + old_config.use_gitignore = False + serialized = old_config.to_dict() + assert serialized["use_gitignore"] is False + assert Configuration.from_dict(serialized).use_gitignore is False + + +def test_config_manager_persists_disabled_gitignore( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("CODEWIKI_NO_KEYRING", "1") + monkeypatch.setattr(config_manager_module, "CONFIG_DIR", tmp_path) + monkeypatch.setattr(config_manager_module, "CONFIG_FILE", tmp_path / "config.json") + monkeypatch.setattr(config_manager_module, "CREDENTIALS_FILE", tmp_path / "credentials.json") + + ConfigManager().save(use_gitignore=False) + loaded = ConfigManager() + + assert loaded.load() is True + assert loaded.get_config().use_gitignore is False + + +def test_cli_exposes_runtime_and_persistent_gitignore_switches() -> None: + runner = CliRunner() + generate_help = runner.invoke(cli, ["generate", "--help"]) + config_help = runner.invoke(cli, ["config", "set", "--help"]) + + assert generate_help.exit_code == 0 + assert "--use-gitignore / --no-gitignore" in generate_help.output + assert config_help.exit_code == 0 + assert "--use-gitignore / --no-gitignore" in config_help.output + + +def test_mcp_analysis_tools_default_to_gitignore_enabled() -> None: + tools = {tool.name: tool for tool in asyncio.run(list_tools())} + + for tool_name in ("analyze_repo", "generate_docs"): + schema = tools[tool_name].inputSchema["properties"]["use_gitignore"] + assert schema == { + "type": "boolean", + "description": "Apply Git ignore rules before analysis (default: true)", + "default": True, + }