feat: implement automatic file watcher system for real-time index updates

Add comprehensive file monitoring capabilities using watchdog library to automatically
rebuild project index when files change, eliminating need for manual refresh operations.

Key features:
- Real-time file system monitoring with cross-platform support
- Intelligent debouncing to batch rapid file changes (configurable)
- Background async index rebuilding without blocking search operations
- Configurable exclude patterns and file type filtering
- Graceful fallback to manual refresh if watcher unavailable
- New MCP tools for file watcher status and configuration management

Technical improvements:
- Add FileWatcherService with Observer pattern for file monitoring
- Implement background rebuild system in IndexService using asyncio
- Enhanced ProjectService to automatically start file watcher on project setup
- Remove auto-refresh rate limiting (replaced by proactive monitoring)
- Add watchdog>=3.0.0 dependency for cross-platform file monitoring

This eliminates the previous 5-second rate limit and provides near-instantaneous
index updates when files change, greatly improving developer experience.

Author: johnhuang316

pyproject.toml

@@ -14,6 +14,7 @@ authors = [
 ]
 dependencies = [
     "mcp>=0.3.0",
+    "watchdog>=3.0.0",
 ]
 
 [project.urls]

src/code_index_mcp/project_settings.py

@@ -568,3 +568,48 @@ def refresh_available_strategies(self):
         print("Refreshing available search strategies...")
         self.available_strategies = _get_available_strategies()
         print(f"Available strategies found: {[s.name for s in self.available_strategies]}")
+    
+    def get_file_watcher_config(self) -> dict:
+        """
+        Get file watcher specific configuration.
+        
+        Returns:
+            dict: File watcher configuration with defaults
+        """
+        config = self.load_config()
+        default_config = {
+            "enabled": True,
+            "debounce_seconds": 3.0,
+            "additional_exclude_patterns": [],
+            "monitored_extensions": [],  # Empty = use all supported extensions
+            "exclude_patterns": [
+                ".git", ".svn", ".hg",
+                "node_modules", "__pycache__", ".venv", "venv",
+                ".DS_Store", "Thumbs.db",
+                "dist", "build", "target", ".idea", ".vscode",
+                ".pytest_cache", ".coverage", ".tox",
+                "bin", "obj"
+            ]
+        }
+        
+        # Merge with loaded config
+        file_watcher_config = config.get("file_watcher", {})
+        for key, default_value in default_config.items():
+            if key not in file_watcher_config:
+                file_watcher_config[key] = default_value
+        
+        return file_watcher_config
+    
+    def update_file_watcher_config(self, updates: dict) -> None:
+        """
+        Update file watcher configuration.
+        
+        Args:
+            updates: Dictionary of configuration updates
+        """
+        config = self.load_config()
+        if "file_watcher" not in config:
+            config["file_watcher"] = self.get_file_watcher_config()
+        
+        config["file_watcher"].update(updates)
+        self.save_config(config)

src/code_index_mcp/server.py

@@ -22,7 +22,7 @@
 from .project_settings import ProjectSettings
 from .services import (
     ProjectService, IndexService, SearchService,
-    FileService, SettingsService
+    FileService, SettingsService, FileWatcherService
 )
 from .services.settings_service import manage_temp_directory
 from .utils import (
@@ -37,6 +37,7 @@ class CodeIndexerContext:
     file_count: int = 0
     file_index: dict = field(default_factory=dict)
     index_cache: dict = field(default_factory=dict)
+    file_watcher_service: FileWatcherService = None
 
 @asynccontextmanager
 async def indexer_lifespan(_server: FastMCP) -> AsyncIterator[CodeIndexerContext]:
@@ -49,17 +50,23 @@ async def indexer_lifespan(_server: FastMCP) -> AsyncIterator[CodeIndexerContext
     # Initialize settings manager with skip_load=True to skip loading files
     settings = ProjectSettings(base_path, skip_load=True)
 
-    # Initialize context
+    # Initialize context - file watcher will be initialized later when project path is set
     context = CodeIndexerContext(
         base_path=base_path,
-        settings=settings
+        settings=settings,
+        file_watcher_service=None
     )
 
     try:
         print("Server ready. Waiting for user to set project path...")
         # Provide context to the server
         yield context
     finally:
+        # Stop file watcher if it was started
+        if context.file_watcher_service:
+            print("Stopping file watcher service...")
+            await context.file_watcher_service.stop_monitoring()
+
         # Only save index if project path has been set
         if context.base_path and context.index_cache:
             print(f"Saving index for project: {context.base_path}")
@@ -203,18 +210,18 @@ def refresh_index(ctx: Context) -> str:
     Manually refresh the project index when files have been added/removed/moved.
 
     Use when:
-    - After AI/LLM has created, modified, or deleted files
-    - After git operations (checkout, merge, pull) that change files
+    - File watcher is disabled or unavailable
+    - After large-scale operations (git checkout, merge, pull) that change many files
+    - When you want immediate index rebuild without waiting for file watcher debounce
     - When find_files results seem incomplete or outdated
-    - For immediate refresh without waiting for auto-refresh rate limits
+    - For troubleshooting suspected index synchronization issues
 
     Important notes for LLMs:
-    - This tool bypasses the 5-second rate limit that applies to auto-refresh
-    - Always available for immediate use when you know files have changed
+    - Always available as backup when file watcher is not working
     - Performs full project re-indexing for complete accuracy
     - Use when you suspect the index is stale after file system changes
-    - **Always call this after modifying files programmatically**
-    - Essential for find_files tool to see new/changed files
+    - **Call this after programmatic file modifications if file watcher seems unresponsive**
+    - Complements the automatic file watcher system
 
     Returns:
         Success message with total file count
@@ -254,6 +261,75 @@ def refresh_search_tools(ctx: Context) -> str:
     """
     return SearchService(ctx).refresh_search_tools()
 
+@mcp.tool()
+@handle_mcp_tool_errors(return_type='dict')
+def get_file_watcher_status(ctx: Context) -> Dict[str, Any]:
+    """Get file watcher service status and statistics."""
+    try:
+        # Get file watcher service from context
+        file_watcher_service = None
+        if hasattr(ctx.request_context.lifespan_context, 'file_watcher_service'):
+            file_watcher_service = ctx.request_context.lifespan_context.file_watcher_service
+        
+        if not file_watcher_service:
+            return {"status": "not_initialized", "message": "File watcher service not initialized"}
+        
+        # Get status from file watcher service
+        status = file_watcher_service.get_status()
+        
+        # Add index service status
+        index_service = IndexService(ctx)
+        rebuild_status = index_service.get_rebuild_status()
+        status["rebuild_status"] = rebuild_status
+        
+        # Add configuration
+        if hasattr(ctx.request_context.lifespan_context, 'settings') and ctx.request_context.lifespan_context.settings:
+            file_watcher_config = ctx.request_context.lifespan_context.settings.get_file_watcher_config()
+            status["configuration"] = file_watcher_config
+        
+        return status
+        
+    except Exception as e:
+        return {"status": "error", "message": f"Failed to get file watcher status: {e}"}
+
+@mcp.tool()
+@handle_mcp_tool_errors(return_type='str')
+def configure_file_watcher(
+    ctx: Context,
+    enabled: bool = None,
+    debounce_seconds: float = None,
+    additional_exclude_patterns: list = None
+) -> str:
+    """Configure file watcher service settings."""
+    try:
+        # Get settings from context
+        if not hasattr(ctx.request_context.lifespan_context, 'settings') or not ctx.request_context.lifespan_context.settings:
+            return "Settings not available - project path not set"
+        
+        settings = ctx.request_context.lifespan_context.settings
+        
+        # Build updates dictionary
+        updates = {}
+        if enabled is not None:
+            updates["enabled"] = enabled
+        if debounce_seconds is not None:
+            updates["debounce_seconds"] = debounce_seconds
+        if additional_exclude_patterns is not None:
+            updates["additional_exclude_patterns"] = additional_exclude_patterns
+        
+        if not updates:
+            return "No configuration changes specified"
+        
+        # Update configuration
+        settings.update_file_watcher_config(updates)
+        
+        # If file watcher is running, we would need to restart it for changes to take effect
+        # For now, just return success message with note about restart
+        return f"File watcher configuration updated: {updates}. Restart may be required for changes to take effect."
+        
+    except Exception as e:
+        return f"Failed to update file watcher configuration: {e}"
+
 # ----- PROMPTS -----
 
 @mcp.prompt()

src/code_index_mcp/services/__init__.py

@@ -24,12 +24,14 @@
 from .search_service import SearchService
 from .file_service import FileService
 from .settings_service import SettingsService
+from .file_watcher_service import FileWatcherService
 
 __all__ = [
     'BaseService',
     'ProjectService',
     'IndexService', 
     'SearchService',
     'FileService',
-    'SettingsService'
+    'SettingsService',
+    'FileWatcherService'
 ]