Improve the autocomplete for the sidekick /drop /add commands

In `aicodebot/coder.py`, a new method `filtered_file_list` has been added to filter files in a directory based on `.gitignore` and other ignore patterns. This method is used in the `SidekickCompleter` class to provide better command completion for file paths. The `SidekickCompleter` class has been moved from `aicodebot/helpers.py` to `aicodebot/coder.py` to better align with its usage.

In `aicodebot/helpers.py`, the `SidekickCompleter` class has been removed as it's now in `aicodebot/coder.py`.

In `tests/test_coder.py`, a new test has been added for the `filtered_file_list` method to ensure it's working as expected.

Author: TechNickAI

aicodebot/cli.py

@@ -1,13 +1,7 @@
 from aicodebot import version as aicodebot_version
-from aicodebot.coder import CREATIVE_TEMPERATURE, DEFAULT_MAX_TOKENS, Coder
+from aicodebot.coder import CREATIVE_TEMPERATURE, DEFAULT_MAX_TOKENS, Coder, SidekickCompleter
 from aicodebot.config import get_config_file, get_local_data_dir, read_config
-from aicodebot.helpers import (
-    RichLiveCallbackHandler,
-    SidekickCompleter,
-    create_and_write_file,
-    exec_and_get_output,
-    logger,
-)
+from aicodebot.helpers import RichLiveCallbackHandler, create_and_write_file, exec_and_get_output, logger
 from aicodebot.learn import load_documents_from_repo, store_documents
 from aicodebot.prompts import DEFAULT_PERSONALITY, PERSONALITIES, generate_files_context, get_prompt
 from datetime import datetime
@@ -494,7 +488,7 @@ def show_file_context(files):
             token_length = Coder.get_token_length(Path(file).read_text())
             console.print(f"\t{file} ({humanize.intcomma(token_length)} tokens)")
 
-    files = set(files)  # Dedupe
+    files = set(files)  # Create a set for deduplication
     if files:
         show_file_context(files)
 

aicodebot/coder.py

@@ -3,6 +3,7 @@
 from langchain.chat_models import ChatOpenAI
 from openai.api_resources import engine
 from pathlib import Path
+from prompt_toolkit.completion import Completer, Completion
 import fnmatch, functools, openai, os, re, subprocess, tiktoken
 
 DEFAULT_MAX_TOKENS = 512
@@ -28,6 +29,31 @@ def clone_repo(repo_url, repo_dir):
             logger.info(f"Cloning {repo_url} to {repo_dir}")
             subprocess.run(["git", "clone", repo_url, repo_dir], check=True)
 
+    @classmethod
+    def filtered_file_list(cls, path, ignore_patterns=None, use_gitignore=True):
+        """Walk a directory and return a list of files that are not ignored."""
+        ignore_patterns = ignore_patterns.copy() if ignore_patterns else []
+
+        base_path = Path(path)
+
+        if use_gitignore:
+            # Note: .gitignore files can exist in sub directories as well, such as * in __pycache__ directories
+            gitignore_file = base_path / ".gitignore"
+            if gitignore_file.exists():
+                with gitignore_file.open() as f:
+                    ignore_patterns.extend(line.strip() for line in f if line.strip() and not line.startswith("#"))
+
+        out = []
+        if base_path.is_dir():
+            if not any(fnmatch.fnmatch(base_path.name, pattern) for pattern in ignore_patterns):
+                out.append(base_path)
+                for item in base_path.iterdir():
+                    out += cls.filtered_file_list(item, ignore_patterns, use_gitignore)
+        elif not any(fnmatch.fnmatch(base_path.name, pattern) for pattern in ignore_patterns):
+            out.append(base_path)
+
+        return out
+
     @classmethod
     def generate_directory_structure(cls, path, ignore_patterns=None, use_gitignore=True, indent=0):
         """Generate a text representation of the directory structure of a path."""
@@ -246,3 +272,26 @@ def parse_github_url(repo_url):
 
         owner, repo = match.groups()
         return owner, repo
+
+
+class SidekickCompleter(Completer):
+    """A custom prompt_toolkit completer for sidekick."""
+
+    def get_completions(self, document, complete_event):
+        # Get the text before the cursor
+        text = document.text_before_cursor
+
+        supported_commands = ["/add", "/drop", "/edit", "/files", "/quit"]
+
+        # If the text starts with a slash, it's a command
+        if text.startswith("/"):
+            for command in supported_commands:
+                if command.startswith(text):
+                    yield Completion(command, start_position=-len(text))
+
+        if text.startswith(("/add ", "/drop ")):
+            # If the text starts with /add or /drop, it's a file
+            files = Coder.filtered_file_list(".", use_gitignore=True, ignore_patterns=[".git"])
+            for file in files:
+                if str(file).startswith(text.split()[-1]):
+                    yield Completion(str(file), start_position=-len(text.split()[-1]))

aicodebot/helpers.py

@@ -1,7 +1,6 @@
 from langchain.callbacks.base import BaseCallbackHandler
 from loguru import logger
 from pathlib import Path
-from prompt_toolkit.completion import Completer, Completion
 from rich.markdown import Markdown
 import os, subprocess, sys
 
@@ -32,29 +31,6 @@ def create_and_write_file(filename, text, overwrite=False):
         f.write(text)
 
 
-class SidekickCompleter(Completer):
-    """A custom prompt_toolkit completer for sidekick."""
-
-    def get_completions(self, document, complete_event):
-        # Get the text before the cursor
-        text = document.text_before_cursor
-
-        supported_commands = ["/add", "/drop", "/edit", "/files", "/quit"]
-
-        # If the text starts with a slash, it's a command
-        if text.startswith("/"):
-            for command in supported_commands:
-                if command.startswith(text):
-                    yield Completion(command, start_position=-len(text))
-
-        if text.startswith(("/add ", "/drop ")):
-            # If the text starts with /add or /drop, it's a file
-            files = Path().rglob("*")
-            for file in files:
-                if str(file).startswith(text.split()[-1]):
-                    yield Completion(str(file), start_position=-len(text.split()[-1]))
-
-
 def exec_and_get_output(command):
     """Execute a command and return its output as a string."""
     logger.debug(f"Executing command: {' '.join(command)}")

tests/test_coder.py

@@ -40,6 +40,9 @@ def test_generate_directory_structure(tmp_path):
     assert "- [File] file.txt" in directory_structure_no_gitignore
     assert "- [File] sub_file" in directory_structure_no_gitignore
 
+    file_list = Coder.filtered_file_list(".", use_gitignore=True, ignore_patterns=[".git"])
+    assert len(file_list) > 10
+
 
 def test_get_token_length():
     text = ""