Skip to content

Commit e5f6d67

Browse files
authored
Re-enable a test in SmartApplyTests.cs (#382)
The SmartApply tests were failing inconsistently due to several issues: **Problems Fixed** Apply button not clickable - The button exists in DOM with tw-hidden class on narrow viewports. Changed wait strategy from `WaitForAsync()` (waits for visible) to `WaitForAsync(State.Attached)` (waits for element in DOM), then remove hidden class before clicking. **Missing file context** - File context chips weren't appearing in chat input on the first SmartApply test run, causing the LLM to respond without code suggestions (no Apply button). Added a dummy warm-up test to initialize file context functionality. The ApplyLastSuggestionFor helper now waits for context chips before sending prompts. **Insufficient timeout** - Increased timeout to 60s to accommodate LLM response times. **Test file improvements** - Added intentional typo to `Manager.cs` to ensure LLM reliably provides code suggestions. ### Known Issue The dummy warm-up test is a workaround. The root cause of why file context chips don't appear on first test run needs investigation - likely a timing or initialization issue. ## Test plan Green and more deterministic CI. <!-- REQUIRED; info at https://docs-legacy.sourcegraph.com/dev/background-information/testing_principles -->
1 parent d7abb43 commit e5f6d67

File tree

2 files changed

+23
-13
lines changed

2 files changed

+23
-13
lines changed

src/Cody.VisualStudio.Tests/SmartApplyTests.cs

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,57 +38,66 @@ public SmartApplyTests(ITestOutputHelper output) : base(output)
3838
}
3939

4040
[VsFact(Version = VsVersion.VS2022)]
41-
public async Task Apply_Suggestion_Is_Modifying_Point_Document()
41+
public async Task Apply_Suggestion_Is_Modifying_Dummy_Document()
4242
{
43-
// given
43+
// TODO: This is a workaround. Without this test, the first SmartApply test fails
44+
// because file context chips don't appear in the chat input, causing the LLM to
45+
// respond without code suggestions. Running this dummy test first somehow fixes it.
46+
// The root cause needs investigation - likely a timing or initialization issue
47+
// in how file context is added to chat on first document open.
4448
await NewChat();
49+
await OpenSolution(SolutionsPaths.GetConsoleApp1File("ConsoleApp1.sln"));
50+
await OpenDocument(SolutionsPaths.GetConsoleApp1File(@"ConsoleApp1\Point.cs"));
51+
}
4552

53+
[VsFact(Version = VsVersion.VS2022)]
54+
public async Task Apply_Suggestion_Is_Modifying_Point_Document()
55+
{
56+
await NewChat();
4657
await OpenSolution(SolutionsPaths.GetConsoleApp1File("ConsoleApp1.sln"));
4758
await OpenDocument(SolutionsPaths.GetConsoleApp1File(@"ConsoleApp1\Point.cs"));
4859

4960
var originalText = await GetActiveDocumentText();
5061

51-
// when
5262
await ApplyLastSuggestionFor("Suggest improvements");
5363

5464
var modifiedText = await GetActiveDocumentText();
5565

56-
// then
5766
Assert.NotEqual(modifiedText, originalText);
5867
}
5968

60-
[VsFact(Version = VsVersion.VS2022, Skip = "Unstable")]
69+
[VsFact(Version = VsVersion.VS2022)]
6170
public async Task Apply_Suggestion_Is_Modifying_Manager_Document()
6271
{
63-
// given
6472
await NewChat();
65-
6673
await OpenSolution(SolutionsPaths.GetConsoleApp1File("ConsoleApp1.sln"));
6774
await OpenDocument(SolutionsPaths.GetConsoleApp1File(@"ConsoleApp1\Manager.cs"));
6875

6976
var originalText = await GetActiveDocumentText();
7077

71-
// when
7278
await ApplyLastSuggestionFor("Suggest improvements in Print() method");
7379

7480
var modifiedText = await GetActiveDocumentText();
7581

76-
// then
7782
Assert.NotEqual(modifiedText, originalText);
7883
}
7984

8085
private async Task ApplyLastSuggestionFor(string chatText)
8186
{
87+
var contextChipLocator = Page.Locator("[aria-label='Chat message'] span[data-lexical-decorator='true']");
88+
await contextChipLocator.First.WaitForAsync(new() { Timeout = 5000 });
89+
8290
await EnterChatTextAndSend(chatText);
8391

8492
var apply = Page.Locator("span", new() { HasText = "Apply" }).Last;
8593

86-
// checking if Chat window is too narrow to show "Apply" text
94+
await apply.WaitForAsync(new() { Timeout = 60000, State = WaitForSelectorState.Attached });
95+
8796
var hasHiddenClass = await apply.EvaluateAsync<bool>(@"element => element.classList.contains('tw-hidden')");
8897
if (hasHiddenClass)
89-
await apply.EvaluateAsync("element => element.classList.remove('tw-hidden')"); // force shows "Apply" text so it will be possible to click on it
98+
await apply.EvaluateAsync("element => element.classList.remove('tw-hidden')");
9099

91-
await apply.ClickAsync(new() { Force = true });
100+
await apply.ClickAsync(new() { Force = true});
92101

93102
await EditAppliedAsync();
94103
}

src/Cody.VisualStudio.Tests/TestProjects/ConsoleApp/ConsoleApp1/Manager.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ public Manager2() { }
66

77
public void Print()
88
{
9-
Console.WriteLine("Hello, World!");
9+
var mesage = "Hello, World!";
10+
Console.WriteLine(mesage);
1011

1112

1213
Console.WriteLine("Bye, World!");

0 commit comments

Comments
 (0)