Skip to content

fix(amazonq): Added end to end UI tests for /test workflow. #5476

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 19 commits into
base: main
Choose a base branch
from
Open

fix(amazonq): Added end to end UI tests for /test workflow. #5476

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
30f9b7e
add e2e test for reject case
Randall-Jiang Mar 12, 2025
c587104
add e2e test for progress bar and cancel
Randall-Jiang Mar 20, 2025
4c9d171
add more test cases
Randall-Jiang Mar 28, 2025
940686d
fix the bugs
Randall-Jiang Mar 29, 2025
9aebccf
fix the formatting
Randall-Jiang Mar 29, 2025
347e1e7
fix assert
Randall-Jiang Mar 30, 2025
b8e782c
test
Randall-Jiang Mar 30, 2025
61444a3
fix gradlew condition to make it only run test
Randall-Jiang Mar 30, 2025
ea9e0e1
fix the test and change back gradlew file
Randall-Jiang Mar 31, 2025
a1992af
remove the generated test file
Randall-Jiang Mar 31, 2025
e5a38f7
if acknowledge message pop, would click it for user
Randall-Jiang Mar 31, 2025
5e03962
fix the clicking disclaimer
Randall-Jiang Mar 31, 2025
67b73cf
add acknowledge button
Randall-Jiang Mar 31, 2025
467a7db
add 5 mins timeout
Randall-Jiang Mar 31, 2025
be94bb7
use assertThat
Randall-Jiang Mar 31, 2025
9a960b6
Merge branch 'main' into e2e-test
Randall-Jiang Mar 31, 2025
f9eb952
lint
Randall-Jiang Apr 1, 2025
c598212
comment out the feedback button check, add it to TO-DO
Randall-Jiang Apr 1, 2025
6fc3014
remove comment code
Randall-Jiang Apr 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion gradle.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ toolkitVersion=3.62-SNAPSHOT
publishToken=
publishChannel=

ideProfileName=2025.1
ideProfileName=2024.3

remoteRobotPort=8080

Expand Down
257 changes: 257 additions & 0 deletions ...ter/tst-243+/software/aws/toolkits/jetbrains/uitests/testTests/QTestGenerationChatTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,153 @@ class QTestGenerationChatTest {
setupTestEnvironment()
}

@Test
fun `test method not found error handling`() {
val testCase = TestCase(
IdeProductProvider.IC,
LocalProjectInfo(
Paths.get("tstData", "qTestGenerationTestProject/")
)
).useRelease(System.getProperty("org.gradle.project.ideProfileName"))

// inject connection
useExistingConnectionForTest()

Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply {
System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path ->
pluginConfigurator.installPluginFromPath(
Path.of(path)
)
}

copyExistingConfig(Paths.get("tstData", "configAmazonQTests"))
updateGeneralSettings()
}.runIdeWithDriver()
.useDriverAndCloseIde {
waitForProjectOpen()
openFile(Paths.get("testModule1", "HappyPath.java").toString())
Thread.sleep(30000)
val result = executePuppeteerScript(testMethodNotFoundErrorScript)
assertThat(result)
.contains(
"new tab opened",
"Error message displayed correctly",
"Input field re-enabled after error",
)
}
}

@Test
fun `test cancel button during test generation`() {
val testCase = TestCase(
IdeProductProvider.IC,
LocalProjectInfo(
Paths.get("tstData", "qTestGenerationTestProject/")
)
).useRelease(System.getProperty("org.gradle.project.ideProfileName"))

// inject connection
useExistingConnectionForTest()

Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply {
System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path ->
pluginConfigurator.installPluginFromPath(
Path.of(path)
)
}

copyExistingConfig(Paths.get("tstData", "configAmazonQTests"))
updateGeneralSettings()
}.runIdeWithDriver()
.useDriverAndCloseIde {
waitForProjectOpen()
openFile(Paths.get("testModule1", "HappyPath.java").toString())
Thread.sleep(30000)
val result = executePuppeteerScript(testCancelButtonScript)
assertThat(result)
.contains(
"new tab opened",
"Progress bar text displayed",
"Cancel button found",
"Cancel button clicked",
"Test generation cancelled successfully",
"Input field re-enabled after cancellation",
)
}
}

@Test
fun `test documentation generation error handling`() {
val testCase = TestCase(
IdeProductProvider.IC,
LocalProjectInfo(
Paths.get("tstData", "qTestGenerationTestProject/")
)
).useRelease(System.getProperty("org.gradle.project.ideProfileName"))

// inject connection
useExistingConnectionForTest()

Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply {
System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path ->
pluginConfigurator.installPluginFromPath(
Path.of(path)
)
}

copyExistingConfig(Paths.get("tstData", "configAmazonQTests"))
updateGeneralSettings()
}.runIdeWithDriver()
.useDriverAndCloseIde {
waitForProjectOpen()
openFile(Paths.get("testModule1", "HappyPath.java").toString())
Thread.sleep(30000)
val result = executePuppeteerScript(testDocumentationErrorScript)
assertThat(result)
.contains(
"new tab opened",
"Error message displayed correctly",
"Input field re-enabled after error",
)
}
}

@Test
fun `test remove function error handling`() {
val testCase = TestCase(
IdeProductProvider.IC,
LocalProjectInfo(
Paths.get("tstData", "qTestGenerationTestProject/")
)
).useRelease(System.getProperty("org.gradle.project.ideProfileName"))

// inject connection
useExistingConnectionForTest()

Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply {
System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path ->
pluginConfigurator.installPluginFromPath(
Path.of(path)
)
}

copyExistingConfig(Paths.get("tstData", "configAmazonQTests"))
updateGeneralSettings()
}.runIdeWithDriver()
.useDriverAndCloseIde {
waitForProjectOpen()
openFile(Paths.get("testModule1", "HappyPath.java").toString())
Thread.sleep(30000)
val result = executePuppeteerScript(testRemoveFunctionErrorScript)
assertThat(result)
.contains(
"new tab opened",
"Error message displayed correctly",
"Input field re-enabled after error",
)
}
}

@Test
fun `can run a test from the chat`() {
val testCase = TestCase(
Expand Down Expand Up @@ -193,6 +340,116 @@ class QTestGenerationChatTest {
}
}

@Test
fun `test reject path from the chat`() {
val testCase = TestCase(
IdeProductProvider.IC,
LocalProjectInfo(
Paths.get("tstData", "qTestGenerationTestProject/")
)
).useRelease(System.getProperty("org.gradle.project.ideProfileName"))

// inject connection
useExistingConnectionForTest()

Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply {
System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path ->
pluginConfigurator.installPluginFromPath(
Path.of(path)
)
}

copyExistingConfig(Paths.get("tstData", "configAmazonQTests"))
updateGeneralSettings()
}.runIdeWithDriver()
.useDriverAndCloseIde {
waitForProjectOpen()
openFile(Paths.get("testModule1", "HappyPath.java").toString())
Thread.sleep(30000)
val result = executePuppeteerScript(testRejectPathScript)
assertThat(result)
.contains(
"new tab opened",
"View Diff opened",
"Result Reject",
"Unit test generation completed.",
"Input field re-enabled after rejection",
)
}
}

@Test
fun `test NL error from the chat`() {
val testCase = TestCase(
IdeProductProvider.IC,
LocalProjectInfo(
Paths.get("tstData", "qTestGenerationTestProject/")
)
).useRelease(System.getProperty("org.gradle.project.ideProfileName"))

// inject connection
useExistingConnectionForTest()

Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply {
System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path ->
pluginConfigurator.installPluginFromPath(
Path.of(path)
)
}

copyExistingConfig(Paths.get("tstData", "configAmazonQTests"))
updateGeneralSettings()
}.runIdeWithDriver()
.useDriverAndCloseIde {
waitForProjectOpen()
openFile(Paths.get("testModule1", "HappyPath.java").toString())
Thread.sleep(30000)
val result = executePuppeteerScript(testNLErrorPathScript)
assertThat(result)
.contains(
"new tab opened",
"Command entered: /test /something/",
"Error message displayed correctly"
)
}
}

@Test
fun `test progress bar during test generation`() {
val testCase = TestCase(
IdeProductProvider.IC,
LocalProjectInfo(
Paths.get("tstData", "qTestGenerationTestProject/")
)
).useRelease(System.getProperty("org.gradle.project.ideProfileName"))

// inject connection
useExistingConnectionForTest()

Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply {
System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path ->
pluginConfigurator.installPluginFromPath(
Path.of(path)
)
}

copyExistingConfig(Paths.get("tstData", "configAmazonQTests"))
updateGeneralSettings()
}.runIdeWithDriver()
.useDriverAndCloseIde {
waitForProjectOpen()
openFile(Paths.get("testModule1", "HappyPath.java").toString())
Thread.sleep(30000)
val result = executePuppeteerScript(testProgressBarScript)
assertThat(result)
.contains(
"new tab opened",
"Progress bar text displayed",
"Test generation completed successfully"
)
}
}

companion object {
@JvmStatic
@AfterAll
Expand Down
Loading
Loading