Skip to content

Commit f8109fe

Browse files
VampireVampire
authored
Make GHA workflow script execution able to run in parallel (#2092)
1 parent 8601cbb commit f8109fe

File tree

6 files changed

+253
-81
lines changed

6 files changed

+253
-81
lines changed

‎build-logic/preprocess-workflows/preprocess-workflows.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ plugins {
44
}
55

66
dependencies {
7-
implementation(libs.workflows.kotlin.compilerEmbeddable)
7+
compileOnly(libs.workflows.kotlin.compilerEmbeddable)
88
}
99

1010
gradlePlugin {

‎build-logic/preprocess-workflows/src/main/groovy/org/spockframework/gradle/DetermineImportedFiles.groovy‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package org.spockframework.gradle
2+
3+
import groovy.transform.CompileStatic
4+
import org.gradle.api.DefaultTask
5+
import org.gradle.api.file.ConfigurableFileCollection
6+
import org.gradle.api.file.ProjectLayout
7+
import org.gradle.api.file.RegularFileProperty
8+
import org.gradle.api.tasks.*
9+
import org.gradle.workers.WorkerExecutor
10+
11+
import javax.inject.Inject
12+
13+
@CompileStatic
14+
@UntrackedTask(because = 'imported files can import other files so inputs are not determinable upfront')
15+
abstract class DetermineImportedFiles extends DefaultTask {
16+
@InputFile
17+
abstract RegularFileProperty getMainKtsFile()
18+
19+
@InputFiles
20+
abstract ConfigurableFileCollection getKotlinCompilerEmbeddableClasspath()
21+
22+
@OutputFile
23+
abstract RegularFileProperty getImportedFiles()
24+
25+
@Inject
26+
abstract WorkerExecutor getWorkerExecutor()
27+
28+
@Inject
29+
abstract ProjectLayout getLayout()
30+
31+
@TaskAction
32+
def determineImportedFiles() {
33+
workerExecutor.classLoaderIsolation {
34+
it.classpath.from(kotlinCompilerEmbeddableClasspath)
35+
}.submit(DetermineImportedFilesWorkAction) {
36+
it.projectDirectory.set(layout.projectDirectory)
37+
it.mainKtsFile.set(mainKtsFile)
38+
it.importedFiles.set(importedFiles)
39+
}
40+
}
41+
}

‎build-logic/preprocess-workflows/src/main/groovy/org/spockframework/gradle/DetermineImportedFilesWorkAction.groovy‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package org.spockframework.gradle
2+
3+
import groovy.transform.CompileStatic
4+
import org.gradle.api.file.DirectoryProperty
5+
import org.gradle.api.file.RegularFileProperty
6+
import org.gradle.workers.WorkAction
7+
import org.gradle.workers.WorkParameters
8+
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
9+
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
10+
import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer
11+
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalFileSystem
12+
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalVirtualFile
13+
import org.jetbrains.kotlin.com.intellij.psi.PsiManager
14+
import org.jetbrains.kotlin.config.CompilerConfiguration
15+
import org.jetbrains.kotlin.psi.KtFile
16+
import org.jetbrains.kotlin.psi.KtLiteralStringTemplateEntry
17+
import org.jetbrains.kotlin.psi.KtStringTemplateExpression
18+
19+
import static org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles.JVM_CONFIG_FILES
20+
import static org.jetbrains.kotlin.config.CommonConfigurationKeys.MESSAGE_COLLECTOR_KEY
21+
22+
@CompileStatic
23+
abstract class DetermineImportedFilesWorkAction implements WorkAction<Parameters> {
24+
@Override
25+
void execute() {
26+
def projectDirectory = parameters.projectDirectory.get().asFile
27+
parameters
28+
.mainKtsFile
29+
.get()
30+
.asFile
31+
.with { getImportedFiles(it) }
32+
.collect { projectDirectory.relativePath(it).toString().replace('\\', '/') }
33+
.unique()
34+
.sort()
35+
.join('\n')
36+
.tap { parameters.importedFiles.get().asFile.text = it }
37+
}
38+
39+
private List<File> getImportedFiles(File workflowScript) {
40+
if (!workflowScript.file) {
41+
return []
42+
}
43+
44+
return PsiManager
45+
.getInstance(
46+
KotlinCoreEnvironment
47+
.createForProduction(
48+
Disposer.newDisposable(),
49+
new CompilerConfiguration().tap {
50+
it.put(MESSAGE_COLLECTOR_KEY, MessageCollector.@Companion.NONE)
51+
},
52+
JVM_CONFIG_FILES
53+
)
54+
.project
55+
)
56+
.findFile(
57+
new CoreLocalVirtualFile(
58+
new CoreLocalFileSystem(),
59+
workflowScript.toPath()
60+
)
61+
)
62+
.with { it as KtFile }
63+
.fileAnnotationList
64+
?.annotationEntries
65+
?.findAll { it.shortName?.asString() == 'Import' }
66+
*.valueArgumentList
67+
?.collectMany { it?.arguments ?: [] }
68+
*.argumentExpression
69+
?.findAll { it instanceof KtStringTemplateExpression }
70+
?.collect { it as KtStringTemplateExpression }
71+
*.entries
72+
*.first()
73+
?.findAll { it instanceof KtLiteralStringTemplateEntry }
74+
?.collect { it as KtLiteralStringTemplateEntry }
75+
?.collect { new File(workflowScript.parentFile, it.text) }
76+
?.collectMany { getImportedFiles(it) + it }
77+
?: []
78+
}
79+
80+
static interface Parameters extends WorkParameters {
81+
DirectoryProperty getProjectDirectory()
82+
83+
RegularFileProperty getMainKtsFile()
84+
85+
RegularFileProperty getImportedFiles()
86+
}
87+
}

‎build-logic/preprocess-workflows/src/main/groovy/org/spockframework/gradle/PreprocessGithubWorkflow.groovy‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package org.spockframework.gradle
2+
3+
import groovy.transform.CompileStatic
4+
import org.gradle.api.DefaultTask
5+
import org.gradle.api.file.ConfigurableFileCollection
6+
import org.gradle.api.file.RegularFileProperty
7+
import org.gradle.api.provider.Property
8+
import org.gradle.api.provider.Provider
9+
import org.gradle.api.tasks.*
10+
import org.gradle.jvm.toolchain.JavaLauncher
11+
import org.gradle.workers.WorkerExecutor
12+
13+
import javax.inject.Inject
14+
15+
@CompileStatic
16+
abstract class PreprocessGithubWorkflow extends DefaultTask {
17+
@InputFile
18+
abstract RegularFileProperty getWorkflowScript()
19+
20+
@InputFiles
21+
abstract ConfigurableFileCollection getImportedFiles()
22+
23+
@InputFiles
24+
abstract ConfigurableFileCollection getKotlinCompilerClasspath()
25+
26+
@InputFiles
27+
abstract ConfigurableFileCollection getMainKtsClasspath()
28+
29+
@Nested
30+
abstract Property<JavaLauncher> getJavaLauncher()
31+
32+
@OutputFile
33+
Provider<File> getWorkflowFile() {
34+
workflowScript.map {
35+
def workflowScript = it.asFile
36+
workflowScript.toPath().resolveSibling("${workflowScript.name - ~/\.main\.kts$/}.yaml").toFile()
37+
}
38+
}
39+
40+
@Inject
41+
abstract WorkerExecutor getWorkerExecutor()
42+
43+
PreprocessGithubWorkflow() {
44+
group = 'github workflows'
45+
}
46+
47+
@TaskAction
48+
def determineImportedFiles() {
49+
workerExecutor.noIsolation().submit(PreprocessGithubWorkflowWorkAction) {
50+
it.workflowScript.set(workflowScript)
51+
it.kotlinCompilerClasspath.from(kotlinCompilerClasspath)
52+
it.mainKtsClasspath.from(mainKtsClasspath)
53+
it.javaExecutable.set(javaLauncher.map { it.executablePath })
54+
}
55+
}
56+
}

‎build-logic/preprocess-workflows/src/main/groovy/org/spockframework/gradle/PreprocessGithubWorkflowWorkAction.groovy‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package org.spockframework.gradle
2+
3+
import groovy.transform.CompileStatic
4+
import org.gradle.api.file.ConfigurableFileCollection
5+
import org.gradle.api.file.RegularFileProperty
6+
import org.gradle.process.ExecOperations
7+
import org.gradle.workers.WorkAction
8+
import org.gradle.workers.WorkParameters
9+
10+
import javax.inject.Inject
11+
12+
@CompileStatic
13+
abstract class PreprocessGithubWorkflowWorkAction implements WorkAction<Parameters> {
14+
@Inject
15+
abstract ExecOperations getExecOperations()
16+
17+
@Override
18+
void execute() {
19+
// work-around for https://youtrack.jetbrains.com/issue/KT-74830
20+
Exception lastException = null
21+
for (i in 1..5) {
22+
try {
23+
execOperations.javaexec {
24+
it.executable = parameters.javaExecutable.get().asFile.absolutePath
25+
it.classpath(parameters.kotlinCompilerClasspath)
26+
it.mainClass.set('org.jetbrains.kotlin.cli.jvm.K2JVMCompiler')
27+
it.args('-no-stdlib', '-no-reflect')
28+
it.args('-classpath', parameters.mainKtsClasspath.asPath)
29+
it.args('-script', parameters.workflowScript.get().asFile.absolutePath)
30+
31+
// work-around for https://youtrack.jetbrains.com/issue/KT-42101
32+
it.systemProperty('kotlin.main.kts.compiled.scripts.cache.dir', '')
33+
}
34+
return
35+
} catch (Exception e) {
36+
lastException = e
37+
}
38+
}
39+
throw lastException
40+
}
41+
42+
static interface Parameters extends WorkParameters {
43+
RegularFileProperty getWorkflowScript()
44+
45+
ConfigurableFileCollection getKotlinCompilerClasspath()
46+
47+
ConfigurableFileCollection getMainKtsClasspath()
48+
49+
RegularFileProperty getJavaExecutable()
50+
}
51+
}

‎build-logic/preprocess-workflows/src/main/groovy/org/spockframework/gradle/PreprocessWorkflowsPlugin.groovy‎

Lines changed: 17 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -20,69 +20,47 @@ import groovy.transform.CompileStatic
2020
import org.gradle.api.Plugin
2121
import org.gradle.api.Project
2222
import org.gradle.api.artifacts.VersionCatalogsExtension
23-
import org.gradle.api.tasks.JavaExec
2423
import org.gradle.jvm.toolchain.JavaLanguageVersion
2524
import org.gradle.jvm.toolchain.JavaToolchainService
26-
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
27-
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
28-
import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer
29-
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalFileSystem
30-
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalVirtualFile
31-
import org.jetbrains.kotlin.com.intellij.psi.PsiManager
32-
import org.jetbrains.kotlin.config.CompilerConfiguration
33-
import org.jetbrains.kotlin.psi.KtFile
34-
import org.jetbrains.kotlin.psi.KtLiteralStringTemplateEntry
35-
import org.jetbrains.kotlin.psi.KtStringTemplateExpression
36-
37-
import static org.jetbrains.kotlin.cli.common.CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY
38-
import static org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles.JVM_CONFIG_FILES
3925

4026
@CompileStatic
4127
class PreprocessWorkflowsPlugin implements Plugin<Project> {
4228
void apply(Project project) {
4329
def libs = project.extensions.getByType(VersionCatalogsExtension).find('libs').orElseThrow(AssertionError::new)
30+
def kotlinCompilerEmbeddableClasspath = project.configurations.detachedConfiguration(
31+
libs.findLibrary('workflows-kotlin-compilerEmbeddable').orElseThrow(AssertionError::new).get(),
32+
)
4433
def kotlinCompilerClasspath = project.configurations.detachedConfiguration(
4534
libs.findLibrary('workflows-kotlin-compiler').orElseThrow(AssertionError::new).get(),
4635
libs.findLibrary('workflows-kotlin-scriptingCompiler').orElseThrow(AssertionError::new).get()
4736
)
48-
def kotlinScriptClasspath = project.configurations.detachedConfiguration(
37+
def mainKtsClasspath = project.configurations.detachedConfiguration(
4938
libs.findLibrary('workflows-kotlin-mainKts').orElseThrow(AssertionError::new).get()
5039
).tap {
5140
it.transitive = false
5241
}
5342

5443
def preprocessWorkflows = project.tasks.register('preprocessWorkflows') {
55-
it.group = 'github actions'
44+
it.group = 'github workflows'
5645
}
5746
project.file('.github/workflows').eachFileMatch(~/.*\.main\.kts$/) { workflowScript ->
5847
def workflowName = workflowScript.name - ~/\.main\.kts$/
5948
def pascalCasedWorkflowName = workflowName
6049
.replaceAll(/-\w/) { String it -> it[1].toUpperCase() }
6150
.replaceFirst(/^\w/) { String it -> it[0].toUpperCase() }
62-
def preprocessWorkflow = project.tasks.register("preprocess${pascalCasedWorkflowName}Workflow", JavaExec) {
63-
it.group = 'github actions'
64-
65-
it.inputs
66-
.file(workflowScript)
67-
.withPropertyName('workflowScript')
68-
it.inputs
69-
.files(getImportedFiles(project.file(workflowScript)))
70-
.withPropertyName("importedFiles")
71-
it.outputs
72-
.file(new File(workflowScript.parent, "${workflowName}.yaml"))
73-
.withPropertyName('workflowFile')
74-
75-
it.javaLauncher.set project.extensions.getByType(JavaToolchainService).launcherFor {
51+
def determineImportedFiles = project.tasks.register("determineImportedFilesFor${pascalCasedWorkflowName}Workflow", DetermineImportedFiles) {
52+
it.mainKtsFile.set(workflowScript)
53+
it.importedFiles.set(project.layout.buildDirectory.file("importedFilesFor${pascalCasedWorkflowName}Workflow.txt"))
54+
it.kotlinCompilerEmbeddableClasspath.from(kotlinCompilerEmbeddableClasspath)
55+
}
56+
def preprocessWorkflow = project.tasks.register("preprocess${pascalCasedWorkflowName}Workflow", PreprocessGithubWorkflow) {
57+
it.workflowScript.set(workflowScript)
58+
it.importedFiles.from(determineImportedFiles.flatMap { it.importedFiles }.map { it.asFile.readLines() })
59+
it.kotlinCompilerClasspath.from(kotlinCompilerClasspath)
60+
it.mainKtsClasspath.from(mainKtsClasspath)
61+
it.javaLauncher.set(project.extensions.getByType(JavaToolchainService).launcherFor {
7662
it.languageVersion.set(JavaLanguageVersion.of(17))
77-
}
78-
it.classpath(kotlinCompilerClasspath)
79-
it.mainClass.set 'org.jetbrains.kotlin.cli.jvm.K2JVMCompiler'
80-
it.args('-no-stdlib', '-no-reflect')
81-
it.args('-classpath', kotlinScriptClasspath.asPath)
82-
it.args('-script', workflowScript.absolutePath)
83-
84-
// work-around for https://youtrack.jetbrains.com/issue/KT-42101
85-
it.systemProperty('kotlin.main.kts.compiled.scripts.cache.dir', '')
63+
})
8664
}
8765
project.pluginManager.withPlugin('io.spring.nohttp') {
8866
// iff both tasks are run, workflow files should be generated before checkstyle check
@@ -95,45 +73,4 @@ class PreprocessWorkflowsPlugin implements Plugin<Project> {
9573
}
9674
}
9775
}
98-
99-
private List<File> getImportedFiles(File workflowScript) {
100-
if (!workflowScript.file) {
101-
return []
102-
}
103-
104-
return PsiManager
105-
.getInstance(
106-
KotlinCoreEnvironment
107-
.createForProduction(
108-
Disposer.newDisposable(),
109-
new CompilerConfiguration().tap {
110-
it.put(MESSAGE_COLLECTOR_KEY, MessageCollector.@Companion.NONE)
111-
},
112-
JVM_CONFIG_FILES
113-
)
114-
.project
115-
)
116-
.findFile(
117-
new CoreLocalVirtualFile(
118-
new CoreLocalFileSystem(),
119-
workflowScript.toPath()
120-
)
121-
)
122-
.with { it as KtFile }
123-
.fileAnnotationList
124-
?.annotationEntries
125-
?.findAll { it.shortName?.asString() == "Import" }
126-
*.valueArgumentList
127-
?.collectMany { it?.arguments ?: [] }
128-
*.argumentExpression
129-
?.findAll { it instanceof KtStringTemplateExpression }
130-
?.collect { it as KtStringTemplateExpression }
131-
*.entries
132-
*.first()
133-
?.findAll { it instanceof KtLiteralStringTemplateEntry }
134-
?.collect { it as KtLiteralStringTemplateEntry }
135-
?.collect { new File(workflowScript.parentFile, it.text) }
136-
?.collectMany { getImportedFiles(it) + it }
137-
?: []
138-
}
13976
}

0 commit comments

Comments
 (0)