build: optimizations for incremental builds and faster CI (#13033)

Co-authored-by: Andrew Sikowitz <[email protected]>

Author: chakru-r

.github/workflows/docker-unified.yml

@@ -447,7 +447,7 @@ jobs:
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel
-          mv ./datahub-frontend/build/distributions/datahub-frontend-*.zip datahub-frontend.zip
+          mv ./datahub-frontend/build/stage/main .
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

build.gradle

@@ -91,7 +91,7 @@ plugins {
   // TODO id "org.gradlex.java-ecosystem-capabilities" version "1.0"
 }
 
-apply from: "gradle/docker/docker.gradle"
+apply from: "gradle/docker/docker-utils.gradle"
 
 project.ext.spec = [
     'product' : [
@@ -112,12 +112,12 @@ project.ext.spec = [
 
 project.ext.externalDependency = [
     'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10", // max version due to licensing
+    'akkaParsing': "com.typesafe.akka:akka-parsing_$playScalaVersion:10.2.10", // akka-parsing is part of akka-http, so use akka http version
     'akkaActor': "com.typesafe.akka:akka-actor_$playScalaVersion:$akkaVersion",
     'akkaStream': "com.typesafe.akka:akka-stream_$playScalaVersion:$akkaVersion",
     'akkaActorTyped': "com.typesafe.akka:akka-actor-typed_$playScalaVersion:$akkaVersion",
     'akkaSlf4j': "com.typesafe.akka:akka-slf4j_$playScalaVersion:$akkaVersion",
     'akkaJackson': "com.typesafe.akka:akka-serialization-jackson_$playScalaVersion:$akkaVersion",
-    'akkaParsing': "com.typesafe.akka:akka-parsing_$playScalaVersion:$akkaVersion",
     'akkaProtobuf': "com.typesafe.akka:akka-protobuf-v3_$playScalaVersion:$akkaVersion",
     'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
     'antlr4': 'org.antlr:antlr4:4.9.3',

datahub-frontend/build.gradle

@@ -1,12 +1,12 @@
 plugins {
   id 'scala'
-  id 'com.palantir.docker'
   id 'org.gradle.playframework'
 }
 
 apply from: '../gradle/versioning/versioning.gradle'
 apply from: './play.gradle'
 apply from: '../gradle/coverage/java-coverage.gradle'
+apply from: '../gradle/docker/docker.gradle'
 
 ext {
   docker_repo = 'datahub-frontend-react'
@@ -34,7 +34,7 @@ model {
 }
 
 task myTar(type: Tar) {
-  compression = Compression.GZIP
+  compression = Compression.NONE
 
   from("${buildDir}/stage")
 
@@ -66,23 +66,27 @@ distributions {
   }
 }
 
+task unversionZip(type: Copy, dependsOn: [':datahub-web-react:distZip', distZip]) {
+
+  from ("${buildDir}/distributions")
+  include "datahub-frontend-${version}.zip"
+  into "${buildDir}/distributions"
+  rename "datahub-frontend-${version}.zip", "datahub-frontend.zip"
+}
+
 docker {
+  dependsOn(stageMainDist)
   name "${docker_registry}/${docker_repo}:v${version}"
-  version "v${version}"
   dockerfile file("${rootProject.projectDir}/docker/${docker_dir}/Dockerfile")
+  files "${buildDir}/stage"
   files fileTree(rootProject.projectDir) {
     include '.dockerignore'
     include 'docker/monitoring/*'
     include "docker/${docker_dir}/*"
   }.exclude {
     i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
   }
-  tag("Debug", "${docker_registry}/${docker_repo}:debug")
-
-  // platform('linux/arm64', 'linux/amd64')
-  buildx(true)
-  load(true)
-  push(false)
+  additionalTag("Debug", "${docker_registry}/${docker_repo}:debug")
 
   // Add build args if they are defined (needed for some CI or enterprise environments)
   def dockerBuildArgs = [:]
@@ -101,20 +105,6 @@ docker {
   }
 }
 
-task unversionZip(type: Copy, dependsOn: [':datahub-web-react:distZip', dist]) {
-  from ("${buildDir}/distributions")
-  include "datahub-frontend-${version}.zip"
-  into "${buildDir}/docker/"
-  rename "datahub-frontend-${version}.zip", "datahub-frontend.zip"
-}
-tasks.getByPath(":datahub-frontend:docker").dependsOn(unversionZip)
-
-task cleanLocalDockerImages {
-  doLast {
-    rootProject.ext.cleanLocalDockerImages(docker_registry, docker_repo, "${version}")
-  }
-}
-dockerClean.finalizedBy(cleanLocalDockerImages)
 
 // gradle 8 fixes
 tasks.getByName('createDatahub-frontendTarDist').dependsOn 'stageMainDist'

datahub-graphql-core/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id 'java'
-    id "io.github.kobylynskyi.graphql.codegen" version "4.1.1"
+    id "io.github.kobylynskyi.graphql.codegen" version "5.10.0"
 }
 
 apply from: '../gradle/coverage/java-coverage.gradle'
@@ -41,6 +41,7 @@ graphqlCodegen {
     generateApis = true
     generateParameterizedFieldsResolvers = false
     modelValidationAnnotation = "@javax.annotation.Nonnull"
+    addGeneratedAnnotation = false  // Skips timestamps in generated files which forces re-compile
     customTypesMapping = [
         Long: "Long",
         Float: "Float"

datahub-upgrade/build.gradle

@@ -1,11 +1,11 @@
 plugins {
   id 'org.springframework.boot'
   id 'java'
-  id 'com.palantir.docker'
 }
 
 apply from: "../gradle/versioning/versioning.gradle"
 apply from: "../gradle/coverage/java-coverage.gradle"
+apply from: "../gradle/docker/docker.gradle"
 
 ext {
   docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry
@@ -169,8 +169,8 @@ task runNoCode(type: Exec) {
 }
 
 docker {
+  dependsOn(bootJar)
   name "${docker_registry}/${docker_repo}:v${version}"
-  version "v${version}"
   dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
   files bootJar.outputs.files
   files fileTree(rootProject.projectDir) {
@@ -181,12 +181,7 @@ docker {
   }.exclude {
     i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
   }
-  tag("Debug", "${docker_registry}/${docker_repo}:debug")
-
-  // platform('linux/arm64', 'linux/amd64')
-  buildx(true)
-  load(true)
-  push(false)
+  additionalTag("Debug", "${docker_registry}/${docker_repo}:debug")
 
   // Add build args if they are defined (needed for some CI or enterprise environments)
   def dockerBuildArgs = [:]
@@ -204,12 +199,4 @@ docker {
     buildArgs(dockerBuildArgs)
   }
 }
-tasks.getByPath(":datahub-upgrade:docker").dependsOn([bootJar])
-
-task cleanLocalDockerImages {
-  doLast {
-    rootProject.ext.cleanLocalDockerImages(docker_registry, docker_repo, "${version}")
-  }
-}
-dockerClean.finalizedBy(cleanLocalDockerImages)
 

datahub-web-react/.gitignore

@@ -27,4 +27,7 @@ yarn-error.log*
 
 # gql codegen
 *.generated.ts
-/.vscode
+/.vscode
+
+.yarn-test-sentinel
+.yarn-lint-sentinel

datahub-web-react/build.gradle

@@ -80,6 +80,19 @@ task yarnServe(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
 task yarnTest(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
   // Explicitly runs in non-watch mode.
   args = ['run', project.hasProperty('withCoverage') ? 'test-coverage' : 'test', 'run']
+  def test_sentinel = "${buildDir}/.yarn-test-sentinel"
+  outputs.file(test_sentinel)
+  inputs.files(project.fileTree(dir: 'src', include: ['**/*.ts', '**/*.tsx']))
+  doLast {
+    // touch a file with name yarn-lint.txt in the build directory
+    def file = file(test_sentinel)
+    if (!file.exists()) {
+      file.createNewFile()
+    } else {
+      file.setLastModified(System.currentTimeMillis())
+    }
+  }
+  outputs.cacheIf { true }
 }
 
 task yarnLint(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
@@ -90,6 +103,19 @@ test.dependsOn([yarnInstall, yarnTest, yarnLint])
 
 task yarnLintFix(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
   args = ['run', 'lint-fix']
+  def lint_sentinel = "${buildDir}/.yarn-lint-sentinel"
+  outputs.file(lint_sentinel)
+  inputs.files(project.fileTree(dir: 'src', include: ['**/*.ts', '**/*.tsx']))
+  doLast {
+    // touch a file with name yarn-lint.txt in the build directory
+    def file = file(lint_sentinel)
+    if (!file.exists()) {
+      file.createNewFile()
+    } else {
+      file.setLastModified(System.currentTimeMillis())
+    }
+  }
+  outputs.cacheIf { true }
 }
 
 task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
@@ -130,9 +156,9 @@ distZip {
 }
 
 jar {
-  dependsOn distZip
+  dependsOn yarnBuild
   into('public') {
-    from zipTree(distZip.outputs.files.first())
+    from 'dist'
   }
   archiveClassifier = 'assets'
 }

docker/build.gradle

@@ -80,11 +80,25 @@ ext {
                     preserveVolumes: true
             ]
     ]
+
+    //  only for debug variants of quickstart to enable <variant>Reload tasks. 
+    // The actual service name needs the profile to be appended,  <container-name>-<profile>
+    moduleToContainer = [
+        ':metadata-service:war': 'datahub-gms',
+        ':datahub-frontend': 'frontend',
+        ':datahub-upgrade': 'system-update',
+        ':metadata-jobs:mce-consumer-job': 'datahub-mce-consumer',
+        ':metadata-jobs:mae-consumer-job': 'datahub-mae-consumer',
+
+    ]
 }
 
 // Register all quickstart tasks
 quickstart_configs.each { taskName, config ->
-    tasks.register(taskName)
+    tasks.register(taskName) {
+        group = 'quickstart'
+    }
+    
 }
 
 // Dynamically create all quickstart tasks and configurations
@@ -150,12 +164,57 @@ quickstart_configs.each { taskName, config ->
     }
 }
 
+// Register all quickstart tasks
+quickstart_configs.each { taskName, config ->
+    tasks.register("prepareAll${taskName}"){
+        group = 'quickstart-ci'
+    }
+}
+
+quickstart_configs.each { taskName, config ->
+    if (config.modules) {
+        tasks.getByName("prepareAll${taskName}").dependsOn(
+            config.modules.collect { it + ':dockerPrepare' }
+        )
+    }
+}
+
+quickstart_configs.each { taskName, config ->
+    tasks.register("buildImagesFromCache${taskName}") {
+        group = 'quickstart-ci'
+    }
+}
+
+quickstart_configs.each { taskName, config ->
+    if (config.modules) {
+        tasks.getByName("buildImagesFromCache${taskName}").dependsOn(
+            config.modules.collect { it + ':dockerFromCache' }
+        )
+    }
+}
+
+
+quickstart_configs.each { taskName, config ->
+    tasks.register("buildImages${taskName}") {
+        group = 'quickstart-ci'
+    }
+}
+
+quickstart_configs.each { taskName, config ->
+    if (config.modules) {
+        tasks.getByName("buildImages${taskName}").dependsOn(
+            config.modules.collect { it + ':dockerTag' }
+        )
+    }
+}
+
 tasks.register('minDockerCompose2.20', Exec) {
     executable 'bash'
     args '-c', 'echo -e "$(docker compose version --short)\n2.20"|sort --version-sort --check=quiet --reverse'
 }
 
 tasks.register('quickstartNuke') {
+    group = 'quickstart'
     doFirst {
         quickstart_configs.each { taskName, config ->
             dockerCompose."${taskName}".removeVolumes = !config.preserveVolumes
@@ -165,6 +224,7 @@ tasks.register('quickstartNuke') {
 }
 
 tasks.register('quickstartDown') {
+    group = 'quickstart'
     finalizedBy(tasks.withType(ComposeDownForced))
 }
 
@@ -173,12 +233,36 @@ tasks.withType(ComposeUp).configureEach {
     dependsOn tasks.named("minDockerCompose2.20")
 }
 
-task debugReload(type: Exec) {
-    def cmd = ['docker compose -p datahub --profile debug'] + ['-f', compose_base] + [
-            'restart',
-            'datahub-gms-debug',
-            'system-update-debug',
-            'frontend-debug'
-    ]
-    commandLine 'bash', '-c', cmd.join(" ")
-}
+// Register all quickstart Reload tasks. For quickstartDebug, the reload task is DebugReload. (Taskname without quickstart prefix)
+quickstart_configs.each { taskName, config ->
+    if (config.isDebug) {
+        def reloadTaskName = taskName.replaceFirst(/^quickstart/, "")
+        tasks.register("${reloadTaskName}Reload", Exec) {
+            dependsOn tasks.named("prepareAll${taskName}")
+            group = 'quickstart'
+            description = "Build and reload only changed containers for the ${taskName} task"
+            doFirst {
+                def executedTasks = project.gradle.taskGraph.allTasks.findAll { it.state.executed }
+                def containersToRestart = []
+
+                moduleToContainer.each { modulePath, containerName ->
+                    def moduleProject = project.project(modulePath)
+                    def dockerPrepareTask = moduleProject.tasks.findByName('dockerPrepare')
+                    
+                    if (dockerPrepareTask && executedTasks.contains(dockerPrepareTask) && !dockerPrepareTask.state.upToDate) {
+                        containersToRestart << "${containerName}-${config.profile}"
+                    }
+                }
+                
+                // Only restart containers that had their modules rebuilt
+                if (containersToRestart) {
+                    def cmd = ["docker compose -p datahub --profile ${config.profile}"] + ['-f', compose_base] + ['restart'] + containersToRestart
+                    commandLine 'bash', '-c', cmd.join(" ")
+                } else {
+                    // If no containers need restart, make this a no-op
+                    commandLine 'bash', '-c', 'echo "No containers need restarting - all modules are up to date"'
+                }    
+            }
+        }
+    }
+}