Skip to content

Commit 2f90ce0

Browse files
dblockdblock
authored
Jenkins incremental build (opensearch-project#1147)
* Do not rebuild unless there were changes. Signed-off-by: dblock <[email protected]> * Skip docker build if the artifacts don't exist. Signed-off-by: dblock <[email protected]> * Undo pipeline helper change. Signed-off-by: dblock <[email protected]> * Fixup paths. Signed-off-by: dblock <[email protected]>
1 parent 79f8348 commit 2f90ce0

10 files changed

+332
-67
lines changed

‎.groovylintrc.json

+6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@
44
"CompileStatic": {
55
"enabled": false
66
},
7+
"DuplicateMapLiteral": {
8+
"enabled": false
9+
},
10+
"DuplicateNumberLiteral": {
11+
"enabled": false
12+
},
713
"DuplicateStringLiteral": {
814
"enabled": false
915
},

‎tests/jenkins/TestIncrementalBuild.groovy

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package jenkins.tests
10+
11+
import org.junit.*
12+
import java.util.*
13+
14+
class TestIncrementalBuild extends BuildPipelineTest {
15+
@Before
16+
void setUp() {
17+
super.setUp()
18+
19+
binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42')
20+
binding.setVariable('BUILD_NUMBER', '33')
21+
binding.setVariable('PUBLIC_ARTIFACT_URL', 'https://ci.opensearch.org/dbc')
22+
binding.setVariable('JOB_NAME', 'vars-build')
23+
binding.setVariable('ARTIFACT_BUCKET_NAME', 'artifact-bucket')
24+
binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account')
25+
binding.setVariable('STAGE_NAME', 'stage')
26+
27+
helper.registerAllowedMethod("withCredentials", [List, Closure], { list, closure ->
28+
closure.delegate = delegate
29+
return helper.callClosure(closure)
30+
})
31+
32+
helper.registerAllowedMethod("sha1", [String], { filename ->
33+
return 'sha1'
34+
})
35+
36+
helper.registerAllowedMethod("s3Upload", [Map])
37+
helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
38+
closure.delegate = delegate
39+
return helper.callClosure(closure)
40+
})
41+
42+
helper.registerAllowedMethod("git", [Map])
43+
}
44+
45+
@Test
46+
public void test() {
47+
helper.registerAllowedMethod("s3DoesObjectExist", [Map], { args ->
48+
return true
49+
})
50+
51+
helper.registerAllowedMethod("httpRequest", [Map], { args ->
52+
return [
53+
status: 404
54+
]
55+
})
56+
57+
super.testPipeline("tests/jenkins/jobs/IncrementalBuild_Jenkinsfile")
58+
}
59+
}

‎tests/jenkins/TestVars.groovy

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ class TestVars extends BuildPipelineTest {
1717
binding.setVariable('INPUT_MANIFEST', '2.0.0/opensearch-2.0.0.yml')
1818
binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42')
1919
binding.setVariable('BUILD_NUMBER', '33')
20-
binding.setVariable('PUBLIC_ARTIFACT_URL', 'https://ci.opensearch.org/dbc/')
20+
binding.setVariable('PUBLIC_ARTIFACT_URL', 'https://ci.opensearch.org/dbc')
2121
binding.setVariable('JOB_NAME', 'vars-build')
2222
binding.setVariable('ARTIFACT_BUCKET_NAME', 'artifact-bucket')
2323
binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account')

‎tests/jenkins/data/opensearch-1.3.0.yml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
schema-version: '1.0'
2+
build:
3+
name: OpenSearch
4+
version: 1.3.0
5+
ci:
6+
image:
7+
name: "opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028"
8+
args: "-e JAVA_HOME=/opt/java/openjdk-11"
9+
components:
10+
- name: OpenSearch
11+
ref: '1.x'
12+
repository: https://github.com/opensearch-project/OpenSearch.git
13+
checks:
14+
- gradle:publish
15+
- gradle:properties:version

‎tests/jenkins/data/opensearch-1.3.0.yml.lock

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
build:
2+
architecture: x64
3+
name: OpenSearch
4+
platform: darwin
5+
snapshot: false
6+
version: 1.3.0
7+
ci:
8+
image:
9+
args: -e JAVA_HOME=/opt/java/openjdk-11
10+
name: opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028
11+
components:
12+
- checks:
13+
- gradle:publish
14+
- gradle:properties:version
15+
name: OpenSearch
16+
ref: 83e005b287816c0135d0a9b762054eaf5dcfe9df
17+
repository: https://github.com/opensearch-project/OpenSearch.git
18+
schema-version: '1.0'

‎tests/jenkins/jobs/IncrementalBuild_Jenkinsfile

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
pipeline {
2+
agent none
3+
stages {
4+
stage('build and assemble, sha exists') {
5+
steps {
6+
script {
7+
buildAssembleUpload(
8+
dryRun: false, // will check whether SHA exists, which is stubbed to return true
9+
manifest: 'tests/jenkins/data/opensearch-1.3.0.yml',
10+
platform: 'linux',
11+
architecture: 'x64'
12+
)
13+
}
14+
}
15+
}
16+
stage('build and assemble, sha does not exist') {
17+
steps {
18+
script {
19+
buildAssembleUpload(
20+
dryRun: true, // will skip checking whether SHA exists or not
21+
manifest: 'tests/jenkins/data/opensearch-1.3.0.yml',
22+
platform: 'linux',
23+
architecture: 'x64'
24+
)
25+
}
26+
}
27+
}
28+
stage('docker, no build') {
29+
steps {
30+
script {
31+
buildDockerImage(
32+
manifest: 'tests/jenkins/data/opensearch-1.3.0.yml',
33+
dryRun: false, // will check whether .tar.gz exists, which is stubbed to false and skip
34+
)
35+
}
36+
}
37+
}
38+
stage('docker, existing build') {
39+
steps {
40+
script {
41+
buildDockerImage(
42+
manifest: 'tests/jenkins/data/opensearch-1.3.0.yml',
43+
dryRun: true, // will skip checking whether .tar.gz exists, and build the docker image
44+
)
45+
}
46+
}
47+
}
48+
}
49+
}

‎tests/jenkins/jobs/IncrementalBuild_Jenkinsfile.txt

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
IncrementalBuild_Jenkinsfile.run()
2+
IncrementalBuild_Jenkinsfile.pipeline(groovy.lang.Closure)
3+
IncrementalBuild_Jenkinsfile.echo(Executing on agent [label:none])
4+
IncrementalBuild_Jenkinsfile.stage(build and assemble, sha exists, groovy.lang.Closure)
5+
IncrementalBuild_Jenkinsfile.script(groovy.lang.Closure)
6+
IncrementalBuild_Jenkinsfile.buildAssembleUpload({dryRun=false, manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64})
7+
buildAssembleUpload.buildManifest({dryRun=false, manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true})
8+
buildManifest.sh(./build.sh tests/jenkins/data/opensearch-1.3.0.yml -p linux -a x64 --lock)
9+
buildAssembleUpload.sha1(tests/jenkins/data/opensearch-1.3.0.yml.lock)
10+
buildAssembleUpload.echo(Manifest SHA: sha1)
11+
buildAssembleUpload.legacySCM(groovy.lang.Closure)
12+
buildAssembleUpload.library({identifier=jenkins@20211123, retriever=null})
13+
buildAssembleUpload.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml.lock})
14+
InputManifest.asBoolean()
15+
InputManifest.getSHAsRoot(vars-build, linux, x64)
16+
buildAssembleUpload.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
17+
buildAssembleUpload.s3DoesObjectExist({bucket=artifact-bucket, path=vars-build/1.3.0/shas/linux/x64/sha1.yml})
18+
Messages.asBoolean()
19+
Messages.add(stage, Skipped stage, vars-build/1.3.0/shas/linux/x64/sha1.yml exists.)
20+
buildAssembleUpload.writeFile({file=messages/stage.msg, text=Skipped stage, vars-build/1.3.0/shas/linux/x64/sha1.yml exists.})
21+
buildAssembleUpload.stash({includes=messages/*, name=messages-stage})
22+
buildAssembleUpload.echo(Skipping, vars-build/1.3.0/shas/linux/x64/sha1.yml already exists.)
23+
IncrementalBuild_Jenkinsfile.stage(build and assemble, sha does not exist, groovy.lang.Closure)
24+
IncrementalBuild_Jenkinsfile.script(groovy.lang.Closure)
25+
IncrementalBuild_Jenkinsfile.buildAssembleUpload({dryRun=true, manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64})
26+
buildAssembleUpload.buildManifest({dryRun=true, manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64, lock=true})
27+
buildManifest.sh(echo ./build.sh tests/jenkins/data/opensearch-1.3.0.yml -p linux -a x64 --lock)
28+
buildAssembleUpload.sha1(tests/jenkins/data/opensearch-1.3.0.yml)
29+
buildAssembleUpload.echo(Manifest SHA: sha1)
30+
buildAssembleUpload.legacySCM(groovy.lang.Closure)
31+
buildAssembleUpload.library({identifier=jenkins@20211123, retriever=null})
32+
buildAssembleUpload.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml})
33+
InputManifest.asBoolean()
34+
InputManifest.getSHAsRoot(vars-build, linux, x64)
35+
buildAssembleUpload.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
36+
buildAssembleUpload.buildManifest({dryRun=true, manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64})
37+
buildManifest.sh(echo ./build.sh tests/jenkins/data/opensearch-1.3.0.yml -p linux -a x64)
38+
buildAssembleUpload.assembleManifest({dryRun=true, manifest=tests/data/opensearch-build-1.1.0.yml, platform=linux, architecture=x64})
39+
assembleManifest.legacySCM(groovy.lang.Closure)
40+
assembleManifest.library({identifier=jenkins@20211123, retriever=null})
41+
assembleManifest.readYaml({file=tests/data/opensearch-build-1.1.0.yml})
42+
BuildManifest.asBoolean()
43+
BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
44+
assembleManifest.sh(echo ./assemble.sh "tests/data/opensearch-build-1.1.0.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64)
45+
buildAssembleUpload.uploadArtifacts({dryRun=true, manifest=tests/jenkins/data/opensearch-1.3.0.yml, platform=linux, architecture=x64})
46+
uploadArtifacts.legacySCM(groovy.lang.Closure)
47+
uploadArtifacts.library({identifier=jenkins@20211123, retriever=null})
48+
uploadArtifacts.readYaml({file=tests/data/opensearch-build-1.1.0.yml})
49+
BuildManifest.asBoolean()
50+
BuildManifest.getArtifactRoot(vars-build, 33)
51+
uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.1.0/33/linux/x64)
52+
uploadArtifacts.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
53+
uploadArtifacts.echo(s3Upload(file: 'builds', bucket: artifact-bucket, path: vars-build/1.1.0/33/linux/x64/builds))
54+
uploadArtifacts.echo(s3Upload(file: 'dist', bucket: artifact-bucket, path: vars-build/1.1.0/33/linux/x64/dist))
55+
BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
56+
Messages.asBoolean()
57+
Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml
58+
https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/manifest.yml)
59+
uploadArtifacts.writeFile({file=messages/stage.msg, text=https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml
60+
https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/dist/opensearch/manifest.yml})
61+
uploadArtifacts.stash({includes=messages/*, name=messages-stage})
62+
buildAssembleUpload.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
63+
buildAssembleUpload.echo(s3Upload(bucket: artifact-bucket, file: tests/jenkins/data/opensearch-1.3.0.yml, path: vars-build/1.3.0/shas/linux/x64/sha1.yml))
64+
IncrementalBuild_Jenkinsfile.stage(docker, no build, groovy.lang.Closure)
65+
IncrementalBuild_Jenkinsfile.script(groovy.lang.Closure)
66+
IncrementalBuild_Jenkinsfile.buildDockerImage({manifest=tests/jenkins/data/opensearch-1.3.0.yml, dryRun=false})
67+
buildDockerImage.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
68+
buildDockerImage.legacySCM(groovy.lang.Closure)
69+
buildDockerImage.library({identifier=jenkins@20211123, retriever=null})
70+
buildDockerImage.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml})
71+
InputManifest.asBoolean()
72+
InputManifest.getPublicDistUrl(https://ci.opensearch.org/dbc, vars-build, 33, linux, x64)
73+
InputManifest.getPublicDistUrl(https://ci.opensearch.org/dbc, vars-build, 33, linux, arm64)
74+
buildDockerImage.httpRequest({httpMode=HEAD, url=https://ci.opensearch.org/dbc/vars-build/1.3.0/33/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz})
75+
buildDockerImage.httpRequest({httpMode=HEAD, url=https://ci.opensearch.org/dbc/vars-build/1.3.0/33/linux/arm64/dist/opensearch/opensearch-1.3.0-linux-arm64.tar.gz})
76+
buildDockerImage.echo(Skipping docker build.
77+
https://ci.opensearch.org/dbc/vars-build/1.3.0/33/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz (404)
78+
https://ci.opensearch.org/dbc/vars-build/1.3.0/33/linux/arm64/dist/opensearch/opensearch-1.3.0-linux-arm64.tar.gz (404))
79+
IncrementalBuild_Jenkinsfile.stage(docker, existing build, groovy.lang.Closure)
80+
IncrementalBuild_Jenkinsfile.script(groovy.lang.Closure)
81+
IncrementalBuild_Jenkinsfile.buildDockerImage({manifest=tests/jenkins/data/opensearch-1.3.0.yml, dryRun=true})
82+
buildDockerImage.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
83+
buildDockerImage.legacySCM(groovy.lang.Closure)
84+
buildDockerImage.library({identifier=jenkins@20211123, retriever=null})
85+
buildDockerImage.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml})
86+
InputManifest.asBoolean()
87+
InputManifest.getPublicDistUrl(https://ci.opensearch.org/dbc, vars-build, 33, linux, x64)
88+
InputManifest.getPublicDistUrl(https://ci.opensearch.org/dbc, vars-build, 33, linux, arm64)
89+
buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY, value=https://github.com/opensearch-project/opensearch-build})
90+
buildDockerImage.string({name=DOCKER_BUILD_GIT_REPOSITORY_REFERENCE, value=main})
91+
buildDockerImage.string({name=DOCKER_BUILD_SCRIPT_WITH_COMMANDS, value=id && pwd && cd docker/release && curl -sSL https://ci.opensearch.org/dbc/vars-build/1.3.0/33/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz -o opensearch-x64.tgz && curl -sSL https://ci.opensearch.org/dbc/vars-build/1.3.0/33/linux/arm64/dist/opensearch/opensearch-1.3.0-linux-arm64.tar.gz -o opensearch-arm64.tgz && bash build-image-multi-arch.sh -v 1.3.0 -f ./dockerfiles/opensearch.al2.dockerfile -p opensearch -a 'x64,arm64' -r opensearchstaging/opensearch -t 'opensearch-x64.tgz,opensearch-arm64.tgz' -n 33})
92+
buildDockerImage.booleanParam({name=IS_STAGING, value=true})
93+
buildDockerImage.echo(dockerBuild: { build job: 'docker-build', parameters: [null, null, null, null] })

0 commit comments

Comments
 (0)