Skip to content

Commit ce8c779

Browse files
yosifkityosifkit
authored
Merge pull request #16 from infosiftr/faster-sources
Update sources.sh to use bashbrew 0.1.10's `ArchGitChecksum`
2 parents 19ca057 + 59b029c commit ce8c779

File tree

6 files changed

+48
-153
lines changed

6 files changed

+48
-153
lines changed

‎.github/workflows/ci.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ jobs:
2525
run: |
2626
# not doing "uses: docker-library/bashbrew@xxx" because it'll build which is slow and we don't need more than just bashbrew here
2727
mkdir .bin
28-
wget -O .bin/bashbrew 'https://github.com/docker-library/bashbrew/releases/download/v0.1.9/bashbrew-amd64'
29-
echo '8cdd7adc707b972040577006f7a05b8e9d4dd362be5069f862fd1885f2eb107a *.bin/bashbrew' | sha256sum --strict --check -
28+
wget -O .bin/bashbrew 'https://github.com/docker-library/bashbrew/releases/download/v0.1.11/bashbrew-amd64'
29+
echo '6203635644d0efef2886f8ea9c487995a7abc4166db7a4773e94f89c943a4b04 *.bin/bashbrew' | sha256sum --strict --check -
3030
chmod +x .bin/bashbrew
3131
.bin/bashbrew --version
3232
echo "$PWD/.bin" >> "$GITHUB_PATH"

‎.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
11
builds
2-
tar-scrubber

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.21
44

55
require (
66
github.com/containerd/containerd v1.6.19
7-
github.com/docker-library/bashbrew v0.1.9
7+
github.com/docker-library/bashbrew v0.1.11
88
github.com/opencontainers/image-spec v1.1.0-rc2.0.20221013174636-8159c8264e2e
99
github.com/sirupsen/logrus v1.9.0
1010
golang.org/x/time v0.5.0

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ github.com/containerd/containerd v1.6.19/go.mod h1:HZCDMn4v/Xl2579/MvtOC2M206i+J
33
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
44
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
55
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6-
github.com/docker-library/bashbrew v0.1.9 h1:mDRel5sLJzpWbbnacZ2Y0RL6pRsCD/DI57vwEIX0uHQ=
7-
github.com/docker-library/bashbrew v0.1.9/go.mod h1:fp+ljAv22z5OK3k7gutU/1eZz0lIRnM6SSvfJd9ABEE=
6+
github.com/docker-library/bashbrew v0.1.11 h1:9S6jYFu0+RaqEAfvS2lh7jcaDkcvFi2maB2aU3yb0TM=
7+
github.com/docker-library/bashbrew v0.1.11/go.mod h1:6fyRRSm4vgBAgTw87EsfOT7wXKsc4JA9I5cdQJmwOm8=
88
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
99
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
1010
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=

‎sources.sh

+43-95
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,6 @@ defaultArchNamespaces='
2424
: "${BASHBREW_ARCH_NAMESPACES=$defaultArchNamespaces}"
2525
export BASHBREW_ARCH_NAMESPACES
2626

27-
dir="$(dirname "$BASH_SOURCE")"
28-
dir="$(readlink -ve "$dir")"
29-
if [ "$dir/tar-scrubber.go" -nt "$dir/tar-scrubber" ] || [ "$dir/.go-env.sh" -nt "$dir/tar-scrubber" ]; then
30-
{
31-
echo "building '$dir/tar-scrubber' from 'tar-scrubber.go'"
32-
"$dir/.go-env.sh" go build -v -o tar-scrubber tar-scrubber.go
33-
ls -l "$dir/tar-scrubber"
34-
} >&2
35-
fi
36-
[ -x "$dir/tar-scrubber" ]
37-
export tarScrubber="$dir/tar-scrubber"
38-
3927
# let's resolve all the external pins so we can inject those too
4028
libraryDir="${BASHBREW_LIBRARY:-"$HOME/docker/official-images/library"}"
4129
libraryDir="$(readlink -ve "$libraryDir")"
@@ -51,99 +39,58 @@ for tag in $externalPins; do
5139
externalPinsJson="$(jq <<<"$externalPinsJson" -c --arg tag "${tag#library/}" --arg digest "$digest" '.[$tag] = $digest')"
5240
done
5341

54-
_sha256() {
55-
sha256sum "$@" | cut -d' ' -f1
56-
}
57-
58-
json="$(
59-
bashbrew cat --build-order --format '
60-
{{- range $e := .SortedEntries false -}}
61-
{{- range $a := $e.Architectures -}}
62-
{{- $archNs := archNamespace $a -}}
63-
{{- with $e -}}
64-
{
65-
"repo": {{ $.RepoName | json }},
66-
"arch": {{ $a | json }},
67-
"platformString": {{ (ociPlatform $a).String | json }},
68-
"platform": {{ ociPlatform $a | json }},
69-
"gitCache": {{ gitCache | json }},
70-
"tags": {{ $.Tags namespace false . | json }},
71-
"archTags": {{ if $archNs -}} {{ $.Tags $archNs false . | json }} {{- else -}} [] {{- end }},
42+
bashbrew cat --build-order --format '
43+
{{- range $e := .SortedEntries false -}}
44+
{{- range $a := $e.Architectures -}}
45+
{{- $archNs := archNamespace $a -}}
46+
{{- with $e -}}
47+
{{- $sum := $.ArchGitChecksum $a . -}}
48+
{{- $file := .ArchFile $a -}}
49+
{{- $builder := .ArchBuilder $a -}}
50+
{
51+
"sourceId": {{ join "\n" $sum $file $builder "" | sha256sum | json }},
52+
"reproducibleGitChecksum": {{ $sum | json }},
53+
"allTags": [],
54+
"entry": {
7255
"GitRepo": {{ .ArchGitRepo $a | json }},
7356
"GitFetch": {{ .ArchGitFetch $a | json }},
7457
"GitCommit": {{ .ArchGitCommit $a | json }},
7558
"Directory": {{ .ArchDirectory $a | json }},
76-
"File": {{ .ArchFile $a | json }},
77-
"Builder": {{ .ArchBuilder $a | json }},
78-
"froms": {{ $.ArchDockerFroms $a . | json }}
59+
"File": {{ $file | json }},
60+
"Builder": {{ $builder | json }},
61+
"SOURCE_DATE_EPOCH": {{ ($.ArchGitTime $a .).Unix | json }}
62+
},
63+
"arches": {
64+
{{ $a | json }}: {
65+
"tags": {{ $.Tags namespace false . | json }},
66+
"archTags": {{ if $archNs -}} {{ $.Tags $archNs false . | json }} {{- else -}} [] {{- end }},
67+
"froms": {{ $.ArchDockerFroms $a . | json }},
68+
"platformString": {{ (ociPlatform $a).String | json }},
69+
"platform": {{ ociPlatform $a | json }},
70+
"parents": { }
71+
}
7972
}
80-
{{- end -}}
73+
}
8174
{{- end -}}
8275
{{- end -}}
83-
' "$@"
84-
)"
85-
86-
shell="$(
87-
jq <<<"$json" -r '
88-
. as $e
89-
| {
90-
reproducibleGitChecksum: (
91-
[
92-
# TODO do this inside bashbrew? (could then use go-git to make an even more determistic tarball instead of munging Git afterwards, and could even do things like munge the Dockerfile to remove no-rebuild variance like comments and non-COPY-ed files)
93-
"git -C \(.gitCache | @sh) archive --format=tar \(.GitCommit + ":" + (.Directory | if . == "." then "" else . + "/" end) | @sh)",
94-
"\(env.tarScrubber | @sh) --sha256",
95-
empty
96-
] | join(" | ")
97-
),
98-
sourceId: "printf \("%s\\n" | @sh) \"$reproducibleGitChecksum\" \(.File | @sh) \(.Builder | @sh) | _sha256", # the combination of things that might cause a rebuild # TODO consider making this a compressed JSON object like buildId
99-
SOURCE_DATE_EPOCH: "git -C \(.gitCache | @sh) show --no-patch --format=format:%ct \(.GitCommit | @sh)",
100-
}
101-
| to_entries
102-
| [
103-
"printf >&2 \("%s (%s): " | @sh) \($e.tags[0]) \($e.arch)",
104-
empty
105-
]
106-
+ map(.key + "=\"$(" + .value + ")\"")
107-
+ [
108-
"export \(map(.key) | join(" "))",
109-
"printf >&2 \("%s\\n" | @sh) \"$sourceId\"",
110-
(
111-
$e
112-
| {
113-
allTags: (.tags + .archTags),
114-
entry: {
115-
GitRepo: .GitRepo,
116-
GitFetch: .GitFetch,
117-
GitCommit: .GitCommit,
118-
Directory: .Directory,
119-
File: .File,
120-
Builder: .Builder,
121-
},
122-
arches: {
123-
(.arch): {
124-
tags: .tags,
125-
archTags: .archTags,
126-
froms: .froms,
127-
platformString: .platformString,
128-
platform: .platform,
129-
},
130-
},
131-
} as $obj
132-
| "jq <<<\($obj | tojson | @sh) -c \("{ sourceId: env.sourceId, reproducibleGitChecksum: env.reproducibleGitChecksum } + . | .entry.SOURCE_DATE_EPOCH = (env.SOURCE_DATE_EPOCH | tonumber)" | @sh)"
133-
),
134-
empty
135-
]
136-
| join("\n")
137-
'
138-
)"
139-
json="$(set -Eeuo pipefail; eval "$shell")"
140-
jq <<<"$json" -s --argjson pins "$externalPinsJson" '
76+
{{- end -}}
77+
' "$@" | jq 3>&1 1>&2 2>&3- -r '
78+
# https://github.com/jqlang/jq/issues/2063 - "stderr" cannot functionally output a string correctly until jq 1.7+ (which is very very recent), so we hack around it to get some progress output by using Bash to swap stdout and stderr so we can output our objects to stderr and our progress text to stdout and "fix it in post"
79+
# TODO balk / error at multiple arches entries
80+
first(.arches[].tags[], .arches[].archTags[]) as $tag
81+
| first(.arches | keys_unsorted[]) as $arch
82+
| stderr
83+
| "\($tag) (\($arch)): \(.sourceId)"
84+
# TODO if we could get jq 1.7+ for sure, we can drop this entire "jq" invocation and instead have the reduce loop of the following invocation print status strings directly to "stderr"
85+
' | jq -n --argjson pins "$externalPinsJson" '
14186
def unique_unsorted:
14287
# https://unix.stackexchange.com/a/738744/153467
14388
reduce .[] as $a ([]; if IN(.[]; $a) then . else . += [$a] end)
14489
;
145-
reduce .[] as $in ({};
146-
.[$in.sourceId] |= (
90+
reduce inputs as $in ({};
91+
# TODO if we can get support for joining two arrays directly in bashbrew (sprig?) then we can avoid this .arches[] nonsense
92+
($in | .allTags = ([ $in.arches[].tags[], $in.arches[].archTags[] ] | unique_unsorted)) as $in
93+
| .[$in.sourceId] |=
14794
if . == null then
14895
$in
14996
else
@@ -167,8 +114,9 @@ jq <<<"$json" -s --argjson pins "$externalPinsJson" '
167114
.entry = $in.entry
168115
else . end
169116
end
170-
)
171117
)
118+
# TODO a lot of this could be removed/parsed during the above reduce, since it has to parse things in build order anyhow
119+
# TODO actually, instead, this bit should be a totally separate script so the use case of "combine sources.json files together" works better 👀
172120
| (
173121
reduce to_entries[] as $e ({};
174122
$e.key as $sourceId

‎tar-scrubber.go

-52
This file was deleted.

0 commit comments

Comments
 (0)