Skip to content

Commit 1539d3e

Browse files
gerhardolgerhardol
committed
Download check for cache directory
Add a marker .download file to validate the contents in cache directories. Previously only the existence of the directory was used, so if the download was aborted the cache directory had to be deleted manually if this occurred (with a likely cryptic error message). If the .download check file does not exist, the directory will be deleted and downloaded again. It is also possible to check the contents with a checksum. If not matching, the directory will be deleted and downloaded again. For Git repos the repos can be deleted if the status is not clean, a checksum is not relevant (but used in the tests).
1 parent 0bc73f4 commit 1539d3e

File tree

4 files changed

+279
-6
lines changed

4 files changed

+279
-6
lines changed

Diff for: ‎README.md

+14
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,12 @@ You can use `CPM_SOURCE_CACHE` on GitHub Actions workflows [cache](https://githu
196196
The directory where the version for a project is stored is by default the hash of the arguments to `CPMAddPackage()`.
197197
If for instance the patch command uses external files, the directory name can be set with the argument `CUSTOM_CACHE_KEY`.
198198

199+
It is possible to check the integrity of the downloaded content with a checksum by adding a [checksum command](test/unit/checksum_directory.sh) to `CPMAddPackage()`.
200+
Checksum validation can be done in two ways:
201+
202+
- Setting the option `CPM_CHECK_CACHE_CHECKSUM` to validate to the checksum calculated when downloading the project.
203+
- Providing the checksum in the call to `CPMAddPackage()`.
204+
199205
### CPM_DOWNLOAD_ALL
200206

201207
If set, CPM will forward all calls to `CPMFindPackage` as `CPMAddPackage`.
@@ -219,6 +225,14 @@ Note that this does not apply to dependencies that have been defined with a trut
219225
If set, CPM use additional directory level in cache to improve readability of packages names in IDEs like CLion. It changes cache structure, so all dependencies are downloaded again. There is no problem to mix both structures in one cache directory but then there may be 2 copies of some dependencies.
220226
This can also be set as an environmental variable.
221227

228+
### CPM_CHECK_CACHE_CHECKSUM
229+
230+
Enable validation of the checksum for a cache directory if a command to checksum the directory is provided. The validation is performed to a supplied checksum if provided, otherwise the checksum detected when downloading the dependency.
231+
232+
If `GIT_TAG` is set, `git-status` will check the status, checksum command is not required.
233+
234+
If the check fails, an existing directory will be deleted and downloaded again.
235+
222236
## Local package override
223237

224238
Library developers are often in the situation where they work on a locally checked out dependency at the same time as on a consumer project.

Diff for: ‎cmake/CPM.cmake

+83-6
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,11 @@ option(CPM_USE_NAMED_CACHE_DIRECTORIES
129129
"Use additional directory of package name in cache on the most nested level."
130130
$ENV{CPM_USE_NAMED_CACHE_DIRECTORIES}
131131
)
132+
option(
133+
CPM_CHECK_CACHE_CHECKSUM
134+
"If a package is stored in cache and there is a command to provide checksum, check the checksum when the cache dir exists."
135+
$ENV{CPM_CHECK_CACHE_CHECKSUM}
136+
)
132137

133138
set(CPM_VERSION
134139
${CURRENT_CPM_VERSION}
@@ -601,9 +606,10 @@ function(CPMAddPackage)
601606
EXCLUDE_FROM_ALL
602607
SOURCE_SUBDIR
603608
CUSTOM_CACHE_KEY
609+
CUSTOM_CACHE_CHECKSUM_VALUE
604610
)
605611

606-
set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND PATCHES)
612+
set(multiValueArgs URL OPTIONS DOWNLOAD_COMMAND PATCHES CUSTOM_CACHE_CHECKSUM_COMMAND)
607613

608614
cmake_parse_arguments(CPM_ARGS "" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}")
609615

@@ -789,15 +795,65 @@ function(CPMAddPackage)
789795
get_filename_component(download_directory ${download_directory} ABSOLUTE)
790796
list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${download_directory})
791797

792-
if(CPM_SOURCE_CACHE)
793-
file(LOCK ${download_directory}/../cmake.lock)
798+
file(LOCK ${download_directory}/../cmake.lock)
799+
800+
if(EXISTS ${download_directory} AND NOT EXISTS ${download_directory}.download)
801+
message(
802+
WARNING
803+
"Cache for ${CPM_ARGS_NAME} is missing .download, downloading. (${download_directory}.download)"
804+
)
805+
file(REMOVE_RECURSE ${download_directory})
794806
endif()
795807

796-
if(EXISTS ${download_directory})
797-
if(CPM_SOURCE_CACHE)
798-
file(LOCK ${download_directory}/../cmake.lock RELEASE)
808+
if(EXISTS ${download_directory}
809+
AND CPM_ARGS_CUSTOM_CACHE_CHECKSUM_COMMAND
810+
AND (CPM_CHECK_CACHE_CHECKSUM OR DEFINED CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE)
811+
)
812+
if(CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE)
813+
# Explicit checksum provided, ignore value in .downloaded
814+
set(expected_checksum ${CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE})
815+
else()
816+
file(READ ${download_directory}.download expected_checksum)
817+
string(STRIP "${expected_checksum}" expected_checksum)
799818
endif()
800819

820+
if(expected_checksum)
821+
execute_process(
822+
COMMAND ${CPM_ARGS_CUSTOM_CACHE_CHECKSUM_COMMAND}
823+
WORKING_DIRECTORY ${download_directory}
824+
OUTPUT_VARIABLE checksum
825+
OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY
826+
)
827+
if(NOT expected_checksum STREQUAL checksum)
828+
message(
829+
WARNING
830+
"Checksum mismatch for ${CPM_ARGS_NAME}, removing (${expected_checksum} != ${checksum})"
831+
)
832+
file(REMOVE_RECURSE ${download_directory})
833+
endif()
834+
else()
835+
message(
836+
WARNING
837+
"Checksum cannot be verified for ${CPM_ARGS_NAME}, no existing value (${expected_checksum})"
838+
)
839+
endif()
840+
endif()
841+
if(EXISTS ${download_directory}
842+
AND DEFINED CPM_ARGS_GIT_TAG
843+
AND NOT (PATCH_COMMAND IN_LIST CPM_ARGS_UNPARSED_ARGUMENTS)
844+
)
845+
# warn if cache has been changed since checkout
846+
cpm_check_git_working_dir_is_clean(${download_directory} ${CPM_ARGS_GIT_TAG} IS_CLEAN)
847+
if(NOT ${IS_CLEAN})
848+
message(WARNING "${CPM_INDENT} Cache for ${CPM_ARGS_NAME} (${download_directory}) is dirty")
849+
if(CPM_CHECK_CACHE_CHECKSUM OR DEFINED CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE)
850+
file(REMOVE_RECURSE ${download_directory})
851+
endif()
852+
endif()
853+
endif()
854+
if(EXISTS ${download_directory})
855+
# Directory content is considered OK
856+
file(LOCK ${download_directory}/../cmake.lock RELEASE)
801857
cpm_store_fetch_properties(
802858
${CPM_ARGS_NAME} "${download_directory}"
803859
"${CPM_FETCHCONTENT_BASE_DIR}/${lower_case_name}-build"
@@ -894,6 +950,27 @@ function(CPMAddPackage)
894950

895951
cpm_fetch_package("${CPM_ARGS_NAME}" ${DOWNLOAD_ONLY} populated ${CPM_ARGS_UNPARSED_ARGUMENTS})
896952
if(CPM_SOURCE_CACHE AND download_directory)
953+
if(${populated})
954+
if(CPM_ARGS_CUSTOM_CACHE_CHECKSUM_COMMAND)
955+
execute_process(
956+
COMMAND ${CPM_ARGS_CUSTOM_CACHE_CHECKSUM_COMMAND}
957+
WORKING_DIRECTORY ${download_directory}
958+
OUTPUT_VARIABLE checksum
959+
OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY
960+
)
961+
if(CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE AND NOT CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE
962+
STREQUAL checksum
963+
)
964+
message(
965+
FATAL_ERROR
966+
"Checksum mismatch for ${CPM_ARGS_NAME} (${CPM_ARGS_CUSTOM_CACHE_CHECKSUM_VALUE} != ${checksum})"
967+
)
968+
endif()
969+
else()
970+
set(checksum "")
971+
endif()
972+
file(WRITE ${download_directory}.download ${checksum})
973+
endif()
897974
file(LOCK ${download_directory}/../cmake.lock RELEASE)
898975
endif()
899976
if(${populated} AND ${CMAKE_VERSION} VERSION_LESS "3.28.0")

Diff for: ‎test/unit/cache.cmake

+125
Original file line numberDiff line numberDiff line change
@@ -153,3 +153,128 @@ execute_process(
153153

154154
assert_equal(${ret} "0")
155155
assert_exists("${CPM_SOURCE_CACHE_DIR}/fibonacci/my_custom_unique_dir")
156+
157+
# Cache checksum
158+
159+
reset_test()
160+
set(FIBONACCI_VERSION 1.1)
161+
set(FIBONACCI_GIT_TAG "GIT_TAG e9ebf168ca0fffaa4ef8c6fefc6346aaa22f6ed5")
162+
set(TEST_CHECKSUM_DIR "${CPM_SOURCE_CACHE_DIR}/fibonacci/my_checksummed_dir")
163+
set(TEST_CHECKSUM_VALUE
164+
d2e7c040116d3f4f153eee84cb884fd5008a31b480b739bf86d3872d978cfaa74d82e07581a1861fdb07e717be5b658eb520e2f4e29fbf0ce248bfef478c1971
165+
)
166+
167+
# OK download
168+
169+
set(FIBONACCI_PACKAGE_ARGS
170+
"${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh"
171+
)
172+
update_cmake_lists()
173+
174+
execute_process(
175+
COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
176+
"-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
177+
)
178+
179+
assert_equal(${ret} "0")
180+
assert_exists("${TEST_CHECKSUM_DIR}.download")
181+
file(READ "${TEST_CHECKSUM_DIR}.download" chksum)
182+
assert_equal("${chksum}" "${TEST_CHECKSUM_VALUE}")
183+
184+
# Test download again if .download file is missing
185+
186+
file(REMOVE "${TEST_CHECKSUM_DIR}.download")
187+
file(REMOVE "${TEST_CHECKSUM_DIR}/include/fibonacci.h")
188+
189+
set(FIBONACCI_PACKAGE_ARGS
190+
"${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh"
191+
)
192+
update_cmake_lists()
193+
194+
execute_process(
195+
COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
196+
"-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
197+
)
198+
199+
assert_equal(${ret} "0")
200+
assert_exists("${TEST_CHECKSUM_DIR}.download")
201+
assert_exists("${TEST_CHECKSUM_DIR}/include/fibonacci.h")
202+
203+
# check checksum for download
204+
205+
set(FIBONACCI_PACKAGE_ARGS
206+
"${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh"
207+
)
208+
update_cmake_lists()
209+
210+
execute_process(
211+
COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
212+
"-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
213+
)
214+
215+
assert_equal(${ret} "0")
216+
217+
# check checksum for download, provided
218+
219+
set(FIBONACCI_PACKAGE_ARGS
220+
"${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh CUSTOM_CACHE_CHECKSUM_VALUE ${TEST_CHECKSUM_VALUE}"
221+
)
222+
update_cmake_lists()
223+
224+
execute_process(
225+
COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
226+
"-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
227+
)
228+
229+
assert_equal(${ret} "0")
230+
231+
# check checksum for download, provided incorrect, this will print a fatal_error (red) error to the
232+
# console
233+
234+
set(FIBONACCI_PACKAGE_ARGS
235+
"${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh CUSTOM_CACHE_CHECKSUM_VALUE invalid_checksum_value"
236+
)
237+
update_cmake_lists()
238+
239+
execute_process(
240+
COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
241+
"-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
242+
)
243+
244+
assert_equal(${ret} "1")
245+
246+
# redownload when checksum is changed
247+
248+
set(FIBONACCI_PACKAGE_ARGS
249+
"${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_COMMAND ${CMAKE_CURRENT_LIST_DIR}/checksum_directory.sh CUSTOM_CACHE_CHECKSUM_VALUE ${TEST_CHECKSUM_VALUE}"
250+
)
251+
update_cmake_lists()
252+
253+
# dummy change, to trigger checksum mismatch
254+
file(WRITE "${TEST_CHECKSUM_DIR}/fail_checksum.txt" "dummy")
255+
256+
execute_process(
257+
COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
258+
"-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
259+
)
260+
261+
assert_equal(${ret} "0")
262+
assert_not_exists("${TEST_CHECKSUM_DIR}/fail_checksum.txt")
263+
264+
# redownload when checksum is changed
265+
266+
set(FIBONACCI_PACKAGE_ARGS
267+
"${FIBONACCI_GIT_TAG} CUSTOM_CACHE_KEY my_checksummed_dir CUSTOM_CACHE_CHECKSUM_VALUE ${TEST_CHECKSUM_VALUE}"
268+
)
269+
update_cmake_lists()
270+
271+
# dummy change, to trigger checksum mismatch
272+
file(WRITE "${TEST_CHECKSUM_DIR}/fail_checksum.txt" "dummy")
273+
274+
execute_process(
275+
COMMAND ${CMAKE_COMMAND} -E env "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE_DIR}" ${CMAKE_COMMAND}
276+
"-S${CMAKE_CURRENT_LIST_DIR}/remote_dependency" "-B${TEST_BUILD_DIR}" RESULT_VARIABLE ret
277+
)
278+
279+
assert_equal(${ret} "0")
280+
assert_not_exists("${TEST_CHECKSUM_DIR}/fail_checksum.txt")

Diff for: ‎test/unit/checksum_directory.sh

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/usr/bin/env bash
2+
3+
# Script to checksum contents recursively in a directory
4+
5+
set -o errexit
6+
set -o nounset
7+
8+
function usage {
9+
echo
10+
echo "Checksum the contents of a directory"
11+
echo "Usage: $0 [-d <directory>]"
12+
echo ""
13+
echo " -d directory Default '.'"
14+
echo " -h Help, this message"
15+
echo " -t Use alternative tar method (requires zstd binary)"
16+
echo " -v Verbose output"
17+
}
18+
19+
dir=.
20+
use_tar=
21+
# sha512 is faster than sha256 for large files, sha1 is even faster
22+
SHA_ALGORITHM=sha512sum
23+
24+
while getopts "d:htv" o; do
25+
case "${o}" in
26+
d)
27+
dir=${OPTARG}
28+
;;
29+
h)
30+
usage
31+
exit 0
32+
;;
33+
t)
34+
use_tar=1
35+
;;
36+
v)
37+
set -x
38+
;;
39+
*)
40+
echo "Incorrect argument switch"
41+
usage
42+
exit 1
43+
;;
44+
esac
45+
done
46+
shift "$((OPTIND-1))"
47+
48+
cd $dir
49+
if [ ! -z $use_tar ]; then
50+
# This is faster for single threads but requires more memory and requires the separate zstd binary
51+
# For a 3 GB data this is 3s vs 'find' below: 5s (one thread) below, 2.5s with 28 threads, 0.7s with 100 files on each line
52+
# Without --fast, just ZSTD_CLEVEL=1 ZSTD_NBTHREADS=0 is about 6s
53+
tar -I "zstd --fast -1 -T0" -cf - . | $SHA_ALGORITHM | cut -f1 -d ' '
54+
else
55+
# In general, there is no point in checksumming Git repos, filter .git here as this is used in tests
56+
find . \( -name .git -prune \) -o -type f -print0 | xargs -n 100 --max-procs=$(nproc) -0 $SHA_ALGORITHM | sort -k 2 | $SHA_ALGORITHM | cut -f1 -d ' '
57+
fi

0 commit comments

Comments
 (0)