Skip to content

Commit c13626f

Browse files
IonutMuthiIonutMuthi
committed
ci: implement cache logic for macos ci
Signed-off-by: IonutMuthi <[email protected]>
1 parent e90ebaf commit c13626f

File tree

2 files changed

+185
-20
lines changed

2 files changed

+185
-20
lines changed

‎azure-pipelines.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,35 @@ stages:
4141
- checkout: self
4242
fetchDepth: 0
4343
clean: true
44+
45+
# Homebrew Cache
46+
- task: Cache@2
47+
inputs:
48+
key: 'homebrew-v1 | "$(vmImage)" | ci/macOS/install_macos_deps.sh | ci/macOS/macos_config.sh | "$(date +%Y-%m)"'
49+
path: '$(Pipeline.Workspace)/.homebrew-cache'
50+
cacheHitVar: 'HOMEBREW_CACHE'
51+
displayName: 'Cache Homebrew Downloads'
52+
condition: or(eq(variables['vmImage'], 'macOS-14'), eq(variables['vmImage'], 'macOS-15'))
53+
54+
# Git Repositories Cache
55+
- task: Cache@2
56+
inputs:
57+
key: 'git-repos-v1 | ci/macOS/macos_config.sh | "$(date +%Y-%m)"'
58+
path: '$(Pipeline.Workspace)/.git-cache'
59+
cacheHitVar: 'GIT_REPOS_CACHE'
60+
displayName: 'Cache Git Repositories'
61+
condition: or(eq(variables['vmImage'], 'macOS-14'), eq(variables['vmImage'], 'macOS-15'))
62+
63+
# Built Dependencies Cache
64+
- task: Cache@2
65+
inputs:
66+
key: 'built-deps-v1 | "$(vmImage)" | ci/macOS/macos_config.sh | ci/macOS/install_macos_deps.sh | "$(date +%Y-%m)"'
67+
path: '$(Build.Repository.LocalPath)/staging/dependencies'
68+
cacheHitVar: 'BUILT_DEPS_CACHE'
69+
displayName: 'Cache Built Dependencies'
70+
condition: or(eq(variables['vmImage'], 'macOS-14'), eq(variables['vmImage'], 'macOS-15'))
71+
72+
4473
- script: ./ci/macOS/install_macos_deps.sh
4574
displayName: 'Build and Install Dependencies'
4675
workingDirectory: $(Build.Repository.LocalPath)

‎ci/macOS/install_macos_deps.sh‎

Lines changed: 156 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,94 @@ set -ex
44
REPO_SRC=$(git rev-parse --show-toplevel)
55
source $REPO_SRC/ci/macOS/macos_config.sh
66

7+
# Cache configuration
8+
CACHE_BASE_DIR="${PIPELINE_WORKSPACE}"
9+
GIT_CACHE_DIR="${CACHE_BASE_DIR}/.git-cache"
10+
HOMEBREW_CACHE_DIR="${CACHE_BASE_DIR}/.homebrew-cache"
11+
CACHE_SIZE_WARNING_GB=8
12+
713
PACKAGES="${QT_FORMULAE} volk spdlog ${BOOST_FORMULAE} pkg-config cmake fftw bison gettext autoconf automake libzip glib libusb glog doxygen wget gnu-sed libmatio dylibbundler libxml2 ghr libsndfile"
814

15+
# Cache size monitoring
16+
check_cache_sizes() {
17+
if [ -d "$CACHE_BASE_DIR" ]; then
18+
local total_size_gb=$(du -sg "$CACHE_BASE_DIR" 2>/dev/null | cut -f1 || echo "0")
19+
echo "Total cache size: ${total_size_gb}GB"
20+
21+
if [ "$total_size_gb" -gt "$CACHE_SIZE_WARNING_GB" ]; then
22+
echo "⚠️ WARNING: Cache size (${total_size_gb}GB) approaching Azure DevOps limits"
23+
fi
24+
fi
25+
}
26+
27+
28+
# Cache cleanup function
29+
cleanup_old_caches() {
30+
echo "Cleaning up old cache directories..."
31+
32+
# Remove any temporary cache directories older than 7 days
33+
if [ -d "$CACHE_BASE_DIR" ]; then
34+
find "$CACHE_BASE_DIR" -name "*.tmp" -type d -mtime +7 -exec rm -rf {} + 2>/dev/null || true
35+
find "$CACHE_BASE_DIR" -name "*.old" -type d -mtime +7 -exec rm -rf {} + 2>/dev/null || true
36+
fi
37+
}
38+
39+
setup_homebrew_cache() {
40+
echo "Setting up Homebrew cache..."
41+
mkdir -p "$HOMEBREW_CACHE_DIR"
42+
43+
export HOMEBREW_CACHE="$HOMEBREW_CACHE_DIR"
44+
#check if cache for homebrew exists
45+
if [ "${HOMEBREW_CACHE}" = "true" ]; then
46+
echo "Homebrew cache found, restoring cached packages"
47+
export HOMEBREW_NO_AUTO_UPDATE=1
48+
else
49+
echo "No Homebrew cache - downloading fresh packages"
50+
fi
51+
}
52+
53+
setup_git_cache() {
54+
echo "Setting up Git cache..."
55+
mkdir -p "$GIT_CACHE_DIR"
56+
57+
if [ "${GIT_REPOS_CACHE}" = "true" ]; then
58+
echo "Git repositories cache found, restoring cached repositories"
59+
# Validate cache isn't corrupted
60+
if [ -d "$GIT_CACHE_DIR" ] && [ "$(ls -A $GIT_CACHE_DIR 2>/dev/null)" ]; then
61+
export GIT_CACHE_ENABLED=true
62+
else
63+
echo "Git cache directory empty - clearing and rebuilding"
64+
rm -rf "$GIT_CACHE_DIR"
65+
mkdir -p "$GIT_CACHE_DIR"
66+
export GIT_CACHE_ENABLED=false
67+
fi
68+
else
69+
echo "No Git cache - cloning fresh repositories"
70+
export GIT_CACHE_ENABLED=false
71+
fi
72+
}
73+
74+
setup_dependencies_cache() {
75+
echo "Setting up dependencies cache..."
76+
77+
if [ "${BUILT_DEPS_CACHE}" = "true" ]; then
78+
echo "Built dependencies cache found, restoring cached dependencies"
79+
# Validate cache isn't corrupted
80+
if [ -d "$STAGING_AREA_DEPS" ] && [ "$(ls -A $STAGING_AREA_DEPS 2>/dev/null)" ]; then
81+
echo "Found cached dependencies in $STAGING_AREA_DEPS"
82+
export DEPENDENCIES_CACHED=true
83+
else
84+
echo "Dependencies cache directory empty or corrupted - clearing and rebuilding"
85+
rm -rf "$STAGING_AREA_DEPS"
86+
export DEPENDENCIES_CACHED=false
87+
fi
88+
else
89+
echo "No dependencies cache - building fresh"
90+
export DEPENDENCIES_CACHED=false
91+
fi
92+
93+
}
94+
995
OS_VERSION=${1:-$(sw_vers -productVersion)}
1096
echo "MacOS version $OS_VERSION"
1197

@@ -24,20 +110,28 @@ install_packages() {
24110
# uninstall cmake before installing other dependencies https://github.com/actions/runner-images/issues/12912
25111
brew uninstall --force cmake || true
26112

27-
brew update
28113
# Workaround for brew taking a long time to upgrade existing packages
29114
# Check if macOS version and upgrade packages only if the version is greater than macOS 12
30115
macos_version=$(sw_vers -productVersion)
31116
major_version=$(echo "$macos_version" | cut -d '.' -f 1)
32-
if [ "$major_version" -gt 12 ]; then
33-
brew upgrade --display-times || true #ignore homebrew upgrade errors
34-
# Workaround: Install or update libtool package only if macOS version is greater than 12
35-
# Note: libtool (v2.4.7) is pre-installed by default, but it can be updated to v2.5.3
36-
PACKAGES="$PACKAGES libtool"
37-
else
117+
118+
# Package installation based on cache status
119+
if [ "${HOMEBREW_CACHE}" = "true" ]; then
120+
echo "Installing packages with cache optimization (skipping update/upgrade)..."
38121
export HOMEBREW_NO_AUTO_UPDATE=1
122+
else
123+
echo "Installing packages fresh..."
124+
brew update
125+
126+
if [ "$major_version" -gt 12 ]; then
127+
brew upgrade --display-times || true #ignore homebrew upgrade errors
128+
# Workaround: Install or update libtool package only if macOS version is greater than 12
129+
# Note: libtool (v2.4.7) is pre-installed by default, but it can be updated to v2.5.3
130+
PACKAGES="$PACKAGES libtool"
131+
fi
39132
fi
40133

134+
41135
brew install --overwrite --display-times $PACKAGES
42136

43137
pip3 install --break-system-packages mako
@@ -69,19 +163,47 @@ clone() {
69163
echo "#######CLONE#######"
70164
mkdir -p $STAGING_AREA
71165
pushd $STAGING_AREA
72-
git clone --recursive https://github.com/sigrokproject/libserialport -b $LIBSERIALPORT_BRANCH libserialport
73-
git clone --recursive https://github.com/analogdevicesinc/libiio.git -b $LIBIIO_VERSION libiio
74-
git clone --recursive https://github.com/analogdevicesinc/libad9361-iio.git -b $LIBAD9361_BRANCH libad9361
75-
git clone --recursive https://github.com/analogdevicesinc/libm2k.git -b $LIBM2K_BRANCH libm2k
76-
git clone --recursive https://github.com/analogdevicesinc/gr-scopy.git -b $GRSCOPY_BRANCH gr-scopy
77-
git clone --recursive https://github.com/analogdevicesinc/gr-m2k.git -b $GRM2K_BRANCH gr-m2k
78-
git clone --recursive https://github.com/analogdevicesinc/gnuradio.git -b $GNURADIO_BRANCH gnuradio
79-
git clone --recursive https://github.com/cseci/qwt.git -b $QWT_BRANCH qwt
80-
git clone --recursive https://github.com/sigrokproject/libsigrokdecode.git -b $LIBSIGROKDECODE_BRANCH libsigrokdecode
81-
git clone --recursive https://github.com/analogdevicesinc/libtinyiiod.git -b $LIBTINYIIOD_BRANCH libtinyiiod
82-
git clone --recursive https://github.com/KDAB/KDDockWidgets.git -b $KDDOCK_BRANCH KDDockWidgets
83-
git clone --recursive https://github.com/KDE/extra-cmake-modules.git -b $ECM_BRANCH extra-cmake-modules
84-
git clone --recursive https://github.com/KDE/karchive.git -b $KARCHIVE_BRANCH karchive
166+
167+
if [ "$GIT_CACHE_ENABLED" = "true" ]; then
168+
echo "Using cached repositories..."
169+
170+
# If cache directory has content, use it
171+
if [ -d "$GIT_CACHE_DIR" ] && [ "$(ls -A $GIT_CACHE_DIR 2>/dev/null)" ]; then
172+
echo "Copying cached repositories"
173+
cp -r "$GIT_CACHE_DIR"/* .
174+
else
175+
echo "Git cache directory empty - cloning fresh"
176+
export GIT_CACHE_ENABLED=false
177+
fi
178+
fi
179+
180+
if [ "$GIT_CACHE_ENABLED" = "false" ]; then
181+
echo "Cloning all repositories fresh..."
182+
183+
git clone --recursive https://github.com/sigrokproject/libserialport -b $LIBSERIALPORT_BRANCH libserialport
184+
git clone --recursive https://github.com/analogdevicesinc/libiio.git -b $LIBIIO_VERSION libiio
185+
git clone --recursive https://github.com/analogdevicesinc/libad9361-iio.git -b $LIBAD9361_BRANCH libad9361
186+
git clone --recursive https://github.com/analogdevicesinc/libm2k.git -b $LIBM2K_BRANCH libm2k
187+
git clone --recursive https://github.com/analogdevicesinc/gr-scopy.git -b $GRSCOPY_BRANCH gr-scopy
188+
git clone --recursive https://github.com/analogdevicesinc/gr-m2k.git -b $GRM2K_BRANCH gr-m2k
189+
git clone --recursive https://github.com/analogdevicesinc/gnuradio.git -b $GNURADIO_BRANCH gnuradio
190+
git clone --recursive https://github.com/cseci/qwt.git -b $QWT_BRANCH qwt
191+
git clone --recursive https://github.com/sigrokproject/libsigrokdecode.git -b $LIBSIGROKDECODE_BRANCH libsigrokdecode
192+
git clone --recursive https://github.com/analogdevicesinc/libtinyiiod.git -b $LIBTINYIIOD_BRANCH libtinyiiod
193+
git clone --recursive https://github.com/KDAB/KDDockWidgets.git -b $KDDOCK_BRANCH KDDockWidgets
194+
git clone --recursive https://github.com/KDE/extra-cmake-modules.git -b $ECM_BRANCH extra-cmake-modules
195+
git clone --recursive https://github.com/KDE/karchive.git -b $KARCHIVE_BRANCH karchive
196+
197+
DEPENDENCY_REPOS="libserialport libiio libad9361 libm2k gr-scopy gr-m2k gnuradio qwt libsigrokdecode libtinyiiod KDDockWidgets extra-cmake-modules karchive"
198+
# Save to cache for next time
199+
if [ -n "$GIT_CACHE_DIR" ]; then
200+
mkdir -p "$GIT_CACHE_DIR"
201+
for repo in $DEPENDENCY_REPOS; do
202+
cp -r "$repo" "$GIT_CACHE_DIR/"
203+
done
204+
fi
205+
fi
206+
85207
popd
86208
}
87209

@@ -342,6 +464,13 @@ build_karchive () {
342464
}
343465

344466
build_deps(){
467+
if [ "$DEPENDENCIES_CACHED" = "true" ]; then
468+
echo "Found cached dependencies in $STAGING_AREA_DEPS"
469+
return 0
470+
fi
471+
472+
echo "Building all dependencies from source..."
473+
345474
build_libserialport
346475
build_libiio
347476
build_libad9361
@@ -357,6 +486,13 @@ build_deps(){
357486
build_karchive
358487
}
359488

489+
# Setup cache management
490+
cleanup_old_caches
491+
setup_homebrew_cache
492+
setup_git_cache
493+
setup_dependencies_cache
494+
check_cache_sizes
495+
360496
install_packages
361497
export_paths
362498
clone

0 commit comments

Comments
 (0)