Skip to content

Commit f08e50b

Browse files
committed
♻️ (CleanupDataFiles.ps1, ImagesToBlobStorage.ps1): enhance file deletion logging with progress tracking
Improve the logging mechanism in the file deletion scripts by adding progress tracking. This change provides better visibility into the deletion process by logging progress at defined intervals (every 10%). This is particularly useful for large datasets, allowing users to monitor the progress and ensuring that the script is functioning as expected. Additionally, the log messages are refined for clarity, indicating the total number of files processed and the completion status.
1 parent 289f1c2 commit f08e50b

File tree

2 files changed

+45
-12
lines changed

2 files changed

+45
-12
lines changed

.powershell/_includes/CleanupDataFiles.ps1

+23-7
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,36 @@ function Delete-LocalDataFiles {
77
)
88
$count = 0
99
try {
10-
Write-InfoLog "Deleting all data file files locally from '$LocalPath'..."
11-
$files = Get-ChildItem -Path $env:LOCAL_IMAGE_PATH -Recurse -Include data.captions.*.srt, data.captions.json, data.json, data.index.classifications.json
10+
Write-InfoLog "Deleting all data files locally from '$LocalPath'..."
11+
$files = Get-ChildItem -Path $LocalPath -Recurse -Include data.captions.*.srt, data.captions.json, data.json, data.index.classifications.json
1212
if ($files.Count -eq 0) {
1313
Write-InfoLog "No files found."
1414
return 0;
1515
}
16+
17+
$totalFiles = $files.Count
1618
$size = ($files | Measure-Object -Property Length -Sum).Sum
1719
$sizeString = "{0:N2} MB" -f ($size / 1MB)
18-
Write-InfoLog "Found ($($files.Count)) files totalling $sizeString."
19-
$files | ForEach-Object {
20+
Write-InfoLog "Found ($totalFiles) files totalling $sizeString."
21+
22+
$lastPercentage = 0 # To track when to log progress
23+
$progressInterval = 10 # Percentage interval for logging
24+
25+
$files | ForEach-Object -Begin { $index = 0 } -Process {
2026
try {
21-
$count++
2227
Remove-Item -Path $_.FullName -Force
2328
Write-DebugLog "Deleted: $($_.FullName)"
29+
$count++
30+
$index++
31+
32+
# Calculate percentage progress
33+
$percentage = [math]::Round(($index / $totalFiles) * 100, 0)
34+
35+
# Log progress at defined intervals (e.g., every 10%)
36+
if ($percentage -ge $lastPercentage + $progressInterval) {
37+
Write-InfoLog "Progress: $percentage% ($index of $totalFiles files deleted)"
38+
$lastPercentage = $percentage
39+
}
2440
}
2541
catch {
2642
Write-ErrorLog "Error deleting file $($_.FullName): $_"
@@ -30,6 +46,6 @@ function Delete-LocalDataFiles {
3046
catch {
3147
Write-ErrorLog "Error during file deletion: $_"
3248
}
33-
Write-InfoLog "Deleted: $count"
49+
Write-InfoLog "Completed: Deleted $count files."
3450
return $count;
35-
}
51+
}

.powershell/_includes/ImagesToBlobStorage.ps1

+22-5
Original file line numberDiff line numberDiff line change
@@ -34,27 +34,44 @@ function Delete-LocalImageFiles {
3434
Write-InfoLog "No image files found."
3535
return 0;
3636
}
37+
38+
$totalFiles = $images.Count
3739
$size = ($images | Measure-Object -Property Length -Sum).Sum
3840
$sizeString = "{0:N2} MB" -f ($size / 1MB)
39-
Write-InfoLog "Found ($($images.Count)) image files of $sizeString."
40-
$images | ForEach-Object {
41+
Write-InfoLog "Found ($totalFiles) image files totalling $sizeString."
42+
43+
$lastPercentage = 0 # Tracks when to log progress
44+
$progressInterval = 10 # Percentage interval for logging
45+
46+
$images | ForEach-Object -Begin { $index = 0 } -Process {
4147
try {
42-
$count++
4348
Remove-Item -Path $_.FullName -Force
4449
Write-DebugLog "Deleted: $($_.FullName)"
50+
$count++
51+
$index++
52+
53+
# Calculate percentage progress
54+
$percentage = [math]::Round(($index / $totalFiles) * 100, 0)
55+
56+
# Log progress at defined intervals
57+
if ($percentage -ge $lastPercentage + $progressInterval) {
58+
Write-InfoLog "Progress: $percentage% ($index of $totalFiles image files deleted)"
59+
$lastPercentage = $percentage
60+
}
4561
}
4662
catch {
4763
Write-ErrorLog "Error deleting file $($_.FullName): $_"
4864
}
4965
}
5066
}
5167
catch {
52-
Write-ErrorLog "Error during file deletion: $_"
68+
Write-ErrorLog "Error during image file deletion: $_"
5369
}
54-
Write-InfoLog "Deleted: $count"
70+
Write-InfoLog "Completed: Deleted $count image files."
5571
return $count;
5672
}
5773

74+
5875
# Method 3: Rewrite image links in .html files using regex
5976
function Rewrite-ImageLinks {
6077
param (

0 commit comments

Comments
 (0)