db: incorporate blob files into EstimateDiskUsage

This patch ensures that blob files are also included
in disk usage estimation.

Fixes: #4621

Author: annrpom

db.go

@@ -501,9 +501,12 @@ type DB struct {
 		// annotators contains various instances of manifest.Annotator which
 		// should be protected from concurrent access.
 		annotators struct {
-			totalSize    *manifest.Annotator[uint64]
-			remoteSize   *manifest.Annotator[uint64]
-			externalSize *manifest.Annotator[uint64]
+			// totalFileSize is the sum of the size of all files in the
+			// database. This includes local, remote, and external sstables --
+			// along with blob files.
+			totalFileSize *manifest.Annotator[uint64]
+			remoteSize    *manifest.Annotator[uint64]
+			externalSize  *manifest.Annotator[uint64]
 		}
 	}
 
@@ -2375,24 +2378,30 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) {
 	return destLevels, nil
 }
 
-// makeFileSizeAnnotator returns an annotator that computes the total size of
-// files that meet some criteria defined by filter.
+// makeFileSizeAnnotator returns an annotator that computes the total
+// storage size of files that meet some criteria defined by filter. When
+// applicable, this includes both the sstable size and the size of any
+// referenced blob files.
 func (d *DB) makeFileSizeAnnotator(filter func(f *tableMetadata) bool) *manifest.Annotator[uint64] {
 	return &manifest.Annotator[uint64]{
 		Aggregator: manifest.SumAggregator{
 			AccumulateFunc: func(f *tableMetadata) (uint64, bool) {
 				if filter(f) {
-					return f.Size, true
+					return f.Size + f.EstimatedReferenceSize(), true
 				}
 				return 0, true
 			},
 			AccumulatePartialOverlapFunc: func(f *tableMetadata, bounds base.UserKeyBounds) uint64 {
 				if filter(f) {
-					size, err := d.fileCache.estimateSize(f, bounds.Start, bounds.End.Key)
+					overlappingFileSize, err := d.fileCache.estimateSize(f, bounds.Start, bounds.End.Key)
 					if err != nil {
 						return 0
 					}
-					return size
+					overlapFraction := float64(overlappingFileSize) / float64(f.Size)
+					// Scale the blob reference size proportionally to the file
+					// overlap from the bounds to approximate only the blob
+					// references that overlap with the requested bounds.
+					return overlappingFileSize + uint64(float64(f.EstimatedReferenceSize())*overlapFraction)
 				}
 				return 0
 			},
@@ -2438,9 +2447,10 @@ func (d *DB) EstimateDiskUsageByBackingType(
 	readState := d.loadReadState()
 	defer readState.unref()
 
-	totalSize = *d.mu.annotators.totalSize.VersionRangeAnnotation(readState.current, bounds)
+	totalSize = *d.mu.annotators.totalFileSize.VersionRangeAnnotation(readState.current, bounds)
 	remoteSize = *d.mu.annotators.remoteSize.VersionRangeAnnotation(readState.current, bounds)
 	externalSize = *d.mu.annotators.externalSize.VersionRangeAnnotation(readState.current, bounds)
+
 	return
 }
 

open.go

@@ -417,7 +417,7 @@ func Open(dirname string, opts *Options) (db *DB, err error) {
 	d.newIters = d.fileCache.newIters
 	d.tableNewRangeKeyIter = tableNewRangeKeyIter(d.newIters)
 
-	d.mu.annotators.totalSize = d.makeFileSizeAnnotator(func(f *manifest.TableMetadata) bool {
+	d.mu.annotators.totalFileSize = d.makeFileSizeAnnotator(func(f *manifest.TableMetadata) bool {
 		return true
 	})
 	d.mu.annotators.remoteSize = d.makeFileSizeAnnotator(func(f *manifest.TableMetadata) bool {

tool/testdata/db_space

@@ -2,37 +2,49 @@ db space
 ----
 accepts 1 arg(s), received 0
 
-# covers the whole 4.sst
-
+# Covers the whole 4.sst.
 db space --start=a --end=z
 ../testdata/db-stage-4
 ----
 709
 
-# covers from left of 4.sst to its only data block
-
+# Covers from left of 4.sst to its only data block.
 db space --start=a --end=bar
 ../testdata/db-stage-4
 ----
 62
 
-# covers from 4.sst's only data block to its right
-
+# Covers from 4.sst's only data block to its right.
 db space --start=foo --end=z
 ../testdata/db-stage-4
 ----
 62
 
-# covers non-overlapping range to left of 4.sst
-
+# Covers non-overlapping range to left of 4.sst.
 db space --start=a --end=a
 ../testdata/db-stage-4
 ----
 0
 
-# covers non-overlapping range to right of 4.sst
-
+# Covers non-overlapping range to right of 4.sst.
 db space --start=z --end=z
 ../testdata/db-stage-4
 ----
 0
+
+# Covers the whole 000005.sst and referenced blob file 000006.blob. The size of
+# the referenced blob file is 74 bytes. Note that this sst only has 1 data block
+# with in bounds [a, d].
+db space --start=a --end=z
+testdata/find-db-val-sep
+----
+888
+
+# Covers part of 000005.sst and part of referenced blob file 000006.blob. Note 
+# that the estimated size of the sst from [c, d] is the size of the entire data 
+# block + the estimated partial size of 000006.blob (14 bytes as more bytes are 
+# in [a, b]).
+db space --start=c --end=d
+testdata/find-db-val-sep
+----
+173