CLI Quiet Mode and Custom Progress Writer

.gitignore

@@ -6,3 +6,4 @@ go-pmtiles
 *.json
 *.geojson
 *.tsv.gz
+examples/*.pmtiles

examples/custom_progress.go

@@ -0,0 +1,143 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/protomaps/go-pmtiles/pmtiles"
+)
+
+// CloudProgress represents a custom progress writer
+type CloudProgress struct {
+	serviceName string
+}
+
+// CustomProgress tracks progress for individual operations
+type CustomProgress struct {
+	serviceName string
+	operation   string
+	total       int64
+	current     int64
+	startTime   time.Time
+}
+
+// NewCountProgress creates a progress tracker for count-based operations
+func (c *CloudProgress) NewCountProgress(total int64, description string) pmtiles.Progress {
+	fmt.Printf("[%s] Starting operation: %s (total items: %d)\n", c.serviceName, description, total)
+	return &CustomProgress{
+		serviceName: c.serviceName,
+		operation:   description,
+		total:       total,
+		current:     0,
+		startTime:   time.Now(),
+	}
+}
+
+// NewBytesProgress creates a progress tracker for byte-based operations
+func (c *CloudProgress) NewBytesProgress(total int64, description string) pmtiles.Progress {
+	fmt.Printf("[%s] Starting operation: %s (total bytes: %s)\n",
+		c.serviceName, description, formatBytes(total))
+	return &CustomProgress{
+		serviceName: c.serviceName,
+		operation:   description,
+		total:       total,
+		current:     0,
+		startTime:   time.Now(),
+	}
+}
+
+// Write implements io.Writer for byte-based progress tracking
+func (p *CustomProgress) Write(data []byte) (int, error) {
+	p.current += int64(len(data))
+	p.reportProgress()
+
+	// Here you would typically send the progress update to some external service
+
+	return len(data), nil
+}
+
+// Add increments the progress counter for count-based operations
+func (p *CustomProgress) Add(num int) {
+	p.current += int64(num)
+	p.reportProgress()
+
+	// Send progress update to outside service here if needed
+}
+
+// Close finalizes the progress tracking
+func (p *CustomProgress) Close() error {
+	duration := time.Since(p.startTime)
+	fmt.Printf("[%s] Completed: %s in %v\n", p.serviceName, p.operation, duration.Round(time.Millisecond))
+
+	// Send completion notification to outside service here if needed
+	return nil
+}
+
+// reportProgress displays current progress and could send updates to cloud services
+func (p *CustomProgress) reportProgress() {
+	if p.total <= 0 {
+		return
+	}
+
+	percentage := float64(p.current) / float64(p.total) * 100
+	elapsed := time.Since(p.startTime)
+
+	// Create a simple progress bar
+	barWidth := 30
+	filled := int(percentage / 100 * float64(barWidth))
+	bar := strings.Repeat("█", filled) + strings.Repeat("░", barWidth-filled)
+
+	if strings.Contains(p.operation, "bytes") || p.total > 1000 {
+		// Byte-based progress
+		fmt.Printf("[%s] %s: %.1f%% [%s] %s/%s (%.2fs)\n",
+			p.serviceName, p.operation, percentage, bar,
+			formatBytes(p.current), formatBytes(p.total), elapsed.Seconds())
+	} else {
+		// Count-based progress
+		fmt.Printf("[%s] %s: %.1f%% [%s] %d/%d items (%.2fs)\n",
+			p.serviceName, p.operation, percentage, bar,
+			p.current, p.total, elapsed.Seconds())
+	}
+}
+
+// formatBytes converts bytes to human-readable format
+func formatBytes(bytes int64) string {
+	const unit = 1024
+	if bytes < unit {
+		return fmt.Sprintf("%d B", bytes)
+	}
+	div, exp := int64(unit), 0
+	for n := bytes / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
+}
+
+func main() {
+	// Custom progress writer for external services
+	fmt.Println("\nCustom Cloud Progress Writer:")
+	cloudProgress := &CloudProgress{serviceName: "Service"}
+	pmtiles.SetProgressWriter(cloudProgress)
+
+	// Simulate some progress operations
+	fmt.Println("\nSimulating PMTiles operations with custom progress reporting:")
+
+	// Simulate a count-based operation
+	progressCount := cloudProgress.NewCountProgress(100, "Processing tiles")
+	for i := 0; i < 100; i += 10 {
+		progressCount.Add(10)
+		time.Sleep(50 * time.Millisecond) // Simulate work
+	}
+	progressCount.Close()
+
+	// Simulate a bytes-based operation
+	progressBytes := cloudProgress.NewBytesProgress(1024*1024, "Uploading tiles")
+	data := make([]byte, 64*1024) // 64KB chunks
+	for i := 0; i < 16; i++ {
+		progressBytes.Write(data)
+		time.Sleep(30 * time.Millisecond) // Simulate upload time
+	}
+	progressBytes.Close()
+}

main.go

@@ -27,6 +27,8 @@ var (
 )
 
 var cli struct {
+	Quiet bool `help:"Suppress verbose output and progress bars"`
+
 	Show struct {
 		Path       string `arg:""`
 		Bucket     string `help:"Remote bucket"`
@@ -127,6 +129,9 @@ func main() {
 	logger := log.New(os.Stdout, "", log.Ldate|log.Ltime|log.Lshortfile)
 	ctx := kong.Parse(&cli)
 
+	// Set quiet mode globally based on the --quiet flag
+	pmtiles.SetQuietMode(cli.Quiet)
+
 	switch ctx.Command() {
 	case "show <path>":
 		err := pmtiles.Show(logger, os.Stdout, cli.Show.Bucket, cli.Show.Path, cli.Show.HeaderJson, cli.Show.Metadata, cli.Show.Tilejson, cli.Show.PublicURL, false, 0, 0, 0)
@@ -153,14 +158,20 @@ func main() {
 		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 			start := time.Now()
 			statusCode := server.ServeHTTP(w, r)
-			logger.Printf("served %d %s in %s", statusCode, url.PathEscape(r.URL.Path), time.Since(start))
+			if !cli.Quiet {
+				logger.Printf("served %d %s in %s", statusCode, url.PathEscape(r.URL.Path), time.Since(start))
+			}
 		})
 
-		logger.Printf("Serving %s %s on port %d and interface %s with Access-Control-Allow-Origin: %s\n", cli.Serve.Bucket, cli.Serve.Path, cli.Serve.Port, cli.Serve.Interface, cli.Serve.Cors)
+		if !cli.Quiet {
+			logger.Printf("Serving %s %s on port %d and interface %s with Access-Control-Allow-Origin: %s\n", cli.Serve.Bucket, cli.Serve.Path, cli.Serve.Port, cli.Serve.Interface, cli.Serve.Cors)
+		}
 		if cli.Serve.AdminPort > 0 {
 			go func() {
 				adminPort := strconv.Itoa(cli.Serve.AdminPort)
-				logger.Printf("Serving /metrics on port %s and interface %s\n", adminPort, cli.Serve.Interface)
+				if !cli.Quiet {
+					logger.Printf("Serving /metrics on port %s and interface %s\n", adminPort, cli.Serve.Interface)
+				}
 				adminMux := http.NewServeMux()
 				adminMux.Handle("/metrics", promhttp.Handler())
 				logger.Fatal(startHTTPServer(cli.Serve.Interface+":"+adminPort, adminMux))

pmtiles/cluster.go

@@ -2,7 +2,6 @@ package pmtiles
 
 import (
 	"fmt"
-	"github.com/schollz/progressbar/v3"
 	"io"
 	"log"
 	"os"
@@ -41,7 +40,11 @@ func Cluster(logger *log.Logger, InputPMTiles string, deduplicate bool) error {
 		return err
 	}
 
-	bar := progressbar.Default(int64(header.TileEntriesCount))
+	var progress Progress
+	progressWriter := getProgressWriter()
+	if progressWriter != nil {
+		progress = progressWriter.NewCountProgress(int64(header.TileEntriesCount), "")
+	}
 
 	err = IterateEntries(header,
 		func(offset uint64, length uint64) ([]byte, error) {
@@ -52,7 +55,9 @@ func Cluster(logger *log.Logger, InputPMTiles string, deduplicate bool) error {
 			if isNew, newData := resolver.AddTileIsNew(e.TileID, data, e.RunLength); isNew {
 				tmpfile.Write(newData)
 			}
-			bar.Add(1)
+			if progress != nil {
+				progress.Add(1)
+			}
 		})
 
 	if err != nil {
@@ -66,6 +71,8 @@ func Cluster(logger *log.Logger, InputPMTiles string, deduplicate bool) error {
 	if err != nil {
 		return err
 	}
-	fmt.Printf("total directory size %d (%f%% of original)\n", newHeader.RootLength+newHeader.LeafDirectoryLength, float64(newHeader.RootLength+newHeader.LeafDirectoryLength)/float64(header.RootLength+header.LeafDirectoryLength)*100)
+	if !quietMode {
+		fmt.Printf("total directory size %d (%f%% of original)\n", newHeader.RootLength+newHeader.LeafDirectoryLength, float64(newHeader.RootLength+newHeader.LeafDirectoryLength)/float64(header.RootLength+header.LeafDirectoryLength)*100)
+	}
 	return nil
 }

pmtiles/convert.go

@@ -16,10 +16,11 @@ import (
 	"time"
 
 	"github.com/RoaringBitmap/roaring/roaring64"
-	"github.com/schollz/progressbar/v3"
 	"zombiezen.com/go/sqlite"
 )
 
+var quietMode bool
+
 type offsetLen struct {
 	Offset uint64
 	Length uint32
@@ -156,7 +157,9 @@ func convertMbtiles(logger *log.Logger, input string, output string, deduplicate
 		return fmt.Errorf("Failed to convert MBTiles to header JSON, %w", err)
 	}
 
-	logger.Println("Pass 1: Assembling TileID set")
+	if !quietMode {
+		logger.Println("Pass 1: Assembling TileID set")
+	}
 	// assemble a sorted set of all TileIds
 	tileset := roaring64.New()
 	{
@@ -187,10 +190,16 @@ func convertMbtiles(logger *log.Logger, input string, output string, deduplicate
 		return fmt.Errorf("no tiles in MBTiles archive")
 	}
 
-	logger.Println("Pass 2: writing tiles")
+	if !quietMode {
+		logger.Println("Pass 2: writing tiles")
+	}
 	resolve := newResolver(deduplicate, header.TileType == Mvt)
 	{
-		bar := progressbar.Default(int64(tileset.GetCardinality()))
+		var progress Progress
+		progressWriter := getProgressWriter()
+		if progressWriter != nil {
+			progress = progressWriter.NewCountProgress(int64(tileset.GetCardinality()), "")
+		}
 		i := tileset.Iterator()
 		stmt := conn.Prep("SELECT tile_data FROM tiles WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?")
 
@@ -229,21 +238,27 @@ func convertMbtiles(logger *log.Logger, input string, output string, deduplicate
 
 			stmt.ClearBindings()
 			stmt.Reset()
-			bar.Add(1)
+			if progress != nil {
+				progress.Add(1)
+			}
 		}
 	}
 	_, err = finalize(logger, resolve, header, tmpfile, output, jsonMetadata)
 	if err != nil {
 		return err
 	}
-	logger.Println("Finished in ", time.Since(start))
+	if !quietMode {
+		logger.Println("Finished in ", time.Since(start))
+	}
 	return nil
 }
 
 func finalize(logger *log.Logger, resolve *resolver, header HeaderV3, tmpfile *os.File, output string, jsonMetadata map[string]interface{}) (HeaderV3, error) {
-	logger.Println("# of addressed tiles: ", resolve.AddressedTiles)
-	logger.Println("# of tile entries (after RLE): ", len(resolve.Entries))
-	logger.Println("# of tile contents: ", resolve.NumContents())
+	if !quietMode {
+		logger.Println("# of addressed tiles: ", resolve.AddressedTiles)
+		logger.Println("# of tile entries (after RLE): ", len(resolve.Entries))
+		logger.Println("# of tile contents: ", resolve.NumContents())
+	}
 
 	header.AddressedTilesCount = resolve.AddressedTiles
 	header.TileEntriesCount = uint64(len(resolve.Entries))
@@ -258,16 +273,18 @@ func finalize(logger *log.Logger, resolve *resolver, header HeaderV3, tmpfile *o
 
 	rootBytes, leavesBytes, numLeaves := optimizeDirectories(resolve.Entries, 16384-HeaderV3LenBytes, Gzip)
 
-	if numLeaves > 0 {
-		logger.Println("Root dir bytes: ", len(rootBytes))
-		logger.Println("Leaves dir bytes: ", len(leavesBytes))
-		logger.Println("Num leaf dirs: ", numLeaves)
-		logger.Println("Total dir bytes: ", len(rootBytes)+len(leavesBytes))
-		logger.Println("Average leaf dir bytes: ", len(leavesBytes)/numLeaves)
-		logger.Printf("Average bytes per addressed tile: %.2f\n", float64(len(rootBytes)+len(leavesBytes))/float64(resolve.AddressedTiles))
-	} else {
-		logger.Println("Total dir bytes: ", len(rootBytes))
-		logger.Printf("Average bytes per addressed tile: %.2f\n", float64(len(rootBytes))/float64(resolve.AddressedTiles))
+	if !quietMode {
+		if numLeaves > 0 {
+			logger.Println("Root dir bytes: ", len(rootBytes))
+			logger.Println("Leaves dir bytes: ", len(leavesBytes))
+			logger.Println("Num leaf dirs: ", numLeaves)
+			logger.Println("Total dir bytes: ", len(rootBytes)+len(leavesBytes))
+			logger.Println("Average leaf dir bytes: ", len(leavesBytes)/numLeaves)
+			logger.Printf("Average bytes per addressed tile: %.2f\n", float64(len(rootBytes)+len(leavesBytes))/float64(resolve.AddressedTiles))
+		} else {
+			logger.Println("Total dir bytes: ", len(rootBytes))
+			logger.Printf("Average bytes per addressed tile: %.2f\n", float64(len(rootBytes))/float64(resolve.AddressedTiles))
+		}
 	}
 
 	metadataBytes, err := SerializeMetadata(jsonMetadata, Gzip)