Feature: mc cat with parallel and buffer size args

CONTRIBUTING.md

@@ -27,7 +27,7 @@ $ ./mc --help
 
 * When you're ready to create a pull request, be sure to:
     - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request.
-    - Run `go fmt`
+    - Run the lints which will be run in the CI: `golangci-lint run -j8` (install locally from https://github.com/golangci/golangci-lint)
     - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
     - Make sure `make install` completes.
 

cmd/accounting-reader.go

@@ -190,5 +190,5 @@ func (a *accounter) Read(p []byte) (n int, err error) {
 
 	n = len(p)
 	a.Add(int64(n))
-	return
+	return n, nil
 }

cmd/admin-config-reset.go

@@ -78,7 +78,8 @@ func (u configResetMessage) String() (msg string) {
 		msg += console.Colorize("ResetConfigSuccess",
 			fmt.Sprintf("\nPlease restart your server with `%s`.", suggestion))
 	}
-	return
+
+	return msg
 }
 
 // JSON jsonified service status message.

cmd/admin-config-set.go

@@ -73,7 +73,8 @@ func (u configSetMessage) String() (msg string) {
 		msg += console.Colorize("SetConfigSuccess",
 			fmt.Sprintf("\nPlease restart your server '%s'.", suggestion))
 	}
-	return
+
+	return msg
 }
 
 // JSON jsonified service status message.

cmd/cat-main.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
+// Copyright (c) 2015-2025 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -24,12 +24,14 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"runtime"
 	"strings"
 	"syscall"
 	"time"
 	"unicode"
 	"unicode/utf8"
 
+	"github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	"github.com/minio/mc/pkg/probe"
 )
@@ -59,6 +61,15 @@ var catFlags = []cli.Flag{
 		Name:  "part-number",
 		Usage: "download only a specific part number",
 	},
+	cli.IntFlag{
+		Name:  "parallel, P",
+		Usage: "number of parallel downloads (default: 1)",
+		Value: 1,
+	},
+	cli.StringFlag{
+		Name:  "buffer-size",
+		Usage: "total buffer size for parallel downloads, split among workers (e.g. 1GiB, 512MiB)",
+	},
 }
 
 // Display contents of a file.
@@ -101,6 +112,12 @@ EXAMPLES:
 
   7. Display the content of a particular object version
      {{.Prompt}} {{.HelpName}} --vid "3ddac055-89a7-40fa-8cd3-530a5581b6b8" play/my-bucket/my-object
+
+  8. Download a large object with parallel downloads (8 threads, 1GiB total buffer)
+     {{.Prompt}} {{.HelpName}} --parallel 8 --buffer-size 1GiB play/my-bucket/large-file.bin > large-file.bin
+
+  9. Stream large object to mc pipe with parallel downloads for fast bucket-to-bucket copy
+     {{.Prompt}} {{.HelpName}} --parallel 16 --buffer-size 2GiB source/bucket/15tb-file | mc pipe --part-size 128MiB target/bucket/15tb-file
 `,
 }
 
@@ -161,14 +178,16 @@ func (s prettyStdout) Write(input []byte) (int, error) {
 }
 
 type catOpts struct {
-	args      []string
-	versionID string
-	timeRef   time.Time
-	startO    int64
-	tailO     int64
-	partN     int
-	isZip     bool
-	stdinMode bool
+	args          []string
+	versionID     string
+	timeRef       time.Time
+	startO        int64
+	tailO         int64
+	partN         int
+	isZip         bool
+	stdinMode     bool
+	parallel      int
+	bufferSizeStr string
 }
 
 // parseCatSyntax performs command-line input validation for cat command.
@@ -203,6 +222,9 @@ func parseCatSyntax(ctx *cli.Context) catOpts {
 	o.startO = ctx.Int64("offset")
 	o.tailO = ctx.Int64("tail")
 	o.partN = ctx.Int("part-number")
+	o.parallel = ctx.Int("parallel")
+	o.bufferSizeStr = ctx.String("buffer-size")
+
 	if o.tailO != 0 && o.startO != 0 {
 		fatalIf(errInvalidArgument().Trace(), "You cannot specify both --tail and --offset")
 	}
@@ -218,6 +240,17 @@ func parseCatSyntax(ctx *cli.Context) catOpts {
 	if (o.tailO != 0 || o.startO != 0) && o.partN > 0 {
 		fatalIf(errInvalidArgument().Trace(), "You cannot use --part-number with --tail or --offset")
 	}
+	if o.parallel > 1 && (o.tailO != 0 || o.startO != 0 || o.partN > 0 || o.isZip) {
+		fatalIf(errInvalidArgument().Trace(), "You cannot use --parallel with --tail, --offset, --part-number, or --zip")
+	}
+	if o.parallel < 1 {
+		fatalIf(errInvalidArgument().Trace(), "Invalid --parallel value, must be >= 1")
+	}
+	if o.bufferSizeStr != "" {
+		if _, err := humanize.ParseBytes(o.bufferSizeStr); err != nil {
+			fatalIf(probe.NewError(err).Trace(), "Invalid --buffer-size value")
+		}
+	}
 
 	return o
 }
@@ -232,13 +265,16 @@ func catURL(ctx context.Context, sourceURL string, encKeyDB map[string][]prefixS
 	default:
 		versionID := o.versionID
 		var err *probe.Error
+		var client Client
+		var content *ClientContent
+
 		// Try to stat the object, the purpose is to:
 		// 1. extract the size of S3 object so we can check if the size of the
 		// downloaded object is equal to the original one. FS files
 		// are ignored since some of them have zero size though they
 		// have contents like files under /proc.
 		// 2. extract the version ID if rewind flag is passed
-		if client, content, err := url2Stat(ctx, url2StatOptions{
+		if client, content, err = url2Stat(ctx, url2StatOptions{
 			urlStr:                  sourceURL,
 			versionID:               o.versionID,
 			fileAttr:                false,
@@ -270,6 +306,39 @@ func catURL(ctx context.Context, sourceURL string, encKeyDB map[string][]prefixS
 		} else {
 			return err.Trace(sourceURL)
 		}
+
+		// Use parallel reader for large objects (>16MiB) with multiple threads
+		// Default buffer size is smallest of object size, 25% of
+		// available memory or 1GB, configurable via --buffer-size
+		if o.parallel > 1 && size > 16<<20 && client.GetURL().Type == objectStorage {
+			var memStats runtime.MemStats
+			runtime.ReadMemStats(&memStats)
+			bufferSize := min(size, min(int64(memStats.Sys/4), 1<<30))
+			if o.bufferSizeStr != "" {
+				parsed, parseErr := humanize.ParseBytes(o.bufferSizeStr)
+				if parseErr != nil {
+					return probe.NewError(parseErr).Trace(sourceURL)
+				}
+				bufferSize = int64(parsed)
+			}
+
+			// Minimum 5MiB per part
+			partSize := max(bufferSize/int64(o.parallel), 5*1024*1024)
+
+			// Skip parallel download if effective part size would exceed object size
+			if partSize < size {
+				gopts := GetOptions{VersionID: versionID, Zip: o.isZip}
+				pr := NewParallelReader(ctx, client, size, partSize, o.parallel, gopts)
+				if startErr := pr.Start(); startErr != nil {
+					return probe.NewError(startErr).Trace(sourceURL)
+				}
+				reader = pr
+				defer reader.Close()
+				return catOut(reader, size).Trace(sourceURL)
+			}
+		}
+
+		// Use standard single-threaded reader
 		gopts := GetOptions{VersionID: versionID, Zip: o.isZip, RangeStart: o.startO, PartNumber: o.partN}
 		if reader, err = getSourceStreamFromURL(ctx, sourceURL, encKeyDB, getSourceOpts{
 			GetOptions: gopts,

cmd/client-s3.go

@@ -892,8 +892,8 @@ func (c *S3Client) Get(ctx context.Context, opts GetOptions) (io.ReadCloser, *Cl
 	if opts.Zip {
 		o.Set("x-minio-extract", "true")
 	}
-	if opts.RangeStart != 0 {
-		err := o.SetRange(opts.RangeStart, 0)
+	if opts.RangeStart != 0 || opts.RangeEnd != 0 {
+		err := o.SetRange(opts.RangeStart, opts.RangeEnd)
 		if err != nil {
 			return nil, nil, probe.NewError(err)
 		}

cmd/client.go

@@ -60,6 +60,7 @@ type GetOptions struct {
 	VersionID  string
 	Zip        bool
 	RangeStart int64
+	RangeEnd   int64
 	PartNumber int
 	Preserve   bool
 }