multi-platform

See `docs/multi-platform.md`

- [X] image convert (not new in new this commit)
- [X] image inspect
- [X] rmi
- [X] pull
- [X] push
- [X] load
- [X] save
- [X] run
- [X] commit
- [X] build
- [X] compose
- [X] docs
- [X] tests

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

Author: AkihiroSuda

.github/workflows/test.yml

@@ -70,6 +70,8 @@ jobs:
         sudo apt-get purge -y snapd
         sudo losetup -Dv
         sudo losetup -lv
+    - name: "Register QEMU (tonistiigi/binfmt)"
+      run: docker run --privileged --rm tonistiigi/binfmt --install all
     - name: "Run integration tests"
       run: docker run -t --rm --privileged test-integration go test -v ./cmd/nerdctl/... -args -test.kill-daemon
 
@@ -85,6 +87,8 @@ jobs:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 1
+    - name: "Register QEMU (tonistiigi/binfmt)"
+      run: docker run --privileged --rm tonistiigi/binfmt --install all
     - name: "Prepare (network driver=slirp4netns, port driver=builtin)"
       run: DOCKER_BUILDKIT=1 docker build -t test-integration-rootless --target test-integration-rootless --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
     - name: "Test    (network driver=slirp4netns, port driver=builtin)"
@@ -117,6 +121,8 @@ jobs:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 1
+    - name: "Register QEMU (tonistiigi/binfmt)"
+      run: docker run --privileged --rm tonistiigi/binfmt --install all
     - name: "Ensure that the integration test suite is compatible with Docker"
       run: go test -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.kill-daemon
 

README.md

@@ -89,8 +89,6 @@ $ limactl start
 $ lima nerdctl run -d --name nginx -p 127.0.0.1:8080:80 nginx:alpine
 ```
 
-NOTE: ARM Mac requires installing a patched version of QEMU, see [Lima](https://github.com/AkihiroSuda/lima) documentation.
-
 ### FreeBSD
 
 See [`./docs/freebsd.md`](docs/freebsd.md).
@@ -131,6 +129,7 @@ Minor:
 - Specifying a non-image rootfs: `nerdctl run -it --rootfs <ROOTFS> /bin/sh` . The CLI syntax conforms to Podman convention.
 - Connecting a container to multiple networks at once: `nerdctl run --net foo --net bar`
 - Running [FreeBSD jails](./docs/freebsd.md).
+- Better multi-platform support, e.g., `nerdctl pull --all-platforms IMAGE`
 
 Trivial:
 - Inspecting raw OCI config: `nerdctl container inspect --mode=native` .
@@ -291,6 +290,9 @@ Basic flags:
   - Default: "missing"
 - :whale: `--pid=(host)`: PID namespace to use
 
+Platform flags:
+- :whale: `--platform=(amd64|arm64|...)`: Set platform
+
 Network flags:
 - :whale: `--net, --network=(bridge|host|none|<CNI>)`: Connect a container to a network
   - Default: "bridge"
@@ -620,8 +622,9 @@ Flags:
 - :whale: `-q, --quiet`: Suppress the build output and print image ID on success
 - :whale: `--cache-from=CACHE`: External cache sources (eg. user/app:cache, type=local,src=path/to/dir) (compatible with `docker buildx build`)
 - :whale: `--cache-to=CACHE`: Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir) (compatible with `docker buildx build`)
+- :whale: `--platform=(amd64|arm64|...)`: Set target platform for build (compatible with `docker buildx build`)
 
-Unimplemented `docker build` flags: `--add-host`, `--iidfile`, `--label`, `--network`, `--platform`, `--squash`
+Unimplemented `docker build` flags: `--add-host`, `--iidfile`, `--label`, `--network`, `--squash`
 
 ### :whale: nerdctl commit
 Create a new image from a container's changes
@@ -656,13 +659,22 @@ Pull an image from a registry.
 
 Usage: `nerdctl pull [OPTIONS] NAME[:TAG|@DIGEST]`
 
-Unimplemented `docker pull` flags: `--all-tags`, `--disable-content-trust` (default true), `--platform`, `--quiet`
+Flags:
+- :whale: `--platform=(amd64|arm64|...)`: Pull content for a specific platform
+  - :nerd_face: Unlike Docker, this flag can be specified multiple times (`--platform=amd64 --platform=arm64`)
+- :nerd_face: `--all-platforms`: Pull content for all platforms
+
+Unimplemented `docker pull` flags: `--all-tags`, `--disable-content-trust` (default true), `--quiet`
 
 ### :whale: nerdctl push
 Push an image to a registry.
 
 Usage: `nerdctl push [OPTIONS] NAME[:TAG]`
 
+Flags:
+- :nerd_face: `--platform=(amd64|arm64|...)`: Push content for a specific platform
+- :nerd_face: `--all-platforms`: Push content for all platforms
+
 Unimplemented `docker push` flags: `--all-tags`, `--disable-content-trust` (default true), `--quiet`
 
 ### :whale: nerdctl load
@@ -674,6 +686,8 @@ Usage: `nerdctl load [OPTIONS]`
 
 Flags:
 - :whale: `-i, --input`: Read from tar archive file, instead of STDIN
+- :nerd_face: `--platform=(amd64|arm64|...)`: Import content for a specific platform
+- :nerd_face: `--all-platforms`: Import content for all platforms
 
 Unimplemented `docker load` flags: `--quiet`
 
@@ -686,6 +700,8 @@ Usage: `nerdctl save [OPTIONS] IMAGE [IMAGE...]`
 
 Flags:
 - :whale: `-o, --output`: Write to a file, instead of STDOUT
+- :nerd_face: `--platform=(amd64|arm64|...)`: Export content for a specific platform
+- :nerd_face: `--all-platforms`: Export content for all platforms
 
 ### :whale: nerdctl tag
 Create a tag TARGET\_IMAGE that refers to SOURCE\_IMAGE.
@@ -707,6 +723,7 @@ Usage: `nerdctl image inspect [OPTIONS] NAME|ID [NAME|ID...]`
 Flags:
 - :nerd_face: `--mode=(dockercompat|native)`: Inspection mode. "native" produces more information.
 - :whale: `--format`: Format the output using the given Go template, e.g, `{{json .}}`
+- :nerd_face: `--platform=(amd64|arm64|...)`: Inspect a specific platform
 
 ### :nerd_face: nerdctl image convert
 Convert an image format.
@@ -1025,4 +1042,5 @@ Others:
 - [`./docs/stargz.md`](./docs/stargz.md):     Lazy-pulling using Stargz Snapshotter
 - [`./docs/ocicrypt.md`](./docs/ocicrypt.md): Running encrypted images
 - [`./docs/freebsd.md`](./docs/freebsd.md):  Running FreeBSD jails
+- [`./docs/multi-platform.md`](./docs/multi-platform.md):  Multi-platform mode
 - [`./docs/experimental.md`](./docs/experimental.md):  Experimental features

cmd/nerdctl/build.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/containerd/nerdctl/pkg/buildkitutil"
 	"github.com/containerd/nerdctl/pkg/defaults"
+	"github.com/containerd/nerdctl/pkg/platformutil"
 	"github.com/containerd/nerdctl/pkg/strutil"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -63,10 +64,21 @@ func newBuildCommand() *cobra.Command {
 	buildCommand.Flags().StringSlice("cache-from", nil, "External cache sources (eg. user/app:cache, type=local,src=path/to/dir)")
 	buildCommand.Flags().StringSlice("cache-to", nil, "Cache export destinations (eg. user/app:cache, type=local,dest=path/to/dir)")
 
+	// #region platform flags
+	buildCommand.Flags().StringSlice("platform", []string{}, "Set target platform for build (e.g., \"amd64\", \"arm64\")")
+	buildCommand.RegisterFlagCompletionFunc("platform", shellCompletePlatforms)
+	// #endregion
+
 	return buildCommand
 }
 
 func buildAction(cmd *cobra.Command, args []string) error {
+	platform, err := cmd.Flags().GetStringSlice("platform")
+	if err != nil {
+		return err
+	}
+	platform = strutil.DedupeStrSlice(platform)
+
 	buildkitHost, err := cmd.Flags().GetString("buildkit-host")
 	if err != nil {
 		return err
@@ -75,7 +87,7 @@ func buildAction(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	buildctlBinary, buildctlArgs, needsLoading, cleanup, err := generateBuildctlArgs(cmd, args)
+	buildctlBinary, buildctlArgs, needsLoading, cleanup, err := generateBuildctlArgs(cmd, platform, args)
 	if err != nil {
 		return err
 	}
@@ -110,7 +122,11 @@ func buildAction(cmd *cobra.Command, args []string) error {
 	}
 
 	if needsLoading {
-		if err = loadImage(buildctlStdout, cmd, args, false, quiet); err != nil {
+		platMC, err := platformutil.NewMatchComparer(false, platform)
+		if err != nil {
+			return err
+		}
+		if err = loadImage(buildctlStdout, cmd, args, platMC, quiet); err != nil {
 			return err
 		}
 	}
@@ -122,7 +138,7 @@ func buildAction(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func generateBuildctlArgs(cmd *cobra.Command, args []string) (string, []string, bool, func(), error) {
+func generateBuildctlArgs(cmd *cobra.Command, platform, args []string) (string, []string, bool, func(), error) {
 	var needsLoading bool
 	if len(args) < 1 {
 		return "", nil, false, nil, errors.New("context needs to be specified")
@@ -143,6 +159,11 @@ func generateBuildctlArgs(cmd *cobra.Command, args []string) (string, []string,
 	}
 	if output == "" {
 		output = "type=docker"
+		if len(platform) > 1 {
+			// For avoiding `error: failed to solve: docker exporter does not currently support exporting manifest lists`
+			// TODO: consider using type=oci for single-platform build too
+			output = "type=oci"
+		}
 		needsLoading = true
 	}
 	tagValue, err := cmd.Flags().GetStringSlice("tag")
@@ -214,6 +235,10 @@ func generateBuildctlArgs(cmd *cobra.Command, args []string) (string, []string,
 		buildctlArgs = append(buildctlArgs, "--opt=target="+target)
 	}
 
+	if len(platform) > 0 {
+		buildctlArgs = append(buildctlArgs, "--opt=platform="+strings.Join(platform, ","))
+	}
+
 	buildArgsValue, err := cmd.Flags().GetStringSlice("build-arg")
 	if err != nil {
 		return "", nil, false, cleanup, err

cmd/nerdctl/commit.go

@@ -63,7 +63,7 @@ func commitAction(cmd *cobra.Command, args []string) error {
 			if found.MatchCount > 1 {
 				return errors.Errorf("ambiguous ID %q", found.Req)
 			}
-			imageID, err := commit.Commit(ctx, client, found.Container.ID(), opts)
+			imageID, err := commit.Commit(ctx, client, found.Container, opts)
 			if err != nil {
 				return err
 			}

cmd/nerdctl/completion.go

@@ -138,3 +138,17 @@ func shellCompleteVolumeNames(cmd *cobra.Command) ([]string, cobra.ShellCompDire
 	}
 	return candidates, cobra.ShellCompDirectiveNoFileComp
 }
+
+func shellCompletePlatforms(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	candidates := []string{
+		"amd64",
+		"arm64",
+		"riscv64",
+		"ppc64le",
+		"s390x",
+		"386",
+		"arm",          // alias of "linux/arm/v7"
+		"linux/arm/v6", // "arm/v6" is invalid (interpreted as OS="arm", Arch="v7")
+	}
+	return candidates, cobra.ShellCompDirectiveNoFileComp
+}

cmd/nerdctl/compose.go

@@ -22,10 +22,12 @@ import (
 	composecli "github.com/compose-spec/compose-go/cli"
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/platforms"
 	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/composer"
 	"github.com/containerd/nerdctl/pkg/imgutil"
 	"github.com/containerd/nerdctl/pkg/netutil"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
@@ -155,9 +157,17 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 		return true, nil
 	}
 
-	o.EnsureImage = func(ctx context.Context, imageName, pullMode string) error {
+	o.EnsureImage = func(ctx context.Context, imageName, pullMode, platform string) error {
+		ocispecPlatforms := []ocispec.Platform{platforms.DefaultSpec()}
+		if platform != "" {
+			parsed, err := platforms.Parse(platform)
+			if err != nil {
+				return err
+			}
+			ocispecPlatforms = []ocispec.Platform{parsed} // no append
+		}
 		_, imgErr := imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), snapshotter, imageName,
-			pullMode, insecure)
+			pullMode, insecure, ocispecPlatforms)
 		return imgErr
 	}
 

cmd/nerdctl/image_convert.go

@@ -24,13 +24,11 @@ import (
 
 	"github.com/containerd/containerd/images/converter"
 	"github.com/containerd/containerd/images/converter/uncompress"
-	"github.com/containerd/containerd/platforms"
 	refdocker "github.com/containerd/containerd/reference/docker"
-	"github.com/containerd/nerdctl/pkg/strutil"
+	"github.com/containerd/nerdctl/pkg/platformutil"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
 	"github.com/containerd/stargz-snapshotter/recorder"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -71,6 +69,7 @@ func newImageConvertCommand() *cobra.Command {
 
 	// #region platform flags
 	imageConvertCommand.Flags().StringSlice("platform", []string{}, "Convert content for a specific platform")
+	imageConvertCommand.RegisterFlagCompletionFunc("platform", shellCompletePlatforms)
 	imageConvertCommand.Flags().Bool("all-platforms", false, "Convert content for all platforms")
 	// #endregion
 
@@ -107,22 +106,11 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-
-	if !allPlatforms {
-		if pss := strutil.DedupeStrSlice(platform); len(pss) > 0 {
-			var all []ocispec.Platform
-			for _, ps := range pss {
-				p, err := platforms.Parse(ps)
-				if err != nil {
-					return errors.Wrapf(err, "invalid platform %q", ps)
-				}
-				all = append(all, p)
-			}
-			convertOpts = append(convertOpts, converter.WithPlatform(platforms.Ordered(all...)))
-		} else {
-			convertOpts = append(convertOpts, converter.WithPlatform(platforms.DefaultStrict()))
-		}
+	platMC, err := platformutil.NewMatchComparer(allPlatforms, platform)
+	if err != nil {
+		return err
 	}
+	convertOpts = append(convertOpts, converter.WithPlatform(platMC))
 
 	estargz, err := cmd.Flags().GetBool("estargz")
 	if err != nil {

cmd/nerdctl/image_inspect.go

@@ -24,6 +24,8 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/nerdctl/pkg/idutil/imagewalker"
 	"github.com/containerd/nerdctl/pkg/imageinspector"
 	"github.com/containerd/nerdctl/pkg/inspecttypes/dockercompat"
@@ -49,6 +51,12 @@ func newImageInspectCommand() *cobra.Command {
 		return []string{"dockercompat", "native"}, cobra.ShellCompDirectiveNoFileComp
 	})
 	imageInspectCommand.Flags().StringP("format", "f", "", "Format the output using the given Go template, e.g, '{{json .}}'")
+
+	// #region platform flags
+	imageInspectCommand.Flags().String("platform", "", "Inspect a specific platform") // not a slice, and there is no --all-platforms
+	imageInspectCommand.RegisterFlagCompletionFunc("platform", shellCompletePlatforms)
+	// #endregion
+
 	return imageInspectCommand
 }
 
@@ -57,7 +65,20 @@ func imageInspectAction(cmd *cobra.Command, args []string) error {
 		return errors.Errorf("requires at least 1 argument")
 	}
 
-	client, ctx, cancel, err := newClient(cmd)
+	var clientOpts []containerd.ClientOpt
+	platform, err := cmd.Flags().GetString("platform")
+	if err != nil {
+		return err
+	}
+	if platform != "" {
+		platformParsed, err := platforms.Parse(platform)
+		if err != nil {
+			return err
+		}
+		platformM := platforms.Only(platformParsed)
+		clientOpts = append(clientOpts, containerd.WithDefaultPlatform(platformM))
+	}
+	client, ctx, cancel, err := newClient(cmd, clientOpts...)
 	if err != nil {
 		return err
 	}