diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index 933928969e2..9aded201a30 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -15,9 +15,7 @@ import ( "os" "path/filepath" "runtime" - "strconv" "strings" - "time" "github.com/distribution/reference" "github.com/dustin/go-humanize" @@ -1583,33 +1581,8 @@ func tryReverseConvert(c *cli.Context, targetRef string) (bool, error) { // Source image is in Nydus format, perform reverse conversion logrus.Info("Detected Nydus source image, performing reverse conversion to OCI") - // Parse retry delay parameter from string to seconds (int) - retryDelayStr := c.String("push-retry-delay") - retryDelaySeconds := 0 - if retryDelayStr != "" { - // Try to parse as duration first (e.g., "5s", "1m", "1h") - duration, err := time.ParseDuration(retryDelayStr) - if err == nil { - retryDelaySeconds = int(duration.Seconds()) - } else { - // Fallback to parsing as plain integer (for backward compatibility) - seconds, err := strconv.Atoi(retryDelayStr) - if err != nil || seconds < 0 { - logrus.Warnf("failed to parse push-retry-delay(%s): %+v\nusing default value(0 seconds)", retryDelayStr, err) - retryDelaySeconds = 0 - } else { - retryDelaySeconds = seconds - } - } - - if retryDelaySeconds < 0 { - logrus.Warnf("invalid push-retry-delay value(%s): must be non-negative\nusing default value(0 seconds)", retryDelayStr) - retryDelaySeconds = 0 - } - } - - // Build reverse conversion options - reverseOpt := converter.ReverseOpt{ + // Build reverse conversion options using unified Opt + reverseOpt := converter.Opt{ WorkDir: c.String("work-dir"), NydusImagePath: c.String("nydus-image"), Source: c.String("source"), @@ -1620,7 +1593,7 @@ func tryReverseConvert(c *cli.Context, targetRef string) (bool, error) { Platforms: c.String("platform"), OutputJSON: c.String("output-json"), PushRetryCount: c.Int("push-retry-count"), - PushRetryDelay: retryDelaySeconds, + PushRetryDelay: c.String("push-retry-delay"), WithPlainHTTP: c.Bool("plain-http"), } // Execute reverse conversion diff --git a/contrib/nydusify/go.mod b/contrib/nydusify/go.mod index 767e801b9c5..05e3496ca3a 100644 --- a/contrib/nydusify/go.mod +++ b/contrib/nydusify/go.mod @@ -12,8 +12,8 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.17.27 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.8 github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 - github.com/containerd/containerd v1.7.23 - github.com/containerd/containerd/v2 v2.0.5 + github.com/containerd/containerd v1.7.27 + github.com/containerd/containerd/v2 v2.1.2 github.com/containerd/continuity v0.4.5 github.com/containerd/errdefs v1.0.0 github.com/containerd/nydus-snapshotter v0.15.7 @@ -35,7 +35,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/urfave/cli/v2 v2.27.6 github.com/vmihailenco/msgpack/v5 v5.4.1 - golang.org/x/sync v0.14.0 + golang.org/x/sync v0.15.0 golang.org/x/sys v0.33.0 lukechampine.com/blake3 v1.2.1 ) @@ -124,9 +124,11 @@ require ( golang.org/x/text v0.24.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect - google.golang.org/grpc v1.72.0 // indirect + google.golang.org/grpc v1.72.2 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/containerd/containerd/v2 => github.com/nydusaccelerator/containerd/v2 v2.0.0-20250528024712-b96732f49d37 + +replace github.com/containerd/nydus-snapshotter => github.com/J-jxr/nydus-snapshotter v0.0.0-20251105133653-c4c09842ff15 diff --git a/contrib/nydusify/go.sum b/contrib/nydusify/go.sum index b3d6de596f9..6a427884675 100644 --- a/contrib/nydusify/go.sum +++ b/contrib/nydusify/go.sum @@ -6,6 +6,8 @@ github.com/BraveY/snapshotter-converter v0.0.5/go.mod h1:nOVwsdXqdeltxr12x0t0JIb github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/CloudNativeAI/model-spec v0.0.2 h1:uCO86kMk8wwadn8vKs0wT4petig5crByTIngdO3L2cQ= github.com/CloudNativeAI/model-spec v0.0.2/go.mod h1:3U/4zubBfbUkW59ATSg41HnkYyKrKUcKFH/cVdoPQnk= +github.com/J-jxr/nydus-snapshotter v0.0.0-20251105133653-c4c09842ff15 h1:c24ZcjALK6XUEsL4GU+j3GSlBOfQNQIDezhYqWps12w= +github.com/J-jxr/nydus-snapshotter v0.0.0-20251105133653-c4c09842ff15/go.mod h1:/G8lDTa8MYcz2j1zZ/cJcz4DfGdfrM8DBQGCM5mfJBI= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.13.0 h1:/BcXOiS6Qi7N9XqUcv27vkIuVOkBEcWstd2pMlWSeaA= @@ -63,8 +65,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= -github.com/containerd/containerd v1.7.23 h1:H2CClyUkmpKAGlhQp95g2WXHfLYc7whAuvZGBNYOOwQ= -github.com/containerd/containerd v1.7.23/go.mod h1:7QUzfURqZWCZV7RLNEn1XjUCQLEf0bkaK4GjUaZehxw= +github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII= +github.com/containerd/containerd v1.7.27/go.mod h1:xZmPnl75Vc+BLGt4MIfu6bp+fy03gdHAn9bz+FreFR0= github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= @@ -77,8 +79,6 @@ github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/nydus-snapshotter v0.15.7 h1:0/IVOpqM3TClKjzGRhmT3nq38IIZ62eACxGxTKcDk/0= -github.com/containerd/nydus-snapshotter v0.15.7/go.mod h1:eRJqnxQDr48HNop15kZdLZpFF5B6vf6Q11Aq1K0E4Ms= github.com/containerd/platforms v1.0.0-rc.1 h1:83KIq4yy1erSRgOVHNk1HYdPvzdJ5CnsWaRoJX4C41E= github.com/containerd/platforms v1.0.0-rc.1/go.mod h1:J71L7B+aiM5SdIEqmd9wp6THLVRzJGXfNuWCZCllLA4= github.com/containerd/plugin v1.0.0 h1:c8Kf1TNl6+e2TtMHZt+39yAPDbouRH9WAToRjex483Y= @@ -340,8 +340,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -420,8 +420,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= -google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= +google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/contrib/nydusify/pkg/converter/reverse_converter.go b/contrib/nydusify/pkg/converter/reverse_converter.go index 438c0389711..7ad9a29a6f7 100644 --- a/contrib/nydusify/pkg/converter/reverse_converter.go +++ b/contrib/nydusify/pkg/converter/reverse_converter.go @@ -5,62 +5,27 @@ package converter import ( - "archive/tar" - "bytes" - "compress/gzip" "context" - "encoding/json" - "fmt" - "io" "os" - "os/exec" - "path/filepath" - "strings" "time" - "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/containerd/containerd/namespaces" - "github.com/containerd/containerd/v2/plugins/content/local" + "github.com/containerd/platforms" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/converter/provider" - pkgPvd "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider" - "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils" "github.com/goharbor/acceleration-service/pkg/platformutil" // Import snapshotter converter package for Unpack function snapshotterConverter "github.com/containerd/nydus-snapshotter/pkg/converter" ) -// execCommand is a variable that can be replaced in tests -var execCommand = exec.Command - -// ReverseOpt defines options for reverse conversion from Nydus to OCI -type ReverseOpt struct { - WorkDir string - NydusImagePath string - - Source string // Nydus image reference - Target string // OCI image reference - - SourceInsecure bool - TargetInsecure bool - - AllPlatforms bool - Platforms string - - OutputJSON string - - PushRetryCount int - PushRetryDelay int - WithPlainHTTP bool -} - // ReverseConvert converts Nydus image to OCI image -func ReverseConvert(ctx context.Context, opt ReverseOpt) error { +func ReverseConvert(ctx context.Context, opt Opt) error { + start := time.Now() + logrus.Warn("note: conversion from OCI v1 to nydus is currently experimental") logrus.Infof("Starting reverse conversion from Nydus image %s to OCI image %s", opt.Source, opt.Target) ctx = namespaces.WithNamespace(ctx, "nydusify") @@ -87,569 +52,88 @@ func ReverseConvert(ctx context.Context, opt ReverseOpt) error { } defer os.RemoveAll(tmpDir) - // Create provider for registry operations - pvd, err := provider.New(tmpDir, reverseHosts(opt), 0, "", platformMC, 0, nil) + // Create provider for registry operations (reuse hosts for consistent credential/insecure handling) + pvd, err := provider.New(tmpDir, hosts(opt), 0, "", platformMC, 0, nil) if err != nil { return err } - // Parse retry delay (convert seconds to duration) - retryDelay := time.Duration(opt.PushRetryDelay) * time.Second - - // Set push retry configuration - pvd.SetPushRetryConfig(opt.PushRetryCount, retryDelay) - - // Step 1: Pull Nydus image and extract layers - nydusLayers, originalConfig, err := pullNydusImage(ctx, opt, pvd, tmpDir) - if err != nil { - return errors.Wrap(err, "pull nydus image") - } - - // Step 2: Unpack Nydus layers to OCI format - ociLayers, err := unpackNydusLayers(ctx, opt, tmpDir, nydusLayers) + // Parse retry delay using duration string like "5s", "1m" + retryDelay, err := time.ParseDuration(opt.PushRetryDelay) if err != nil { - return errors.Wrap(err, "unpack nydus layers") - } - - // Step 3: Create OCI image config - ociConfig, err := createOCIConfig(originalConfig, ociLayers) - if err != nil { - return errors.Wrap(err, "create oci config") - } - - // Step 4: Push OCI layers and manifest - err = pushOCIImage(ctx, opt, pvd, tmpDir, ociConfig, ociLayers) - if err != nil { - return errors.Wrap(err, "push oci image") - } - - logrus.Infof("Successfully converted Nydus image %s to OCI image %s", opt.Source, opt.Target) - return nil -} - -// reverseHosts creates host configuration for reverse conversion -func reverseHosts(opt ReverseOpt) func(string) (func(string) (string, string, error), bool, error) { - maps := map[string]bool{ - opt.Source: opt.SourceInsecure, - opt.Target: opt.TargetInsecure, - } - return func(ref string) (func(string) (string, string, error), bool, error) { - return func(string) (string, string, error) { return "", "", nil }, maps[ref], nil + return errors.Wrap(err, "parse push retry delay") } -} - -// pullNydusImage pulls Nydus image and extracts layer information -func pullNydusImage(ctx context.Context, opt ReverseOpt, _ *provider.Provider, tmpDir string) ([]ocispec.Descriptor, *ocispec.Image, error) { - logrus.Infof("Pulling Nydus image: %s", opt.Source) - // Create remote for source - remoter, err := pkgPvd.DefaultRemote(opt.Source, opt.SourceInsecure) - if err != nil { - return nil, nil, errors.Wrap(err, "create source remote") - } + pvd.SetPushRetryConfig(opt.PushRetryCount, retryDelay) + // Align plain HTTP behavior with other converter usages if opt.WithPlainHTTP { - remoter.WithHTTP() - } - - // Resolve manifest - manifestDesc, err := remoter.Resolve(ctx) - if utils.RetryWithHTTP(err) { - remoter.MaybeWithHTTP(err) - manifestDesc, err = remoter.Resolve(ctx) - } - if err != nil { - return nil, nil, errors.Wrap(err, "resolve nydus manifest") + pvd.UsePlainHTTP() } - // Pull manifest - manifestReader, err := remoter.Pull(ctx, *manifestDesc, true) - if err != nil { - return nil, nil, errors.Wrap(err, "pull nydus manifest") + // Step 1: Pull source and resolve root descriptor (manifest/index) + if err := pvd.Pull(ctx, opt.Source); err != nil { + return errors.Wrap(err, "provider pull source") } - defer manifestReader.Close() - - manifestBytes, err := io.ReadAll(manifestReader) + logrus.Infof("Pulled source image: %s", opt.Source) + rootDesc, err := pvd.Image(ctx, opt.Source) if err != nil { - return nil, nil, errors.Wrap(err, "read manifest bytes") + return errors.Wrap(err, "get source image descriptor") } - var manifest ocispec.Manifest - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return nil, nil, errors.Wrap(err, "unmarshal manifest") - } - - // Pull image config - configReader, err := remoter.Pull(ctx, manifest.Config, true) + // Step 2: Use reconverter to convert Nydus -> OCI in content store + dstDesc, err := reconvertWithSnapshotter(ctx, pvd, platformMC, tmpDir, []ocispec.Descriptor{*rootDesc}, opt) if err != nil { - return nil, nil, errors.Wrap(err, "pull nydus image config") - } - defer configReader.Close() - - configBytes, err := io.ReadAll(configReader) - if err != nil { - return nil, nil, errors.Wrap(err, "read config bytes") - } - - var config ocispec.Image - if err := json.Unmarshal(configBytes, &config); err != nil { - return nil, nil, errors.Wrap(err, "unmarshal image config") - } - - // Pull all layers - var layers []ocispec.Descriptor - for _, layer := range manifest.Layers { - layerPath := filepath.Join(tmpDir, layer.Digest.Hex()) - layerFile, err := os.Create(layerPath) - if err != nil { - return nil, nil, errors.Wrapf(err, "create layer file %s", layerPath) - } - - layerReader, err := remoter.Pull(ctx, layer, true) - if err != nil { - layerFile.Close() - return nil, nil, errors.Wrapf(err, "pull layer %s", layer.Digest) - } - - _, err = io.Copy(layerFile, layerReader) - layerReader.Close() - layerFile.Close() - if err != nil { - return nil, nil, errors.Wrapf(err, "copy layer %s", layer.Digest) - } - - layers = append(layers, layer) - } - - return layers, &config, nil -} - -// unpackNydusLayers unpacks Nydus layers using converter.Unpack function -func unpackNydusLayers(ctx context.Context, _ ReverseOpt, tmpDir string, nydusLayers []ocispec.Descriptor) ([]ocispec.Descriptor, error) { - logrus.Info("Unpacking Nydus layers to OCI format") - - var ociLayers []ocispec.Descriptor - - for _, layer := range nydusLayers { - // Skip non-nydus layers - if !isNydusLayer(layer) { - ociLayers = append(ociLayers, layer) - continue - } - - // Skip the nydus bootstrap layer - if snapshotterConverter.IsNydusBootstrap(layer) { - logrus.Debugf("skip nydus bootstrap layer %s", layer.Digest.String()) - continue // Don't add bootstrap to OCI layers - } - - layerPath := filepath.Join(tmpDir, layer.Digest.Hex()) - - // Use local.OpenReader to get content.ReaderAt - ra, err := local.OpenReader(layerPath) - if err != nil { - return nil, errors.Wrapf(err, "open layer file %s", layerPath) - } - defer ra.Close() - - // Create output file for the unpacked OCI layer using the original digest as filename - ociLayerPath := filepath.Join(tmpDir, fmt.Sprintf("oci-layer-%s.tar.gz", layer.Digest.Hex())) - ociLayerFile, err := os.Create(ociLayerPath) - if err != nil { - return nil, errors.Wrapf(err, "create oci layer file %s", ociLayerPath) - } - - // Create gzip writer for compression - gzipWriter := gzip.NewWriter(ociLayerFile) - - // Create digesters for uncompressed data only (for DiffIDs) - uncompressedDigester := digest.SHA256.Digester() - - // Use a multi-writer to both write to gzip and calculate uncompressed digest - unpackWriter := io.MultiWriter(gzipWriter, uncompressedDigester.Hash()) - - // Unpack the nydus layer to uncompressed tar data - unpackOpt := snapshotterConverter.UnpackOption{ - WorkDir: tmpDir, - Stream: false, // Use non-streaming mode for simplicity - } - - err = snapshotterConverter.Unpack(ctx, ra, unpackWriter, unpackOpt) - if err != nil { - gzipWriter.Close() - ociLayerFile.Close() - return nil, errors.Wrapf(err, "unpack nydus layer %s", layer.Digest) - } - - // Close gzip writer to finalize compression - err = gzipWriter.Close() - if err != nil { - ociLayerFile.Close() - return nil, errors.Wrapf(err, "close gzip writer for layer %s", layer.Digest) - } - - err = ociLayerFile.Close() - if err != nil { - return nil, errors.Wrapf(err, "close oci layer file %s", ociLayerPath) - } - - // Calculate compressed digest and size from the actual file - compressedDigest, fileSize, err := calculateDigestAndSize(ociLayerPath) - if err != nil { - return nil, errors.Wrapf(err, "calculate digest and size for %s", ociLayerPath) - } - - // Get uncompressed digest - uncompressedDigest := uncompressedDigester.Digest() - - // Create OCI layer descriptor - ociLayer := ocispec.Descriptor{ - Digest: compressedDigest, - Size: fileSize, - MediaType: ocispec.MediaTypeImageLayerGzip, - // Store uncompressed digest for DiffIDs calculation and original digest for file mapping - Annotations: map[string]string{ - "io.containerd.uncompressed": uncompressedDigest.String(), - "io.nydusify.source.layer.digest": layer.Digest.String(), - }, - } - - ociLayers = append(ociLayers, ociLayer) - - logrus.Infof("Successfully converted nydus layer %s to oci layer %s", layer.Digest, ociLayer.Digest) - } - - return ociLayers, nil -} - -// isNydusLayer checks if a layer is a Nydus layer -func isNydusLayer(layer ocispec.Descriptor) bool { - if layer.MediaType == "application/vnd.oci.image.layer.nydus.blob.v1" { - return true - } - if layer.Annotations == nil { - return false - } - _, hasBootstrap := layer.Annotations["containerd.io/snapshot/nydus-bootstrap"] - _, hasBlob := layer.Annotations["containerd.io/snapshot/nydus-blob"] - return hasBootstrap || hasBlob -} - -// nolint: unused -func extractNydusLayer(layerPath, outputDir string) (string, string, error) { - // Open layer file - layerFile, err := os.Open(layerPath) - if err != nil { - return "", "", errors.Wrapf(err, "open layer file %s", layerPath) - } - defer layerFile.Close() - - // Check if it's gzipped - var reader io.Reader = layerFile - if strings.HasSuffix(layerPath, ".gz") || isGzipped(layerFile) { - layerFile.Seek(0, 0) - gzReader, err := gzip.NewReader(layerFile) - if err != nil { - return "", "", errors.Wrap(err, "create gzip reader") - } - defer gzReader.Close() - reader = gzReader + return errors.Wrap(err, "reconvert image with snapshotter") } - // Extract tar contents - tarReader := tar.NewReader(reader) - var bootstrapPath, blobPath string - - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return "", "", errors.Wrap(err, "read tar header") - } - - if header.Typeflag != tar.TypeReg { - continue - } - - filePath := filepath.Join(outputDir, header.Name) - if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil { - return "", "", errors.Wrapf(err, "create directory for %s", filePath) - } - - file, err := os.Create(filePath) - if err != nil { - return "", "", errors.Wrapf(err, "create file %s", filePath) - } - - _, err = io.Copy(file, tarReader) - file.Close() - if err != nil { - return "", "", errors.Wrapf(err, "copy file %s", filePath) - } - - // Identify bootstrap and blob files - if strings.Contains(header.Name, "bootstrap") || header.Name == "image.boot" { - bootstrapPath = filePath - } else if strings.Contains(header.Name, "blob") || strings.HasSuffix(header.Name, ".blob") { - blobPath = filePath - } + // Step 3: Push the converted descriptor to target using provider + if dstDesc == nil { + return errors.New("reconverter returned nil descriptor") } - - return bootstrapPath, blobPath, nil -} - -// isGzipped checks if a file is gzipped -func isGzipped(file *os.File) bool { - buf := make([]byte, 2) - n, err := file.Read(buf) - if err != nil || n < 2 { - return false - } - return buf[0] == 0x1f && buf[1] == 0x8b -} - -// runNydusImageUnpack runs nydus-image unpack command -func runNydusImageUnpack(nydusImagePath, bootstrapPath, blobPath, outputDir string) error { - args := []string{ - "unpack", - "--bootstrap", bootstrapPath, - "--output", outputDir, - } - - if blobPath != "" { - args = append(args, "--blob", blobPath) + logrus.Infof("Pushing converted descriptor to target: %s", opt.Target) + if err := pvd.Push(ctx, *dstDesc, opt.Target); err != nil { + return errors.Wrap(err, "push oci image") } - cmd := exec.Command(nydusImagePath, args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - logrus.Infof("Running: %s %s", nydusImagePath, strings.Join(args, " ")) - return cmd.Run() + logrus.Infof("Successfully converted Nydus image %s to OCI image %s (duration=%s)", opt.Source, opt.Target, time.Since(start)) + return nil } -// createOCILayerTar creates a gzipped tar file from a directory -func createOCILayerTar(sourceDir, targetPath string) error { - tarFile, err := os.Create(targetPath) - if err != nil { - return errors.Wrapf(err, "create tar file %s", targetPath) - } - defer tarFile.Close() - - gzWriter := gzip.NewWriter(tarFile) - defer gzWriter.Close() - - tarWriter := tar.NewWriter(gzWriter) - defer tarWriter.Close() - - return filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // Skip the source directory itself - if path == sourceDir { - return nil - } - - // Calculate relative path - relPath, err := filepath.Rel(sourceDir, path) - if err != nil { - return err - } - - // Create tar header - header, err := tar.FileInfoHeader(info, "") - if err != nil { - return err - } - header.Name = filepath.ToSlash(relPath) - - // Write header - if err := tarWriter.WriteHeader(header); err != nil { - return err - } - - // Write file content if it's a regular file - if info.Mode().IsRegular() { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - _, err = io.Copy(tarWriter, file) - return err - } - - return nil +// reconvertWithSnapshotter wires reconverter default functions with snapshotter layer reconvert func. +// It converts the pulled source descriptor in provider's content store and returns the new descriptor. +func reconvertWithSnapshotter( + ctx context.Context, + pvd *provider.Provider, + platformMC platforms.MatchComparer, + workDir string, + srcDescs []ocispec.Descriptor, + opt Opt, +) (*ocispec.Descriptor, error) { + logrus.Debugf("Start reconvertWithSnapshotter: src_count=%d workDir=%s", len(srcDescs), workDir) + cs := pvd.ContentStore() + + // Prepare layer reconvert func: Nydus blob/bootstrap -> OCI tar layer + layerFn := snapshotterConverter.LayerReconvertFunc(snapshotterConverter.UnpackOption{ + BuilderPath: opt.NydusImagePath, + WorkDir: workDir, + Stream: false, + Compressor: opt.Compressor, }) -} -// calculateDigestAndSize calculates digest and size of a file -func calculateDigestAndSize(filePath string) (digest.Digest, int64, error) { - file, err := os.Open(filePath) - if err != nil { - return "", 0, err - } - defer file.Close() + // Prepare index/manifest convert func using reconverter default + indexFn := snapshotterConverter.DefaultIndexConvertFunc(layerFn, false, platformMC) - digester := digest.Canonical.Digester() - size, err := io.Copy(digester.Hash(), file) - if err != nil { - return "", 0, err - } - - return digester.Digest(), size, nil -} - -// createOCIConfig creates OCI image configuration -func createOCIConfig(originalConfig *ocispec.Image, ociLayers []ocispec.Descriptor) (*ocispec.Image, error) { - // Use current UTC time - now := time.Now().UTC() - - // Copy all information from original config, preserving everything - config := &ocispec.Image{ - Created: &now, - Author: originalConfig.Author, - Platform: originalConfig.Platform, - Config: originalConfig.Config, - RootFS: ocispec.RootFS{ - Type: originalConfig.RootFS.Type, - DiffIDs: make([]digest.Digest, len(ociLayers)), - }, - // Preserve original history completely - History: make([]ocispec.History, len(ociLayers)), - } - - // Calculate diff IDs for uncompressed layers - for i, layer := range ociLayers { - // Use uncompressed digest for DiffIDs - if uncompressedDigestStr, ok := layer.Annotations["io.containerd.uncompressed"]; ok { - uncompressedDigest, err := digest.Parse(uncompressedDigestStr) - if err != nil { - return nil, errors.Wrapf(err, "parse uncompressed digest %s", uncompressedDigestStr) - } - config.RootFS.DiffIDs[i] = uncompressedDigest - } else { - // Fallback to layer digest for non-nydus layers - config.RootFS.DiffIDs[i] = layer.Digest + // The source may include multiple descriptors (e.g., manifests). Convert each and pick the first valid. + for _, src := range srcDescs { + logrus.Debugf("Converting src descriptor: mediaType=%s digest=%s", src.MediaType, src.Digest.String()) + dst, err := indexFn(ctx, cs, src) + if err != nil { + return nil, err } - - if i < len(originalConfig.History) { - config.History[i] = originalConfig.History[i] - } else { - // Use current UTC time for new layers - now := time.Now().UTC() - config.History[i] = ocispec.History{ - Created: &now, - CreatedBy: "nydusify reverse converter", - } + if dst != nil { + return dst, nil } } - - return config, nil -} - -// pushOCIImage pushes the OCI image to target registry -func pushOCIImage(ctx context.Context, opt ReverseOpt, _ *provider.Provider, tmpDir string, config *ocispec.Image, layers []ocispec.Descriptor) error { - logrus.Infof("Pushing OCI image to: %s", opt.Target) - - // Create remote for target - remoter, err := pkgPvd.DefaultRemote(opt.Target, opt.TargetInsecure) - if err != nil { - return errors.Wrap(err, "create target remote") - } - - if opt.WithPlainHTTP { - remoter.WithHTTP() - } - - // Push layers - var pushedLayers []ocispec.Descriptor - for _, layer := range layers { - // Check if this layer was converted (has source digest annotation) - sourceDigest, hasSourceDigest := layer.Annotations["io.nydusify.source.layer.digest"] - if hasSourceDigest { - // Parse the source digest to get the hex part only - parsedDigest, err := digest.Parse(sourceDigest) - if err != nil { - return errors.Wrapf(err, "parse source digest %s", sourceDigest) - } - - // This is a converted layer, find the corresponding file - layerPath := filepath.Join(tmpDir, fmt.Sprintf("oci-layer-%s.tar.gz", parsedDigest.Hex())) - if _, err := os.Stat(layerPath); os.IsNotExist(err) { - return errors.Wrapf(err, "converted layer file not found: %s", layerPath) - } - - layerFile, err := os.Open(layerPath) - if err != nil { - return errors.Wrapf(err, "open layer file %s", layerPath) - } - defer layerFile.Close() - - // Use the layer descriptor which already has the correct digest and size from conversion - err = remoter.Push(ctx, layer, true, layerFile) - if err != nil { - return errors.Wrapf(err, "push layer %s", layer.Digest) - } - } else { - // This is an original layer (non-Nydus or skipped), push from original file - // For now, we'll just add it to pushed layers without actually pushing - // as these layers should already exist in the registry - logrus.Infof("Skipping push for original layer %s (should already exist)", layer.Digest) - } - - pushedLayers = append(pushedLayers, layer) - } - - // Create and push config - configBytes, err := json.Marshal(config) - if err != nil { - return errors.Wrap(err, "marshal config") - } - - configDigest := digest.FromBytes(configBytes) - configDesc := ocispec.Descriptor{ - Digest: configDigest, - Size: int64(len(configBytes)), - MediaType: ocispec.MediaTypeImageConfig, - } - - err = remoter.Push(ctx, configDesc, false, bytes.NewReader(configBytes)) - if err != nil { - return errors.Wrap(err, "push config") - } - - // Create and push manifest - manifest := ocispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - MediaType: ocispec.MediaTypeImageManifest, - Config: configDesc, - Layers: pushedLayers, - } - - manifestBytes, err := json.Marshal(manifest) - if err != nil { - return errors.Wrap(err, "marshal manifest") - } - - manifestDigest := digest.FromBytes(manifestBytes) - manifestDesc := ocispec.Descriptor{ - Digest: manifestDigest, - Size: int64(len(manifestBytes)), - MediaType: ocispec.MediaTypeImageManifest, - } - - err = remoter.Push(ctx, manifestDesc, false, bytes.NewReader(manifestBytes)) - if err != nil { - return errors.Wrap(err, "push manifest") - } - - return nil + return nil, errors.New("no convertible descriptor found") } diff --git a/contrib/nydusify/pkg/converter/reverse_converter_test.go b/contrib/nydusify/pkg/converter/reverse_converter_test.go index e40bc15eaf7..48faea73465 100644 --- a/contrib/nydusify/pkg/converter/reverse_converter_test.go +++ b/contrib/nydusify/pkg/converter/reverse_converter_test.go @@ -5,16 +5,11 @@ package converter import ( - "archive/tar" - "bytes" - "compress/gzip" "context" "fmt" "io" "os" "os/exec" - "path/filepath" - "strings" "testing" "time" @@ -29,6 +24,8 @@ type MockRemoter struct { mock.Mock } +const testPushRetryDelay = "5s" + func (m *MockRemoter) Resolve(ctx context.Context) (*ocispec.Descriptor, error) { args := m.Called(ctx) return args.Get(0).(*ocispec.Descriptor), args.Error(1) @@ -67,12 +64,23 @@ func (m *MockRemoter) ReadSeekCloser(ctx context.Context, desc ocispec.Descripto return args.Get(0).(io.ReadSeekCloser), args.Error(1) } -func fakeExecCommand(command string, args ...string) *exec.Cmd { - cs := []string{"-test.run=TestHelperProcess", "--", command} - cs = append(cs, args...) - cmd := exec.Command(os.Args[0], cs...) - cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} - return cmd +// checkNydusImageAvailable checks if nydus-image tool is available +func checkNydusImageAvailable() bool { + // Check if we're in test mode (using fake command) + if os.Getenv("GO_WANT_HELPER_PROCESS") == "1" { + return true + } + + // Check if nydus-image command exists + _, err := exec.LookPath("nydus-image") + return err == nil +} + +// skipIfNydusImageNotAvailable skips the test if nydus-image is not available +func skipIfNydusImageNotAvailable(t *testing.T) { + if !checkNydusImageAvailable() { + t.Skip("nydus-image tool not available, skipping test") + } } // TestHelperProcess is used to mock external commands @@ -131,322 +139,224 @@ func TestHelperProcess(_ *testing.T) { os.Exit(2) } -func TestPullNydusImage(t *testing.T) { - t.Run("Test invalid source reference", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-pull-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) - - opt := ReverseOpt{ - Source: "invalid://reference", - SourceInsecure: false, - } - - _, _, err = pullNydusImage(context.Background(), opt, nil, tmpDir) - assert.Error(t, err) - assert.Contains(t, err.Error(), "create source remote") - }) - - t.Run("Test empty source reference", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-pull-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) - - opt := ReverseOpt{ - Source: "", - SourceInsecure: false, - } - - _, _, err = pullNydusImage(context.Background(), opt, nil, tmpDir) - assert.Error(t, err) - }) - - t.Run("Test with mock remoter - resolve error", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-pull-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) - - mockRemoter := &MockRemoter{} - mockRemoter.On("Resolve", mock.Anything).Return((*ocispec.Descriptor)(nil), fmt.Errorf("resolve failed")) +// Smoke tests for basic functionality +func TestReverseConvertSmoke(t *testing.T) { + t.Run("Test basic reverse conversion setup", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - opt := ReverseOpt{ + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", Source: "localhost:5000/test:nydus", - SourceInsecure: false, + Target: "localhost:5000/test:oci", + Platforms: "linux/amd64", } - // This would require actual implementation to accept mock remoter - // For now, just test the error path - _, _, err = pullNydusImage(context.Background(), opt, nil, tmpDir) + // This should fail at push retry delay parsing, not provider creation + err := ReverseConvert(context.Background(), opt) assert.Error(t, err) + // Verify the error is related to parsing push retry delay, not provider creation + assert.Contains(t, err.Error(), "parse push retry delay") }) - t.Run("Test with mock remoter - pull layer error", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-pull-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) + t.Run("Test with alpine:latest nydus image", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - mockRemoter := &MockRemoter{} - manifestDesc := &ocispec.Descriptor{ - Digest: "sha256:manifest123", - Size: 1024, - MediaType: ocispec.MediaTypeImageManifest, + // Skip if no test registry is available + if os.Getenv("NYDUS_TEST_REGISTRY") == "" { + t.Skip("NYDUS_TEST_REGISTRY not set, skipping alpine test") } - mockRemoter.On("Resolve", mock.Anything).Return(manifestDesc, nil) - mockRemoter.On("Pull", mock.Anything, mock.Anything, mock.Anything).Return((*bytes.Buffer)(nil), fmt.Errorf("pull failed")) - opt := ReverseOpt{ - Source: "localhost:5000/test:nydus", - SourceInsecure: false, + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: os.Getenv("NYDUS_TEST_REGISTRY") + "/alpine:nydus", + Target: os.Getenv("NYDUS_TEST_REGISTRY") + "/alpine:oci", + Platforms: "linux/amd64", + SourceInsecure: true, + TargetInsecure: true, } - // This would require actual implementation to accept mock remoter - _, _, err = pullNydusImage(context.Background(), opt, nil, tmpDir) - assert.Error(t, err) - }) - - t.Run("Test with invalid manifest JSON", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-pull-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) - - // This test would require mocking the entire pull flow - // For now, just test that the function handles errors - opt := ReverseOpt{ - Source: "localhost:5000/test:nydus", - SourceInsecure: false, + err := ReverseConvert(context.Background(), opt) + // This might succeed if test registry is properly set up + if err != nil { + t.Logf("Reverse conversion failed as expected in test environment: %v", err) } - - _, _, err = pullNydusImage(context.Background(), opt, nil, tmpDir) - assert.Error(t, err) }) - t.Run("Test layer file creation error", func(t *testing.T) { - // Use a read-only directory to cause file creation error - opt := ReverseOpt{ + t.Run("Test experimental warning is logged", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) + + // This test verifies that the experimental warning is properly logged + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", Source: "localhost:5000/test:nydus", - SourceInsecure: false, + Target: "localhost:5000/test:oci", + Platforms: "linux/amd64", } - var err error - _, _, err = pullNydusImage(context.Background(), opt, nil, "/root") + err := ReverseConvert(context.Background(), opt) assert.Error(t, err) + // The warning should be logged before any errors occur }) } -func TestPushOCIImage(t *testing.T) { - t.Run("Test invalid target reference", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-push-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) +// Flow tests for complete conversion process +func TestReverseConvertFlow(t *testing.T) { + t.Run("Test complete conversion flow with alpine", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - opt := ReverseOpt{ - Target: "invalid://reference", - TargetInsecure: false, + // Skip if no test registry is available + if os.Getenv("NYDUS_TEST_REGISTRY") == "" { + t.Skip("NYDUS_TEST_REGISTRY not set, skipping flow test") } - err = pushOCIImage(context.Background(), opt, nil, "", &ocispec.Image{}, []ocispec.Descriptor{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "create target remote") - }) - - t.Run("Test empty target reference", func(t *testing.T) { - opt := ReverseOpt{ - Target: "", - TargetInsecure: false, - } - - err := pushOCIImage(context.Background(), opt, nil, "", &ocispec.Image{}, []ocispec.Descriptor{}) - assert.Error(t, err) - }) - - t.Run("Test with empty config", func(t *testing.T) { - opt := ReverseOpt{ - Target: "localhost:5000/test:oci", - TargetInsecure: false, - } - - err := pushOCIImage(context.Background(), opt, nil, "", nil, []ocispec.Descriptor{}) - assert.Error(t, err) - }) - - t.Run("Test with malformed registry reference", func(t *testing.T) { - opt := ReverseOpt{ - Target: "malformed::reference", - TargetInsecure: false, + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: os.Getenv("NYDUS_TEST_REGISTRY") + "/alpine:nydus", + Target: os.Getenv("NYDUS_TEST_REGISTRY") + "/alpine:oci-reverse", + Platforms: "linux/amd64", + SourceInsecure: true, + TargetInsecure: true, + PushRetryCount: 3, + PushRetryDelay: testPushRetryDelay, } - config := &ocispec.Image{ - Author: "test", + // Test the complete flow + err := ReverseConvert(context.Background(), opt) + if err != nil { + t.Logf("Flow test failed as expected in test environment: %v", err) } - - err := pushOCIImage(context.Background(), opt, nil, "", config, []ocispec.Descriptor{}) - assert.Error(t, err) }) - t.Run("Test with unreachable registry", func(t *testing.T) { - opt := ReverseOpt{ - Target: "unreachable.registry.com/test:oci", - TargetInsecure: false, - } - - config := &ocispec.Image{ - Author: "test", - } - - err := pushOCIImage(context.Background(), opt, nil, "", config, []ocispec.Descriptor{}) - assert.Error(t, err) - }) + t.Run("Test conversion flow with multiple platforms", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - t.Run("Test with plain HTTP to HTTPS registry", func(t *testing.T) { - opt := ReverseOpt{ + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", Target: "localhost:5000/test:oci", - TargetInsecure: false, - WithPlainHTTP: true, - } - - config := &ocispec.Image{ - Author: "test", + Platforms: "linux/amd64,linux/arm64", + AllPlatforms: true, } - err := pushOCIImage(context.Background(), opt, nil, "", config, []ocispec.Descriptor{}) + err := ReverseConvert(context.Background(), opt) assert.Error(t, err) + // Should fail at provider creation, but platform parsing should succeed }) - t.Run("Test layer push failure", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-push-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) - - // Create a test layer file - layerFile := filepath.Join(tmpDir, "oci-layer-0.tar.gz") - f, err := os.Create(layerFile) - assert.NoError(t, err) - f.WriteString("test layer content") - f.Close() + t.Run("Test conversion flow with compression", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - opt := ReverseOpt{ + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", Target: "localhost:5000/test:oci", - TargetInsecure: false, - } - - config := &ocispec.Image{ - Author: "test", - } - - layers := []ocispec.Descriptor{ - { - Digest: "sha256:abc123", - Size: 100, - MediaType: ocispec.MediaTypeImageLayerGzip, - }, + Platforms: "linux/amd64", + Compressor: "gzip", } - err = pushOCIImage(context.Background(), opt, nil, "", config, layers) + err := ReverseConvert(context.Background(), opt) assert.Error(t, err) }) +} - t.Run("Test config push failure", func(t *testing.T) { - opt := ReverseOpt{ - Target: "localhost:5000/test:oci", - TargetInsecure: false, - } - - config := &ocispec.Image{ - Author: "test", - Config: ocispec.ImageConfig{ - Env: []string{"PATH=/usr/bin"}, - }, - } +// Error handling and edge case tests +func TestReverseConvertErrorHandling(t *testing.T) { + t.Run("Test context cancellation", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - err := pushOCIImage(context.Background(), opt, nil, "", config, []ocispec.Descriptor{}) - assert.Error(t, err) - }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately - t.Run("Test manifest push failure", func(t *testing.T) { - opt := ReverseOpt{ + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", Target: "localhost:5000/test:oci", - TargetInsecure: false, - } - - config := &ocispec.Image{ - Author: "test", - Created: &time.Time{}, - Platform: ocispec.Platform{Architecture: "amd64", OS: "linux"}, + Platforms: "linux/amd64", } - err := pushOCIImage(context.Background(), opt, nil, "", config, []ocispec.Descriptor{}) + err := ReverseConvert(ctx, opt) assert.Error(t, err) }) - t.Run("Test malformed target reference", func(t *testing.T) { - opt := ReverseOpt{ - Target: "::invalid::", - TargetInsecure: false, - } - - config := &ocispec.Image{Author: "test"} - - err := pushOCIImage(context.Background(), opt, nil, "", config, []ocispec.Descriptor{}) - assert.Error(t, err) - }) + t.Run("Test invalid platform format", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - t.Run("Test with retry configuration", func(t *testing.T) { - opt := ReverseOpt{ + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", Target: "localhost:5000/test:oci", - TargetInsecure: false, - PushRetryCount: 3, - PushRetryDelay: 1, + Platforms: "invalid-platform-format", + PushRetryDelay: testPushRetryDelay, } - config := &ocispec.Image{Author: "test"} - - err := pushOCIImage(context.Background(), opt, nil, "", config, []ocispec.Descriptor{}) + err := ReverseConvert(context.Background(), opt) assert.Error(t, err) - // Should still fail but with retry logic + assert.Contains(t, err.Error(), "parse platforms") }) -} -// Additional comprehensive test cases for complete coverage -func TestReverseConvertComprehensive(t *testing.T) { - t.Run("Test context cancellation", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately + t.Run("Test invalid retry delay format", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - opt := ReverseOpt{ + opt := Opt{ WorkDir: "./tmp", NydusImagePath: "nydus-image", Source: "localhost:5000/test:nydus", Target: "localhost:5000/test:oci", Platforms: "linux/amd64", + PushRetryDelay: "invalid-duration", } - err := ReverseConvert(ctx, opt) + err := ReverseConvert(context.Background(), opt) assert.Error(t, err) + assert.Contains(t, err.Error(), "parse push retry delay") }) - t.Run("Test with multiple platforms", func(t *testing.T) { - opt := ReverseOpt{ - WorkDir: "./tmp", + t.Run("Test with non-existent work directory", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) + + opt := Opt{ + WorkDir: "/root/readonly/directory", NydusImagePath: "nydus-image", Source: "localhost:5000/test:nydus", Target: "localhost:5000/test:oci", - Platforms: "linux/amd64,linux/arm64", + Platforms: "linux/amd64", + PushRetryDelay: testPushRetryDelay, } err := ReverseConvert(context.Background(), opt) assert.Error(t, err) - // Should fail at provider creation, but platform parsing should succeed }) t.Run("Test with custom retry settings", func(t *testing.T) { - opt := ReverseOpt{ + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) + + opt := Opt{ WorkDir: "./tmp", NydusImagePath: "nydus-image", Source: "localhost:5000/test:nydus", Target: "localhost:5000/test:oci", Platforms: "linux/amd64", PushRetryCount: 5, - PushRetryDelay: 10, + PushRetryDelay: testPushRetryDelay, } err := ReverseConvert(context.Background(), opt) @@ -454,7 +364,10 @@ func TestReverseConvertComprehensive(t *testing.T) { }) t.Run("Test with both insecure flags", func(t *testing.T) { - opt := ReverseOpt{ + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) + + opt := Opt{ WorkDir: "./tmp", NydusImagePath: "nydus-image", Source: "localhost:5000/test:nydus", @@ -467,387 +380,209 @@ func TestReverseConvertComprehensive(t *testing.T) { err := ReverseConvert(context.Background(), opt) assert.Error(t, err) }) -} -// Test edge cases and error conditions -func TestEdgeCases(t *testing.T) { - t.Run("Test calculateDigestAndSize with large file", func(t *testing.T) { - tmpFile, err := os.CreateTemp("", "test-large-*") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) - - // Write a large amount of data - largeData := strings.Repeat("test data ", 100000) - _, err = tmpFile.WriteString(largeData) - assert.NoError(t, err) - tmpFile.Close() - - digest, size, err := calculateDigestAndSize(tmpFile.Name()) - assert.NoError(t, err) - assert.Equal(t, int64(len(largeData)), size) - assert.NotEmpty(t, digest.String()) - }) + t.Run("Test with plain HTTP", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - t.Run("Test isNydusLayer with complex annotations", func(t *testing.T) { - layer := ocispec.Descriptor{ - Annotations: map[string]string{ - "containerd.io/snapshot/nydus-bootstrap": "true", - "containerd.io/snapshot/nydus-blob": "true", - "other.annotation": "value", - }, + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", + Target: "localhost:5000/test:oci", + Platforms: "linux/amd64", + WithPlainHTTP: true, } - result := isNydusLayer(layer) - assert.True(t, result) - }) - - t.Run("Test createOCILayerTar with special files", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-tar-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) - - sourceDir := filepath.Join(tmpDir, "source") - err = os.MkdirAll(sourceDir, 0755) - assert.NoError(t, err) - - // Create various types of files - testFile := filepath.Join(sourceDir, "regular.txt") - err = os.WriteFile(testFile, []byte("regular file"), 0644) - assert.NoError(t, err) - - // Create executable file - execFile := filepath.Join(sourceDir, "executable") - err = os.WriteFile(execFile, []byte("#!/bin/sh\necho hello"), 0755) - assert.NoError(t, err) - - targetPath := filepath.Join(tmpDir, "output.tar.gz") - err = createOCILayerTar(sourceDir, targetPath) - assert.NoError(t, err) - - // Verify tar file was created - stat, err := os.Stat(targetPath) - assert.NoError(t, err) - assert.Greater(t, stat.Size(), int64(0)) + err := ReverseConvert(context.Background(), opt) + assert.Error(t, err) }) } -func TestRunNydusImageUnpack(t *testing.T) { - // Save original exec.Command function - originalExecCommand := execCommand - defer func() { execCommand = originalExecCommand }() +// Performance and concurrency tests +func TestReverseConvertPerformance(t *testing.T) { + t.Run("Test conversion with large retry count", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - t.Run("Test with empty nydus image path", func(t *testing.T) { - execCommand = fakeExecCommand + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", + Target: "localhost:5000/test:oci", + Platforms: "linux/amd64", + PushRetryCount: 100, + PushRetryDelay: testPushRetryDelay, + } - tmpDir, err := os.MkdirTemp("", "test-unpack-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) + err := ReverseConvert(context.Background(), opt) + assert.Error(t, err) + }) - bootstrapPath := filepath.Join(tmpDir, "bootstrap") - err = os.WriteFile(bootstrapPath, []byte("bootstrap data"), 0644) - assert.NoError(t, err) + t.Run("Test conversion timeout", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - blobPath := filepath.Join(tmpDir, "blob") - err = os.WriteFile(blobPath, []byte("blob data"), 0644) - assert.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() - outputDir := filepath.Join(tmpDir, "output") - err = os.MkdirAll(outputDir, 0755) - assert.NoError(t, err) + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", + Target: "localhost:5000/test:oci", + Platforms: "linux/amd64", + } - err = runNydusImageUnpack("", bootstrapPath, blobPath, outputDir) + err := ReverseConvert(ctx, opt) assert.Error(t, err) }) } -func TestUnpackNydusLayers(t *testing.T) { - t.Run("Test with mixed layer types", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-unpack-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) - - // Create test layer files - nydusLayerFile := filepath.Join(tmpDir, "nydus-layer.tar") - regularLayerFile := filepath.Join(tmpDir, "regular-layer.tar.gz") - - // Create nydus layer tar with bootstrap - nydusFile, err := os.Create(nydusLayerFile) - assert.NoError(t, err) - tarWriter := tar.NewWriter(nydusFile) - header := &tar.Header{ - Name: "bootstrap", - Mode: 0644, - Size: 9, - Typeflag: tar.TypeReg, - } - tarWriter.WriteHeader(header) - tarWriter.Write([]byte("bootstrap")) - tarWriter.Close() - nydusFile.Close() - - // Create regular layer tar.gz - regularFile, err := os.Create(regularLayerFile) - assert.NoError(t, err) - gzWriter := gzip.NewWriter(regularFile) - tarWriter = tar.NewWriter(gzWriter) - header = &tar.Header{ - Name: "file.txt", - Mode: 0644, - Size: 4, - Typeflag: tar.TypeReg, - } - tarWriter.WriteHeader(header) - tarWriter.Write([]byte("test")) - tarWriter.Close() - gzWriter.Close() - regularFile.Close() - - layers := []ocispec.Descriptor{ - { - Digest: "sha256:nydus123", - Size: 1024, - MediaType: ocispec.MediaTypeImageLayerGzip, - Annotations: map[string]string{ - "containerd.io/snapshot/nydus-bootstrap": "true", - }, - }, - { - Digest: "sha256:regular123", - Size: 512, - MediaType: ocispec.MediaTypeImageLayerGzip, - }, - } +// Integration tests for real-world scenarios +func TestReverseConvertIntegration(t *testing.T) { + t.Run("Test with real alpine nydus image", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - ociLayers, err := unpackNydusLayers(context.Background(), ReverseOpt{NydusImagePath: "nydus-image"}, tmpDir, layers) - // This will likely fail due to missing nydus-image binary, but we test the logic - if err != nil { - // Should still return some layers processed before error - assert.NotNil(t, ociLayers) - } else { - // If no error, should have processed layers - assert.NotNil(t, ociLayers) + // Skip if no test registry is available + if os.Getenv("NYDUS_TEST_REGISTRY") == "" { + t.Skip("NYDUS_TEST_REGISTRY not set, skipping integration test") } - }) - t.Run("Test with only regular layers", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-unpack-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) - - // Create regular layer tar.gz - regularLayerFile := filepath.Join(tmpDir, "regular-layer.tar.gz") - regularFile, err := os.Create(regularLayerFile) - assert.NoError(t, err) - gzWriter := gzip.NewWriter(regularFile) - tarWriter := tar.NewWriter(gzWriter) - header := &tar.Header{ - Name: "file.txt", - Mode: 0644, - Size: 4, - Typeflag: tar.TypeReg, - } - tarWriter.WriteHeader(header) - tarWriter.Write([]byte("test")) - tarWriter.Close() - gzWriter.Close() - regularFile.Close() - - layers := []ocispec.Descriptor{ - { - Digest: "sha256:regular123", - Size: 512, - MediaType: ocispec.MediaTypeImageLayerGzip, - }, + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: os.Getenv("NYDUS_TEST_REGISTRY") + "/alpine:nydus", + Target: os.Getenv("NYDUS_TEST_REGISTRY") + "/alpine:oci-integration", + Platforms: "linux/amd64", + SourceInsecure: true, + TargetInsecure: true, + PushRetryCount: 3, + PushRetryDelay: testPushRetryDelay, } - ociLayers, err := unpackNydusLayers(context.Background(), ReverseOpt{NydusImagePath: "nydus-image"}, tmpDir, layers) - assert.NoError(t, err) - assert.Equal(t, 1, len(ociLayers)) - assert.Equal(t, layers[0].Digest, ociLayers[0].Digest) - }) - - t.Run("Test with empty layer list", func(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "test-unpack-*") - assert.NoError(t, err) - defer os.RemoveAll(tmpDir) - - layers := []ocispec.Descriptor{} - - ociLayers, err := unpackNydusLayers(context.Background(), ReverseOpt{NydusImagePath: "nydus-image"}, tmpDir, layers) - assert.NoError(t, err) - assert.Equal(t, 0, len(ociLayers)) - }) -} - -// Test utility functions -func TestUtilityFunctions(t *testing.T) { - t.Run("Test isGzipped with gzipped file", func(t *testing.T) { - tmpFile, err := os.CreateTemp("", "test-gzip-*") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) - - // Write gzip header - gzWriter := gzip.NewWriter(tmpFile) - gzWriter.Write([]byte("test data")) - gzWriter.Close() - tmpFile.Close() - - // Reopen for reading - file, err := os.Open(tmpFile.Name()) - assert.NoError(t, err) - defer file.Close() - - result := isGzipped(file) - assert.True(t, result) + // Test the complete integration flow + err := ReverseConvert(context.Background(), opt) + if err != nil { + t.Logf("Integration test failed as expected in test environment: %v", err) + } }) - t.Run("Test isGzipped with regular file", func(t *testing.T) { - tmpFile, err := os.CreateTemp("", "test-regular-*") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) + t.Run("Test conversion with different compressors", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - tmpFile.WriteString("regular file content") - tmpFile.Close() + compressors := []string{"gzip", "zstd", "lz4"} - // Reopen for reading - file, err := os.Open(tmpFile.Name()) - assert.NoError(t, err) - defer file.Close() - - result := isGzipped(file) - assert.False(t, result) - }) - - t.Run("Test calculateDigestAndSize with existing file", func(t *testing.T) { - tmpFile, err := os.CreateTemp("", "test-digest-*") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) - - testData := "test data for digest calculation" - tmpFile.WriteString(testData) - tmpFile.Close() + for _, compressor := range compressors { + t.Run("compressor_"+compressor, func(t *testing.T) { + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", + Target: "localhost:5000/test:oci", + Platforms: "linux/amd64", + Compressor: compressor, + } - digest, size, err := calculateDigestAndSize(tmpFile.Name()) - assert.NoError(t, err) - assert.Equal(t, int64(len(testData)), size) - assert.NotEmpty(t, digest.String()) - assert.True(t, strings.HasPrefix(digest.String(), "sha256:")) + err := ReverseConvert(context.Background(), opt) + assert.Error(t, err) + }) + } }) - t.Run("Test calculateDigestAndSize with non-existent file", func(t *testing.T) { - // Use a cross-platform non-existent path - nonExistentPath := filepath.Join(os.TempDir(), "non-existent-file-12345") - _, _, err := calculateDigestAndSize(nonExistentPath) - assert.Error(t, err) - // Use more generic error checking that works across platforms - assert.True(t, os.IsNotExist(err) || strings.Contains(err.Error(), "no such file") || strings.Contains(err.Error(), "cannot find")) - }) + t.Run("Test conversion with different platforms", func(t *testing.T) { + // Skip if nydus-image tool is not available + skipIfNydusImageNotAvailable(t) - t.Run("Test isNydusLayer with bootstrap annotation", func(t *testing.T) { - layer := ocispec.Descriptor{ - Annotations: map[string]string{ - "containerd.io/snapshot/nydus-bootstrap": "true", - }, + platforms := []string{ + "linux/amd64", + "linux/arm64", + "linux/amd64,linux/arm64", } - result := isNydusLayer(layer) - assert.True(t, result) - }) + for _, platform := range platforms { + t.Run("platform_"+platform, func(t *testing.T) { + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", + Target: "localhost:5000/test:oci", + Platforms: platform, + } - t.Run("Test isNydusLayer with blob annotation", func(t *testing.T) { - layer := ocispec.Descriptor{ - Annotations: map[string]string{ - "containerd.io/snapshot/nydus-blob": "true", - }, + err := ReverseConvert(context.Background(), opt) + assert.Error(t, err) + }) } - - result := isNydusLayer(layer) - assert.True(t, result) }) +} - t.Run("Test isNydusLayer with nydus media type", func(t *testing.T) { - layer := ocispec.Descriptor{ - MediaType: "application/vnd.oci.image.layer.nydus.blob.v1", +// CI-friendly tests that don't require nydus-image tool +func TestReverseConvertCIFriendly(t *testing.T) { + t.Run("Test basic parameter validation without nydus-image", func(t *testing.T) { + // Test parameter validation that doesn't require nydus-image tool + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", + Target: "localhost:5000/test:oci", + Platforms: "invalid-platform-format", + PushRetryDelay: testPushRetryDelay, } - result := isNydusLayer(layer) - assert.True(t, result) + err := ReverseConvert(context.Background(), opt) + assert.Error(t, err) + assert.Contains(t, err.Error(), "parse platforms") }) - t.Run("Test isNydusLayer with regular layer", func(t *testing.T) { - layer := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageLayerGzip, + t.Run("Test invalid retry delay without nydus-image", func(t *testing.T) { + // Test retry delay parsing that doesn't require nydus-image tool + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", + Target: "localhost:5000/test:oci", + Platforms: "linux/amd64", + PushRetryDelay: "invalid-duration", } - result := isNydusLayer(layer) - assert.False(t, result) + err := ReverseConvert(context.Background(), opt) + assert.Error(t, err) + assert.Contains(t, err.Error(), "parse push retry delay") }) - t.Run("Test isNydusLayer with nil annotations", func(t *testing.T) { - layer := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageLayerGzip, - Annotations: nil, + t.Run("Test work directory creation without nydus-image", func(t *testing.T) { + // Test work directory handling that doesn't require nydus-image tool + opt := Opt{ + WorkDir: "/root/readonly/directory", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", + Target: "localhost:5000/test:oci", + Platforms: "linux/amd64", + PushRetryDelay: testPushRetryDelay, } - result := isNydusLayer(layer) - assert.False(t, result) + err := ReverseConvert(context.Background(), opt) + assert.Error(t, err) }) - t.Run("Test reverseHosts with edge cases", func(t *testing.T) { - opt := ReverseOpt{ - Source: "registry.example.com/repo:tag", - Target: "localhost:5000/repo:tag", - SourceInsecure: true, - TargetInsecure: false, - } - - hostFunc := reverseHosts(opt) - - // Test with source reference - _, insecure, err := hostFunc("registry.example.com/repo:tag") - assert.NoError(t, err) - assert.True(t, insecure) - - // Test with target reference - _, insecure, err = hostFunc("localhost:5000/repo:tag") - assert.NoError(t, err) - assert.False(t, insecure) - - // Test with unknown reference - _, insecure, err = hostFunc("unknown.registry.com/repo:tag") - assert.NoError(t, err) - assert.False(t, insecure) - }) + t.Run("Test context cancellation without nydus-image", func(t *testing.T) { + // Test context handling that doesn't require nydus-image tool + ctx, cancel := context.WithCancel(context.Background()) + cancel() // Cancel immediately - t.Run("Test isGzipped with various file sizes", func(t *testing.T) { - tests := []struct { - name string - content []byte - expected bool - }{ - {"Empty file", []byte{}, false}, - {"One byte", []byte{0x1f}, false}, - {"Two bytes - correct first", []byte{0x1f, 0x8b}, true}, - {"Two bytes - incorrect", []byte{0x1f, 0x00}, false}, - {"Three bytes - gzip", []byte{0x1f, 0x8b, 0x08}, true}, - {"Large non-gzip", bytes.Repeat([]byte{0x00}, 1000), false}, + opt := Opt{ + WorkDir: "./tmp", + NydusImagePath: "nydus-image", + Source: "localhost:5000/test:nydus", + Target: "localhost:5000/test:oci", + Platforms: "linux/amd64", + PushRetryDelay: testPushRetryDelay, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tmpFile, err := os.CreateTemp("", "test-gzip-*") - assert.NoError(t, err) - defer os.Remove(tmpFile.Name()) - defer tmpFile.Close() - - _, err = tmpFile.Write(tt.content) - assert.NoError(t, err) - _, err = tmpFile.Seek(0, 0) - assert.NoError(t, err) - - result := isGzipped(tmpFile) - assert.Equal(t, tt.expected, result) - }) - } + err := ReverseConvert(ctx, opt) + assert.Error(t, err) }) }