[sha512] image/copy: allow users to force digest algorithm in CLI (e.g. skopeo copy --force-digest)

image/copy/blob.go

@@ -101,6 +101,11 @@ func (ic *imageCopier) copyBlobFromStream(ctx context.Context, srcReader io.Read
 		Cache:      ic.c.blobInfoCache,
 		IsConfig:   isConfig,
 		EmptyLayer: emptyLayer,
+		Digests:    ic.c.options.digestOptions,
+		// CannotChangeDigestReason requires stream.info.Digest to always be set, and it is:
+		// If ic.cannotModifyManifestReason, stream.info was not modified since its initialization at the top of this
+		// function, and the caller is required to provide a digest.
+		CannotChangeDigestReason: ic.cannotModifyManifestReason,
 	}
 	if !isConfig {
 		options.LayerIndex = &layerIndex
@@ -133,7 +138,10 @@ func (ic *imageCopier) copyBlobFromStream(ctx context.Context, srcReader io.Read
 		return types.BlobInfo{}, fmt.Errorf("Internal error writing blob %s, digest verification failed but was ignored", srcInfo.Digest)
 	}
 	if stream.info.Digest != "" && uploadedInfo.Digest != stream.info.Digest {
-		return types.BlobInfo{}, fmt.Errorf("Internal error writing blob %s, blob with digest %s saved with digest %s", srcInfo.Digest, stream.info.Digest, uploadedInfo.Digest)
+		// If algorithms match, the whole digest values must match
+		if stream.info.Digest.Algorithm() == uploadedInfo.Digest.Algorithm() {
+			return types.BlobInfo{}, fmt.Errorf("Internal error writing blob %s, blob with digest %s saved with digest %s", srcInfo.Digest, stream.info.Digest, uploadedInfo.Digest)
+		}
 	}
 	if digestingReader.validationSucceeded {
 		if err := compressionStep.recordValidatedDigestData(ic.c, uploadedInfo, srcInfo, encryptionStep, decryptionStep); err != nil {

image/copy/copy.go

@@ -14,6 +14,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"go.podman.io/image/v5/docker/reference"
 	internalblobinfocache "go.podman.io/image/v5/internal/blobinfocache"
+	"go.podman.io/image/v5/internal/digests"
 	"go.podman.io/image/v5/internal/image"
 	"go.podman.io/image/v5/internal/imagedestination"
 	"go.podman.io/image/v5/internal/imagesource"
@@ -155,6 +156,35 @@ type Options struct {
 	// In oci-archive: destinations, this will set the create/mod/access timestamps in each tar entry
 	// (but not a timestamp of the created archive file).
 	DestinationTimestamp *time.Time
+
+	// FIXME:
+	// - this reference to an internal type is unusable from the outside even if we made the field public
+	// - what is the actual semantics? Right now it is probably “choices to use when writing to the destination”, TBD
+	// - anyway do we want to expose _all_ of the digests.Options tunables, or fewer?
+	// - … do we want to expose _more_ granularity than that?
+	//   - (“must have at least sha512 integrity when reading”, what does “at least” mean for random pairs of algorithms?)
+	//   - should some of this be in config files, maybe ever per-registry?
+	digestOptions digests.Options
+}
+
+// BrokenSetForceDestinationDigestAlgorithm forces the use of a specific digest algorithm when writing to the destination.
+//
+// UNSTABLE API: This API is incomplete and may be changed or removed at any time.
+// It currently only enforces the digest algorithm for a subset of transports and operations.
+// See https://github.com/containers/container-libs/pull/552 for implementation status.
+func (o *Options) BrokenSetForceDestinationDigestAlgorithm(algo digest.Algorithm) error {
+	if o.digestOptions.MustUseSet() != "" {
+		return fmt.Errorf("digest options are already configured")
+	}
+	if !algo.Available() {
+		return fmt.Errorf("digest algorithm %q is not available", algo.String())
+	}
+	digestOpts, err := digests.MustUse(algo)
+	if err != nil {
+		return fmt.Errorf("failed to set force-digest algorithm: %w", err)
+	}
+	o.digestOptions = digestOpts
+	return nil
 }
 
 // OptionCompressionVariant allows to supply information about
@@ -200,6 +230,13 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
 	if options == nil {
 		options = &Options{}
 	}
+	// FIXME: digestsOptions exists to gradually build the feature. Provide public API once fully implemented.
+	// Set default only if not configured by BrokenSetForceDestinationDigestAlgorithm
+	if options.digestOptions.MustUseSet() == "" {
+		optionsCopy := *options
+		optionsCopy.digestOptions = digests.CanonicalDefault()
+		options = &optionsCopy
+	}
 
 	if err := validateImageListSelection(options.ImageListSelection); err != nil {
 		return nil, err

image/copy/multiple.go

@@ -208,6 +208,12 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte,
 		cannotModifyManifestListReason = "Instructed to preserve digests"
 	}
 
+	// FIXME: Multi-arch not supported yet; would need config digest updates for each instance.
+	// See https://github.com/containers/container-libs/pull/552#discussion_r2611627578
+	if c.options.digestOptions.MustUseSet() != "" {
+		return nil, fmt.Errorf("forcing digest algorithm with multi-arch images is not yet implemented")
+	}
+
 	// Determine if we'll need to convert the manifest list to a different format.
 	forceListMIMEType := c.options.ForceManifestMIMEType
 	switch forceListMIMEType {

image/copy/single.go

@@ -18,6 +18,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/vbauerster/mpb/v8"
 	"go.podman.io/image/v5/docker/reference"
+	"go.podman.io/image/v5/internal/digests"
 	"go.podman.io/image/v5/internal/image"
 	"go.podman.io/image/v5/internal/pkg/platform"
 	"go.podman.io/image/v5/internal/private"
@@ -592,12 +593,27 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanc
 		return nil, "", fmt.Errorf("reading manifest: %w", err)
 	}
 
-	if err := ic.copyConfig(ctx, pendingImage); err != nil {
+	newConfigDigest, err := ic.copyConfig(ctx, pendingImage)
+	if err != nil {
 		return nil, "", err
 	}
 
+	// FIXME: Single image only; multi-arch needs per-instance config digest updates.
+	// See https://github.com/containers/container-libs/pull/552#discussion_r2611627578
+	if newConfigDigest != nil {
+		man, err = ic.updateManifestConfigDigest(man, pendingImage, *newConfigDigest)
+		if err != nil {
+			return nil, "", fmt.Errorf("updating manifest config digest: %w", err)
+		}
+	}
+
 	ic.c.Printf("Writing manifest to image destination\n")
-	manifestDigest, err := manifest.Digest(man)
+	// Choose the digest algorithm based on digest options
+	manifestDigestAlgo, err := ic.c.options.digestOptions.Choose(digests.Situation{})
+	if err != nil {
+		return nil, "", fmt.Errorf("choosing manifest digest algorithm: %w", err)
+	}
+	manifestDigest, err := manifest.DigestWithAlgorithm(man, manifestDigestAlgo)
 	if err != nil {
 		return nil, "", err
 	}
@@ -611,13 +627,33 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanc
 	return man, manifestDigest, nil
 }
 
+// updateManifestConfigDigest updates the config digest in the manifest using the manifest abstraction layer.
+func (ic *imageCopier) updateManifestConfigDigest(manifestBlob []byte, src types.Image, newConfigDigest digest.Digest) ([]byte, error) {
+	_, mt, err := src.Manifest(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("getting manifest type: %w", err)
+	}
+
+	m, err := manifest.FromBlob(manifestBlob, mt)
+	if err != nil {
+		return nil, fmt.Errorf("parsing manifest: %w", err)
+	}
+
+	if err := m.UpdateConfigDigest(newConfigDigest); err != nil {
+		return nil, err
+	}
+
+	return m.Serialize()
+}
+
 // copyConfig copies config.json, if any, from src to dest.
-func (ic *imageCopier) copyConfig(ctx context.Context, src types.Image) error {
+// It returns the new config digest if it changed (due to digest algorithm forcing), or nil otherwise.
+func (ic *imageCopier) copyConfig(ctx context.Context, src types.Image) (*digest.Digest, error) {
 	srcInfo := src.ConfigInfo()
 	if srcInfo.Digest != "" {
 		if err := ic.c.concurrentBlobCopiesSemaphore.Acquire(ctx, 1); err != nil {
 			// This can only fail with ctx.Err(), so no need to blame acquiring the semaphore.
-			return fmt.Errorf("copying config: %w", err)
+			return nil, fmt.Errorf("copying config: %w", err)
 		}
 		defer ic.c.concurrentBlobCopiesSemaphore.Release(1)
 
@@ -645,13 +681,17 @@ func (ic *imageCopier) copyConfig(ctx context.Context, src types.Image) error {
 			return destInfo, nil
 		}()
 		if err != nil {
-			return err
+			return nil, err
 		}
 		if destInfo.Digest != srcInfo.Digest {
-			return fmt.Errorf("Internal error: copying uncompressed config blob %s changed digest to %s", srcInfo.Digest, destInfo.Digest)
+			// If algorithms match, the whole digest values must match
+			if destInfo.Digest.Algorithm() == srcInfo.Digest.Algorithm() {
+				return nil, fmt.Errorf("Internal error: copying uncompressed config blob %s changed digest to %s", srcInfo.Digest, destInfo.Digest)
+			}
+			return &destInfo.Digest, nil
 		}
 	}
-	return nil
+	return nil, nil
 }
 
 // diffIDResult contains both a digest value and an error from diffIDComputationGoroutine.
@@ -743,19 +783,34 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
 			tocDigest = *d
 		}
 
-		reused, reusedBlob, err := ic.c.dest.TryReusingBlobWithOptions(ctx, srcInfo, private.TryReusingBlobOptions{
-			Cache:                   ic.c.blobInfoCache,
-			CanSubstitute:           canSubstitute,
-			EmptyLayer:              emptyLayer,
-			LayerIndex:              &layerIndex,
-			SrcRef:                  srcRef,
-			PossibleManifestFormats: append([]string{ic.manifestConversionPlan.preferredMIMEType}, ic.manifestConversionPlan.otherMIMETypeCandidates...),
-			RequiredCompression:     requiredCompression,
-			OriginalCompression:     srcInfo.CompressionAlgorithm,
-			TOCDigest:               tocDigest,
-		})
-		if err != nil {
-			return types.BlobInfo{}, "", fmt.Errorf("trying to reuse blob %s at destination: %w", srcInfo.Digest, err)
+		// FIXME: Blob reuse disabled when forcing different digest algorithm.
+		canTryReuse := true
+		if forcedAlgo := ic.c.options.digestOptions.MustUseSet(); forcedAlgo != "" {
+			if srcInfo.Digest.Algorithm() != forcedAlgo {
+				logrus.Debugf("Skipping blob reuse for %s: digest algorithm %s doesn't match forced algorithm %s",
+					srcInfo.Digest, srcInfo.Digest.Algorithm(), forcedAlgo)
+				canTryReuse = false
+			}
+		}
+
+		reused := false
+		var reusedBlob private.ReusedBlob
+		if canTryReuse {
+			var err error
+			reused, reusedBlob, err = ic.c.dest.TryReusingBlobWithOptions(ctx, srcInfo, private.TryReusingBlobOptions{
+				Cache:                   ic.c.blobInfoCache,
+				CanSubstitute:           canSubstitute,
+				EmptyLayer:              emptyLayer,
+				LayerIndex:              &layerIndex,
+				SrcRef:                  srcRef,
+				PossibleManifestFormats: append([]string{ic.manifestConversionPlan.preferredMIMEType}, ic.manifestConversionPlan.otherMIMETypeCandidates...),
+				RequiredCompression:     requiredCompression,
+				OriginalCompression:     srcInfo.CompressionAlgorithm,
+				TOCDigest:               tocDigest,
+			})
+			if err != nil {
+				return types.BlobInfo{}, "", fmt.Errorf("trying to reuse blob %s at destination: %w", srcInfo.Digest, err)
+			}
 		}
 		if reused {
 			logrus.Debugf("Skipping blob %s (already present):", srcInfo.Digest)

image/copy/single_test.go

@@ -159,3 +159,22 @@ func TestComputeDiffID(t *testing.T) {
 	_, err = computeDiffID(reader, nil)
 	assert.Error(t, err)
 }
+
+func TestBrokenSetForceDestinationDigestAlgorithm(t *testing.T) {
+	opts := &Options{}
+
+	// First call should succeed
+	err := opts.BrokenSetForceDestinationDigestAlgorithm(digest.SHA256)
+	require.NoError(t, err)
+
+	// Second call should fail because digest options are already configured
+	err = opts.BrokenSetForceDestinationDigestAlgorithm(digest.SHA512)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "digest options are already configured")
+
+	// Test with unavailable algorithm
+	opts2 := &Options{}
+	err = opts2.BrokenSetForceDestinationDigestAlgorithm("sha999")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "is not available")
+}

image/directory/directory_dest.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
+	"go.podman.io/image/v5/internal/digests"
 	"go.podman.io/image/v5/internal/imagedestination/impl"
 	"go.podman.io/image/v5/internal/imagedestination/stubs"
 	"go.podman.io/image/v5/internal/private"
@@ -150,7 +151,11 @@ func (d *dirImageDestination) PutBlobWithOptions(ctx context.Context, stream io.
 		}
 	}()
 
-	digester, stream := putblobdigest.DigestIfUnknown(stream, inputInfo)
+	algorithm, err := options.Digests.Choose(digests.Situation{Preexisting: inputInfo.Digest, CannotChangeAlgorithmReason: options.CannotChangeDigestReason})
+	if err != nil {
+		return private.UploadedBlob{}, err
+	}
+	digester, stream := putblobdigest.DigestIfAlgorithmUnknown(stream, inputInfo, algorithm)
 
 	// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
 	size, err := io.Copy(blobFile, stream)