diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9870d9483b78a..5f136ccc15a83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -437,6 +437,14 @@ jobs: run: | sudo modprobe erofs + - name: Load dm-verity kernel module + run: | + sudo modprobe dm_verity + + - name: Verify veritysetup version + run: | + veritysetup --version + - name: Install containerd env: CGO_ENABLED: 1 diff --git a/internal/dmverity/dmverity.go b/internal/dmverity/dmverity.go new file mode 100644 index 0000000000000..eec65afd7c027 --- /dev/null +++ b/internal/dmverity/dmverity.go @@ -0,0 +1,174 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package dmverity provides functions for working with dm-verity for integrity verification +package dmverity + +import ( + "bufio" + "bytes" + "encoding/hex" + "fmt" + "os" + "strings" + + "github.com/containerd/log" +) + +// VeritySetupCommand represents the type of veritysetup command to execute +type VeritySetupCommand string + +const ( + // FormatCommand corresponds to "veritysetup format" + FormatCommand VeritySetupCommand = "format" + // OpenCommand corresponds to "veritysetup open" + OpenCommand VeritySetupCommand = "open" + // CloseCommand corresponds to "veritysetup close" + CloseCommand VeritySetupCommand = "close" +) + +// DmverityOptions contains configuration options for dm-verity operations +type DmverityOptions struct { + // Salt for hashing, represented as a hex string + Salt string + // Hash algorithm to use (default: sha256) + HashAlgorithm string + // Size of data blocks in bytes (default: 4096) + DataBlockSize uint32 + // Size of hash blocks in bytes (default: 4096) + HashBlockSize uint32 + // Number of data blocks + DataBlocks uint64 + // Offset of hash area in bytes + HashOffset uint64 + // Hash type (default: 1) + HashType uint32 + // Superblock usage flag (false meaning --no-superblock) + UseSuperblock bool + // Debug flag + Debug bool + // UUID for device to use + UUID string + // RootHashFile specifies a file path where the root hash should be saved + RootHashFile string +} + +// DefaultDmverityOptions returns a DmverityOptions struct with default values +func DefaultDmverityOptions() DmverityOptions { + return DmverityOptions{ + Salt: "0000000000000000000000000000000000000000000000000000000000000000", + HashAlgorithm: "sha256", + DataBlockSize: 4096, + HashBlockSize: 4096, + HashType: 1, + UseSuperblock: true, + } +} + +// ValidateOptions validates dm-verity options to ensure they are valid +// before being passed to veritysetup commands +func ValidateOptions(opts *DmverityOptions) error { + if opts == nil { + return fmt.Errorf("options cannot be nil") + } + + // Validate block sizes are power of 2 (kernel requirement) + if opts.DataBlockSize > 0 { + if opts.DataBlockSize&(opts.DataBlockSize-1) != 0 { + return fmt.Errorf("data block size %d must be a power of 2", opts.DataBlockSize) + } + } + + if opts.HashBlockSize > 0 { + if opts.HashBlockSize&(opts.HashBlockSize-1) != 0 { + return fmt.Errorf("hash block size %d must be a power of 2", opts.HashBlockSize) + } + } + + // Validate salt format (must be hex string) + if opts.Salt != "" { + if _, err := hex.DecodeString(opts.Salt); err != nil { + return fmt.Errorf("salt must be a valid hex string: %w", err) + } + } + + return nil +} + +// ValidateRootHash validates that a root hash string is in valid hexadecimal format +func ValidateRootHash(rootHash string) error { + if rootHash == "" { + return fmt.Errorf("root hash cannot be empty") + } + + // Validate root hash (must be hex string) + if _, err := hex.DecodeString(rootHash); err != nil { + return fmt.Errorf("root hash must be a valid hex string: %w", err) + } + + return nil +} + +// ExtractRootHash extracts the root hash from veritysetup format command output. +// It first attempts to read from the root hash file (if specified in opts.RootHashFile), +// then falls back to parsing the stdout output. +// +// Note: This function expects English output when parsing stdout. The calling code +// ensures veritysetup runs with LC_ALL=C and LANG=C to prevent localization issues. +func ExtractRootHash(output string, opts *DmverityOptions) (string, error) { + log.L.Debugf("veritysetup format output:\n%s", output) + + var rootHash string + + // Try to read from root hash file first (if specified) + if opts != nil && opts.RootHashFile != "" { + hashBytes, err := os.ReadFile(opts.RootHashFile) + if err != nil { + return "", fmt.Errorf("failed to read root hash from file %q: %w", opts.RootHashFile, err) + } + // Trim any whitespace/newlines + rootHash = string(bytes.TrimSpace(hashBytes)) + } else { + // Parse stdout output to find the root hash + if output == "" { + return "", fmt.Errorf("output is empty") + } + + scanner := bufio.NewScanner(strings.NewReader(output)) + for scanner.Scan() { + line := scanner.Text() + // Look for the "Root hash:" line + if strings.HasPrefix(line, "Root hash:") { + parts := strings.Split(line, ":") + if len(parts) == 2 { + rootHash = strings.TrimSpace(parts[1]) + break + } + } + } + + if err := scanner.Err(); err != nil { + return "", fmt.Errorf("error scanning output: %w", err) + } + } + + // Validate root hash + if err := ValidateRootHash(rootHash); err != nil { + return "", fmt.Errorf("root hash is invalid: %w", err) + } + + return rootHash, nil +} diff --git a/internal/dmverity/dmverity_linux.go b/internal/dmverity/dmverity_linux.go new file mode 100644 index 0000000000000..ac31b8cbbd2b1 --- /dev/null +++ b/internal/dmverity/dmverity_linux.go @@ -0,0 +1,192 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dmverity + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "time" +) + +const ( + // veritysetupTimeout is the maximum time allowed for a veritysetup command to complete + // Format operations can take time for large devices, but should complete within 5 minutes + veritysetupTimeout = 5 * time.Minute +) + +func IsSupported() (bool, error) { + moduleData, err := os.ReadFile("/proc/modules") + if err != nil { + return false, fmt.Errorf("failed to read /proc/modules: %w", err) + } + if !bytes.Contains(moduleData, []byte("dm_verity")) { + return false, fmt.Errorf("dm_verity module not loaded") + } + + veritysetupPath, err := exec.LookPath("veritysetup") + if err != nil { + return false, fmt.Errorf("veritysetup not found in PATH: %w", err) + } + + cmd := exec.Command(veritysetupPath, "--version") + if _, err := cmd.CombinedOutput(); err != nil { + return false, fmt.Errorf("veritysetup not functional: %w", err) + } + + return true, nil +} + +// runVeritySetup executes a veritysetup command with the given arguments and options +func actions(cmd VeritySetupCommand, args []string, opts *DmverityOptions) (string, error) { + cmdArgs := []string{string(cmd)} + + if opts == nil { + defaultOpts := DefaultDmverityOptions() + opts = &defaultOpts + } + + // Validate options before building command + if err := ValidateOptions(opts); err != nil { + return "", fmt.Errorf("invalid dm-verity options: %w", err) + } + + // Apply options based on command type according to veritysetup man page + switch cmd { + case FormatCommand: + // FORMAT options: --hash, --no-superblock, --format, --data-block-size, + // --hash-block-size, --data-blocks, --hash-offset, --salt, --uuid, --root-hash-file + if opts.Salt != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("--salt=%s", opts.Salt)) + } + if opts.HashAlgorithm != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("--hash=%s", opts.HashAlgorithm)) + } + if opts.DataBlockSize > 0 { + cmdArgs = append(cmdArgs, fmt.Sprintf("--data-block-size=%d", opts.DataBlockSize)) + } + if opts.HashBlockSize > 0 { + cmdArgs = append(cmdArgs, fmt.Sprintf("--hash-block-size=%d", opts.HashBlockSize)) + } + if opts.DataBlocks > 0 { + cmdArgs = append(cmdArgs, fmt.Sprintf("--data-blocks=%d", opts.DataBlocks)) + } + if opts.HashOffset > 0 { + cmdArgs = append(cmdArgs, fmt.Sprintf("--hash-offset=%d", opts.HashOffset)) + } + if opts.HashType == 0 { + cmdArgs = append(cmdArgs, fmt.Sprintf("--format=%d", opts.HashType)) + } + if !opts.UseSuperblock { + cmdArgs = append(cmdArgs, "--no-superblock") + } + if opts.UUID != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("--uuid=%s", opts.UUID)) + } + if opts.RootHashFile != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("--root-hash-file=%s", opts.RootHashFile)) + } + + case OpenCommand: + // OPEN options: --hash-offset, --no-superblock, --root-hash-file + // (ignoring advanced options we don't have: --ignore-corruption, --restart-on-corruption, + // --panic-on-corruption, --ignore-zero-blocks, --check-at-most-once, + // --root-hash-signature, --use-tasklets, --shared) + if opts.HashOffset > 0 { + cmdArgs = append(cmdArgs, fmt.Sprintf("--hash-offset=%d", opts.HashOffset)) + } + if !opts.UseSuperblock { + cmdArgs = append(cmdArgs, "--no-superblock") + } + if opts.RootHashFile != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("--root-hash-file=%s", opts.RootHashFile)) + } + + case CloseCommand: + // CLOSE has minimal options (--deferred, --cancel-deferred not implemented) + // No options from DmverityOptions apply to close + } + + // Debug is not command-specific, can be used with any command + if opts.Debug { + cmdArgs = append(cmdArgs, "--debug") + } + + cmdArgs = append(cmdArgs, args...) + + ctx, cancel := context.WithTimeout(context.Background(), veritysetupTimeout) + defer cancel() + + execCmd := exec.CommandContext(ctx, "veritysetup", cmdArgs...) + // Force C locale to ensure consistent, non-localized output that we can parse reliably + // This prevents localization issues where field names like "Root hash", "Salt", etc. + // might be translated to other languages, breaking our text parsing + execCmd.Env = append(os.Environ(), "LC_ALL=C", "LANG=C") + output, err := execCmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("veritysetup %s failed: %w, output: %s", cmd, err, string(output)) + } + + return string(output), nil +} + +// Format creates a dm-verity hash for a data device and returns the root hash. +// If hashDevice is the same as dataDevice, the hash will be stored on the same device. +func Format(dataDevice, hashDevice string, opts *DmverityOptions) (string, error) { + args := []string{dataDevice, hashDevice} + output, err := actions(FormatCommand, args, opts) + if err != nil { + return "", fmt.Errorf("failed to format dm-verity device: %w, output: %s", err, output) + } + + // Extract the root hash from the format output + // Pass opts so ExtractRootHash can read root hash from file if RootHashFile was specified + rootHash, err := ExtractRootHash(output, opts) + if err != nil { + return "", fmt.Errorf("failed to extract root hash: %w", err) + } + + return rootHash, nil +} + +// Open creates a read-only device-mapper target for transparent integrity verification +func Open(dataDevice string, name string, hashDevice string, rootHash string, opts *DmverityOptions) (string, error) { + var args []string + // If RootHashFile is provided, use the alternate open syntax without root hash as command arg + if opts != nil && opts.RootHashFile != "" { + args = []string{dataDevice, name, hashDevice} + } else { + args = []string{dataDevice, name, hashDevice, rootHash} + } + output, err := actions(OpenCommand, args, opts) + if err != nil { + return "", fmt.Errorf("failed to open dm-verity device: %w, output: %s", err, output) + } + return output, nil +} + +// Close removes a dm-verity target and its underlying device from the device mapper table +func Close(name string) (string, error) { + args := []string{name} + output, err := actions(CloseCommand, args, nil) + if err != nil { + return "", fmt.Errorf("failed to close dm-verity device: %w, output: %s", err, output) + } + return output, nil +} diff --git a/internal/dmverity/dmverity_other.go b/internal/dmverity/dmverity_other.go new file mode 100644 index 0000000000000..c62e55335e1b9 --- /dev/null +++ b/internal/dmverity/dmverity_other.go @@ -0,0 +1,39 @@ +//go:build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dmverity + +import "fmt" + +var errUnsupported = fmt.Errorf("dmverity is only supported on Linux systems") + +func IsSupported() (bool, error) { + return false, errUnsupported +} + +func Format(_ string, _ string, _ *DmverityOptions) (string, error) { + return "", errUnsupported +} + +func Open(_ string, _ string, _ string, _ string, _ *DmverityOptions) (string, error) { + return "", errUnsupported +} + +func Close(_ string) (string, error) { + return "", errUnsupported +} diff --git a/internal/dmverity/dmverity_test.go b/internal/dmverity/dmverity_test.go new file mode 100644 index 0000000000000..0d7fb373dd689 --- /dev/null +++ b/internal/dmverity/dmverity_test.go @@ -0,0 +1,462 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dmverity + +import ( + "bytes" + "os" + "testing" + + "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/containerd/v2/pkg/testutil" + "github.com/docker/go-units" + "github.com/stretchr/testify/assert" +) + +const ( + testDeviceName = "test-verity-device" +) + +func TestDMVerity(t *testing.T) { + testutil.RequiresRoot(t) + + supported, err := IsSupported() + if !supported || err != nil { + t.Skipf("dm-verity is not supported on this system: %v", err) + } + + tempDir := t.TempDir() + _, loopDevice := createLoopbackDevice(t, tempDir, "1Mb") + defer func() { + assert.NoError(t, mount.DetachLoopDevice(loopDevice)) + }() + + opts := DmverityOptions{ + Salt: "0000000000000000000000000000000000000000000000000000000000000000", + HashAlgorithm: "sha256", + DataBlockSize: 4096, + HashBlockSize: 4096, + DataBlocks: 256, + HashOffset: 1048576, + HashType: 1, + UseSuperblock: true, + Debug: false, + } + + t.Run("IsSupported", func(t *testing.T) { + supported, err := IsSupported() + assert.True(t, supported) + assert.NoError(t, err) + }) + + var rootHash string + + t.Run("Format", func(t *testing.T) { + var err error + rootHash, err = Format(loopDevice, loopDevice, &opts) + assert.NoError(t, err) + assert.NotEmpty(t, rootHash) + }) + + t.Run("Format_WithRootHashFile", func(t *testing.T) { + rootHashFile, err := os.CreateTemp(tempDir, "root-hash-*.txt") + assert.NoError(t, err) + rootHashFilePath := rootHashFile.Name() + rootHashFile.Close() + defer os.Remove(rootHashFilePath) + + _, loopDevice2 := createLoopbackDevice(t, tempDir, "1Mb") + defer func() { + assert.NoError(t, mount.DetachLoopDevice(loopDevice2)) + }() + + optsWithFile := opts + optsWithFile.RootHashFile = rootHashFilePath + + rootHashFromFormat, err := Format(loopDevice2, loopDevice2, &optsWithFile) + assert.NoError(t, err) + assert.NotEmpty(t, rootHashFromFormat) + + fileContent, err := os.ReadFile(rootHashFilePath) + assert.NoError(t, err) + fileHash := string(bytes.TrimSpace(fileContent)) + assert.Equal(t, rootHashFromFormat, fileHash) + }) + + t.Run("Format_NoSuperblock", func(t *testing.T) { + _, loopDevice3 := createLoopbackDevice(t, tempDir, "1Mb") + defer func() { + assert.NoError(t, mount.DetachLoopDevice(loopDevice3)) + }() + + optsNoSuperblock := opts + optsNoSuperblock.UseSuperblock = false + + rootHashNoSuperblock, err := Format(loopDevice3, loopDevice3, &optsNoSuperblock) + assert.NoError(t, err) + assert.NotEmpty(t, rootHashNoSuperblock) + }) + + t.Run("Open", func(t *testing.T) { + _, err := Open(loopDevice, testDeviceName, loopDevice, rootHash, &opts) + assert.NoError(t, err) + + _, err = os.Stat("/dev/mapper/" + testDeviceName) + assert.NoError(t, err) + }) + + t.Run("Open_WithRootHashFile", func(t *testing.T) { + // Create a root hash file + rootHashFile, err := os.CreateTemp(tempDir, "root-hash-open-*.txt") + assert.NoError(t, err) + rootHashFilePath := rootHashFile.Name() + + // Write the root hash to the file + _, err = rootHashFile.WriteString(rootHash) + assert.NoError(t, err) + rootHashFile.Close() + defer os.Remove(rootHashFilePath) + + // Create a new loopback device for this test + _, loopDevice4 := createLoopbackDevice(t, tempDir, "1Mb") + defer func() { + assert.NoError(t, mount.DetachLoopDevice(loopDevice4)) + }() + + // Format the device first + optsFormat := opts + _, err = Format(loopDevice4, loopDevice4, &optsFormat) + assert.NoError(t, err) + + // Open with root hash file instead of command-line arg + optsOpen := opts + optsOpen.RootHashFile = rootHashFilePath + deviceName := "test-verity-roothashfile" + _, err = Open(loopDevice4, deviceName, loopDevice4, "", &optsOpen) + assert.NoError(t, err) + + // Verify device was created + _, err = os.Stat("/dev/mapper/" + deviceName) + assert.NoError(t, err) + + // Clean up + _, err = Close(deviceName) + assert.NoError(t, err) + }) + + t.Run("Close", func(t *testing.T) { + _, err := Close(testDeviceName) + assert.NoError(t, err) + + _, err = os.Stat("/dev/mapper/" + testDeviceName) + assert.True(t, os.IsNotExist(err)) + }) +} + +func createLoopbackDevice(t *testing.T, dir string, size string) (string, string) { + file, err := os.CreateTemp(dir, "dmverity-tests-") + assert.NoError(t, err) + + sizeInBytes, err := units.RAMInBytes(size) + assert.NoError(t, err) + + err = file.Truncate(sizeInBytes * 2) + assert.NoError(t, err) + + err = file.Close() + assert.NoError(t, err) + + imagePath := file.Name() + + loopDevice, err := mount.AttachLoopDevice(imagePath) + assert.NoError(t, err) + + return imagePath, loopDevice +} + +func TestValidateOptions(t *testing.T) { + tests := []struct { + name string + opts *DmverityOptions + wantErr bool + errMsg string + }{ + { + name: "nil options", + opts: nil, + wantErr: true, + errMsg: "options cannot be nil", + }, + { + name: "valid options", + opts: func() *DmverityOptions { o := DefaultDmverityOptions(); return &o }(), + wantErr: false, + }, + { + name: "invalid data block size", + opts: func() *DmverityOptions { + o := DefaultDmverityOptions() + o.DataBlockSize = 4097 // Not a power of 2 + return &o + }(), + wantErr: true, + errMsg: "data block size", + }, + { + name: "invalid hash block size", + opts: func() *DmverityOptions { + o := DefaultDmverityOptions() + o.HashBlockSize = 4097 // Not a power of 2 + return &o + }(), + wantErr: true, + errMsg: "hash block size", + }, + { + name: "invalid salt hex", + opts: func() *DmverityOptions { + o := DefaultDmverityOptions() + o.Salt = "not-a-hex-string" + return &o + }(), + wantErr: true, + errMsg: "salt must be a valid hex string", + }, + { + name: "empty salt allowed", + opts: func() *DmverityOptions { + o := DefaultDmverityOptions() + o.Salt = "" + return &o + }(), + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateOptions(tt.opts) + if tt.wantErr { + assert.Error(t, err) + if tt.errMsg != "" { + assert.Contains(t, err.Error(), tt.errMsg) + } + } else { + assert.NoError(t, err) + } + }) + } + + // Test multiple valid power-of-2 sizes + t.Run("valid power of 2 sizes", func(t *testing.T) { + validSizes := []uint32{512, 1024, 2048, 4096, 8192} + for _, size := range validSizes { + opts := DefaultDmverityOptions() + opts.DataBlockSize = size + opts.HashBlockSize = size + err := ValidateOptions(&opts) + assert.NoError(t, err, "size %d should be valid", size) + } + }) +} + +func TestValidateRootHash(t *testing.T) { + tests := []struct { + name string + hash string + wantErr bool + errMsg string + }{ + { + name: "empty root hash", + hash: "", + wantErr: true, + errMsg: "root hash cannot be empty", + }, + { + name: "valid root hash", + hash: "bef46122f85025cf37061b16c04e2a19960a5bbcdbb656b5e91ae7927c0ad807", + wantErr: false, + }, + { + name: "invalid hex characters", + hash: "not-a-valid-hex-hash", + wantErr: true, + errMsg: "root hash must be a valid hex string", + }, + { + name: "odd length hex", + hash: "abc", + wantErr: true, + errMsg: "root hash must be a valid hex string", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateRootHash(tt.hash) + if tt.wantErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errMsg) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExtractRootHash(t *testing.T) { + validOutput := `VERITY header information for /dev/loop0 +UUID: eebbec4a-a914-4089-aca0-22266b21bd2b +Hash type: 1 +Data blocks: 256 +Data block size: 4096 +Hash block size: 4096 +Hash algorithm: sha256 +Salt: 0000000000000000000000000000000000000000000000000000000000000000 +Root hash: bef46122f85025cf37061b16c04e2a19960a5bbcdbb656b5e91ae7927c0ad807` + + tests := []struct { + name string + output string + opts *DmverityOptions + wantErr bool + errMsg string + check func(*testing.T, string) + }{ + { + name: "empty output", + output: "", + wantErr: true, + errMsg: "output is empty", + }, + { + name: "valid output", + output: validOutput, + wantErr: false, + check: func(t *testing.T, rootHash string) { + assert.Equal(t, "bef46122f85025cf37061b16c04e2a19960a5bbcdbb656b5e91ae7927c0ad807", rootHash) + }, + }, + { + name: "missing root hash", + output: `VERITY header information for /dev/loop0 +UUID: eebbec4a-a914-4089-aca0-22266b21bd2b +Hash type: 1 +Data blocks: 256 +Data block size: 4096 +Hash block size: 4096 +Hash algorithm: sha256 +Salt: 0000000000000000000000000000000000000000000000000000000000000000`, + wantErr: true, + errMsg: "root hash", + }, + { + name: "invalid root hash", + output: `VERITY header information for /dev/loop0 +UUID: eebbec4a-a914-4089-aca0-22266b21bd2b +Hash type: 1 +Data blocks: 256 +Data block size: 4096 +Hash block size: 4096 +Hash algorithm: sha256 +Salt: 0000000000000000000000000000000000000000000000000000000000000000 +Root hash: not-a-valid-hex-hash`, + wantErr: true, + errMsg: "root hash is invalid", + }, + { + name: "root hash from nonexistent file", + output: `VERITY header information for /dev/loop0 +UUID: eebbec4a-a914-4089-aca0-22266b21bd2b +Hash type: 1 +Data blocks: 256 +Data block size: 4096 +Hash block size: 4096 +Hash algorithm: sha256 +Salt: 0000000000000000000000000000000000000000000000000000000000000000`, + opts: &DmverityOptions{RootHashFile: "/nonexistent/path/to/hash/file"}, + wantErr: true, + errMsg: "failed to read root hash from file", + }, + { + name: "root hash from file takes priority", + output: `VERITY header information for /dev/loop0 +UUID: eebbec4a-a914-4089-aca0-22266b21bd2b +Hash type: 1 +Data blocks: 256 +Data block size: 4096 +Hash block size: 4096 +Hash algorithm: sha256 +Salt: 0000000000000000000000000000000000000000000000000000000000000000 +Root hash: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, + opts: func() *DmverityOptions { + // Create a temporary file with a different hash + tmpFile, err := os.CreateTemp("", "root-hash-test-*.txt") + assert.NoError(t, err) + tmpFilePath := tmpFile.Name() + fileHash := "bef46122f85025cf37061b16c04e2a19960a5bbcdbb656b5e91ae7927c0ad807" + _, err = tmpFile.WriteString(fileHash) + assert.NoError(t, err) + tmpFile.Close() + // Cleanup will happen after the test runs + t.Cleanup(func() { os.Remove(tmpFilePath) }) + return &DmverityOptions{RootHashFile: tmpFilePath} + }(), + wantErr: false, + check: func(t *testing.T, rootHash string) { + // Should get the hash from file, not from output + assert.Equal(t, "bef46122f85025cf37061b16c04e2a19960a5bbcdbb656b5e91ae7927c0ad807", rootHash) + }, + }, + { + name: "skips header lines", + output: `VERITY header information for /dev/loop0 +# veritysetup format /dev/loop0 /dev/loop0 +UUID: eebbec4a-a914-4089-aca0-22266b21bd2b +Hash type: 1 +Data blocks: 256 +Data block size: 4096 +Hash block size: 4096 +Hash algorithm: sha256 +Salt: 0000000000000000000000000000000000000000000000000000000000000000 +Root hash: bef46122f85025cf37061b16c04e2a19960a5bbcdbb656b5e91ae7927c0ad807`, + wantErr: false, + check: func(t *testing.T, rootHash string) { + assert.Equal(t, "bef46122f85025cf37061b16c04e2a19960a5bbcdbb656b5e91ae7927c0ad807", rootHash) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rootHash, err := ExtractRootHash(tt.output, tt.opts) + if tt.wantErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), tt.errMsg) + } else { + assert.NoError(t, err) + assert.NotEmpty(t, rootHash) + if tt.check != nil { + tt.check(t, rootHash) + } + } + }) + } +} diff --git a/plugins/snapshots/erofs/erofs_linux.go b/plugins/snapshots/erofs/erofs_linux.go index 5e254df7a3a3b..61ff6e86f601a 100644 --- a/plugins/snapshots/erofs/erofs_linux.go +++ b/plugins/snapshots/erofs/erofs_linux.go @@ -25,17 +25,18 @@ import ( "path/filepath" "strings" "syscall" - - "github.com/containerd/continuity/fs" - "github.com/containerd/log" - "github.com/containerd/plugin" - "golang.org/x/sys/unix" + "time" "github.com/containerd/containerd/v2/core/mount" "github.com/containerd/containerd/v2/core/snapshots" "github.com/containerd/containerd/v2/core/snapshots/storage" + "github.com/containerd/containerd/v2/internal/dmverity" "github.com/containerd/containerd/v2/internal/erofsutils" "github.com/containerd/containerd/v2/internal/fsverity" + "github.com/containerd/continuity/fs" + "github.com/containerd/log" + "github.com/containerd/plugin" + "golang.org/x/sys/unix" ) // SnapshotterConfig is used to configure the erofs snapshotter instance @@ -46,6 +47,8 @@ type SnapshotterConfig struct { enableFsverity bool // setImmutable enables IMMUTABLE_FL file attribute for EROFS layers setImmutable bool + // enableDmverity enables dmverity for EROFS layers + enableDmverity bool } // Opt is an option to configure the erofs snapshotter @@ -72,6 +75,13 @@ func WithImmutable() Opt { } } +// WithDmverity enables dmverity for EROFS layers +func WithDmverity() Opt { + return func(config *SnapshotterConfig) { + config.enableDmverity = true + } +} + type MetaStore interface { TransactionContext(ctx context.Context, writable bool) (context.Context, storage.Transactor, error) WithTransaction(ctx context.Context, writable bool, fn storage.TransactionCallback) error @@ -84,6 +94,7 @@ type snapshotter struct { ovlOptions []string enableFsverity bool setImmutable bool + enableDmverity bool } // check if EROFS kernel filesystem is registered or not @@ -131,6 +142,17 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { return nil, fmt.Errorf("EROFS unsupported, please `modprobe erofs`: %w", plugin.ErrSkipPlugin) } + // Ensure fsverity and dmverity are not both enabled + if config.enableFsverity && config.enableDmverity { + return nil, fmt.Errorf("fsverity and dmverity cannot be enabled simultaneously") + } + + // Ensure setImmutable and dmverity are not both enabled + // dmverity needs to modify the file to add hash trees, which conflicts with IMMUTABLE_FL + if config.setImmutable && config.enableDmverity { + return nil, fmt.Errorf("setImmutable and dmverity cannot be enabled simultaneously") + } + // Check fsverity support if enabled if config.enableFsverity { supported, err := fsverity.IsSupported(root) @@ -142,6 +164,16 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { } } + if config.enableDmverity { + supported, err := dmverity.IsSupported() + if err != nil { + return nil, fmt.Errorf("failed to check dmverity support on %q: %w", root, err) + } + if !supported { + return nil, fmt.Errorf("dmverity is not supported on the filesystem of %q", root) + } + } + ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db")) if err != nil { return nil, err @@ -157,11 +189,17 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { ovlOptions: config.ovlOptions, enableFsverity: config.enableFsverity, setImmutable: config.setImmutable, + enableDmverity: config.enableDmverity, }, nil } // Close closes the snapshotter func (s *snapshotter) Close() error { + // Close the metadata store. We intentionally do not close dm-verity devices here + // to avoid disrupting running containers. Devices will be cleaned up when: + // - Snapshots are removed via Remove() + // - The system reboots + // - Manual cleanup is performed return s.ms.Close() } @@ -178,6 +216,91 @@ func (s *snapshotter) layerBlobPath(id string) string { return filepath.Join(s.root, "snapshots", id, "layer.erofs") } +// dmverityDeviceName returns the dm-verity device name for a snapshot ID +func (s *snapshotter) dmverityDeviceName(id string) string { + return fmt.Sprintf("containerd-erofs-%s", id) +} + +func (s *snapshotter) formatLayerBlob(id string) error { + layerBlob := s.layerBlobPath(id) + if _, err := os.Stat(layerBlob); err != nil { + return fmt.Errorf("failed to find valid erofs layer blob: %w", err) + } + if !s.isLayerWithDmverity(id) { + opts := dmverity.DefaultDmverityOptions() + fileinfo, err := os.Stat(layerBlob) + if err != nil { + return fmt.Errorf("failed to stat layer blob: %w", err) + } + + // Open file for truncating + f, err := os.OpenFile(layerBlob, os.O_RDWR, 0644) + if err != nil { + return fmt.Errorf("failed to open layer blob for truncating: %w", err) + } + defer f.Close() + file_size := fileinfo.Size() + // Truncate the file to double its size to provide space for the dm-verity hash tree. + // The hash tree will never exceed the original data size. + // Most filesystems use sparse allocation, so unused space doesn't consume disk. + if err := f.Truncate(file_size * 2); err != nil { + return fmt.Errorf("failed to truncate layer blob: %w", err) + } + opts.HashOffset = uint64(file_size) + + // Create a persistent file for the root hash + rootHashFile := filepath.Join(s.root, "snapshots", id, ".roothash") + opts.RootHashFile = rootHashFile + + _, err = dmverity.Format(layerBlob, layerBlob, &opts) + if err != nil { + return fmt.Errorf("failed to format dmverity: %w", err) + } + } + return nil +} + +// getDmverityDevicePath opens a dm-verity device for the given snapshot ID if not already open, +// and returns the device path. If the device is already open, returns the existing device path. +func (s *snapshotter) getDmverityDevicePath(id string) (string, error) { + layerBlob := s.layerBlobPath(id) + if _, err := os.Stat(layerBlob); err != nil { + return "", fmt.Errorf("failed to find valid erofs layer blob: %w", err) + } + dmName := s.dmverityDeviceName(id) + devicePath := fmt.Sprintf("/dev/mapper/%s", dmName) + if _, err := os.Stat(devicePath); err == nil { + return devicePath, nil + } + + // Calculate the hash offset dynamically from the file size + // The file was truncated to double its original size in formatLayerBlob + fileInfo, err := os.Stat(layerBlob) + if err != nil { + return "", fmt.Errorf("failed to stat layer blob: %w", err) + } + originalSize := uint64(fileInfo.Size() / 2) + + if _, err := os.Stat(devicePath); err != nil { + opts := dmverity.DefaultDmverityOptions() + opts.HashOffset = originalSize + // Use the root hash file instead of passing root hash as command-line arg + opts.RootHashFile = filepath.Join(s.root, "snapshots", id, ".roothash") + _, err = dmverity.Open(layerBlob, dmName, layerBlob, "", &opts) + if err != nil { + return "", fmt.Errorf("failed to open dmverity device: %w", err) + } + + for i := 0; i < 50; i++ { + if _, err := os.Stat(devicePath); err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + } + return devicePath, nil +} + func (s *snapshotter) lowerPath(id string) (mount.Mount, string, error) { layerBlob := s.layerBlobPath(id) if _, err := os.Stat(layerBlob); err != nil { @@ -228,8 +351,18 @@ func (s *snapshotter) mounts(snap storage.Snapshot, info snapshots.Info) ([]moun return nil, err } } + if s.enableDmverity { + devicePath, err := s.getDmverityDevicePath(snap.ID) + if err != nil { + return nil, err + } + m.Source = devicePath + } // We have to force a loop device here since mount[] is static. - m.Options = append(m.Options, "loop") + // However, if we're using dmverity, we don't add loop here + if !s.enableDmverity { + m.Options = append(m.Options, "loop") + } return []mount.Mount{m}, nil } // if we only have one layer/no parents then just return a bind mount as overlay @@ -260,8 +393,18 @@ func (s *snapshotter) mounts(snap storage.Snapshot, info snapshots.Info) ([]moun if err != nil { return nil, err } + if s.enableDmverity { + devicePath, err := s.getDmverityDevicePath(snap.ParentIDs[0]) + if err != nil { + return nil, err + } + m.Source = devicePath + } // We have to force a loop device here too since mount[] is static. - m.Options = append(m.Options, "loop") + // However, if we're using dmverity, it's already a block device + if !s.enableDmverity { + m.Options = append(m.Options, "loop") + } return []mount.Mount{m}, nil } @@ -271,10 +414,9 @@ func (s *snapshotter) mounts(snap storage.Snapshot, info snapshots.Info) ([]moun if err != nil { return nil, err } - // If the lowerdir is actually an EROFS committed layer but - // doesn't have an EROFS mount. Let's recover now. - if mntpoint != m.Source && !isErofs(mntpoint) { + // doesn't have an EROFS mount. Let's recover now. + if !s.enableDmverity && mntpoint != m.Source && !isErofs(mntpoint) { err := m.Mount(mntpoint) // Use loop if the current kernel (6.12+) doesn't support file-backed mount if errors.Is(err, unix.ENOTBLK) { @@ -284,12 +426,23 @@ func (s *snapshotter) mounts(snap storage.Snapshot, info snapshots.Info) ([]moun if err != nil { return nil, err } + } else if s.enableDmverity { + devicePath, err := s.getDmverityDevicePath(snap.ParentIDs[i]) + if err != nil { + return nil, err + } + m.Source = devicePath + if mntpoint != m.Source && !isErofs(mntpoint) { + err = m.Mount(mntpoint) + if err != nil { + return nil, err + } + } } lowerdirs = append(lowerdirs, mntpoint) } options = append(options, fmt.Sprintf("lowerdir=%s", strings.Join(lowerdirs, ":"))) options = append(options, s.ovlOptions...) - return []mount.Mount{{ Type: "overlay", Source: "overlay", @@ -370,6 +523,13 @@ func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snap return s.createSnapshot(ctx, snapshots.KindView, key, parent, opts) } +func (s *snapshotter) isLayerWithDmverity(id string) bool { + if _, err := os.Stat(filepath.Join(s.root, "snapshots", id, ".roothash")); err != nil { + return false + } + return true +} + func setImmutable(path string, enable bool) error { //nolint:revive,staticcheck // silence "don't use ALL_CAPS in Go names; use CamelCase" const ( @@ -450,6 +610,14 @@ func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap log.G(ctx).WithError(err).Warnf("failed to set IMMUTABLE_FL for %s", layerBlob) } } + + if s.enableDmverity { + err := s.formatLayerBlob(id) + // Note: Device opening is deferred until mount time via getDmverityDevicePath + if err != nil { + return fmt.Errorf("failed to format dmverity: %w", err) + } + } return nil }) @@ -536,6 +704,10 @@ func (s *snapshotter) Remove(ctx context.Context, key string) (err error) { log.G(ctx).Warnf("failed to unmount EROFS mount for %v", id) } + if err := s.closeDmverityDevice(id); err != nil { + log.G(ctx).WithError(err).Warnf("failed to close dmverity device for %v", id) + } + for _, dir := range removals { if err := os.RemoveAll(dir); err != nil { log.G(ctx).WithError(err).WithField("path", dir).Warn("failed to remove directory") @@ -557,6 +729,10 @@ func (s *snapshotter) Remove(ctx context.Context, key string) (err error) { } // The layer blob is only persisted for committed snapshots. if k == snapshots.KindCommitted { + if err := s.closeDmverityDevice(id); err != nil { + log.G(ctx).WithError(err).Warnf("failed to close dmverity device for %v", id) + } + // Clear IMMUTABLE_FL before removal, since this flag avoids it. err = setImmutable(s.layerBlobPath(id), false) if err != nil { @@ -642,3 +818,18 @@ func (s *snapshotter) verifyFsverity(path string) error { } return nil } + +// closeDmverityDevice closes the dmverity device for a specific snapshot ID +func (s *snapshotter) closeDmverityDevice(id string) error { + if !s.enableDmverity || !s.isLayerWithDmverity(id) { + return nil + } + + dmName := s.dmverityDeviceName(id) + devicePath := fmt.Sprintf("/dev/mapper/%s", dmName) + if _, err := os.Stat(devicePath); err == nil { + _, err = dmverity.Close(dmName) + return err + } + return nil +} diff --git a/plugins/snapshots/erofs/erofs_linux_test.go b/plugins/snapshots/erofs/erofs_linux_test.go index 195cc41c9302f..8da5f468478c1 100644 --- a/plugins/snapshots/erofs/erofs_linux_test.go +++ b/plugins/snapshots/erofs/erofs_linux_test.go @@ -131,3 +131,41 @@ func TestErofsFsverity(t *testing.T) { t.Fatal("Expected direct write to fsverity-enabled layer to fail") } } + +func TestFsverityAndDmverityMutualExclusion(t *testing.T) { + testutil.RequiresRoot(t) + + root := t.TempDir() + + if !findErofs() { + t.Skip("check for erofs kernel support failed, skipping test") + } + + // Try to create snapshotter with both fsverity and dmverity enabled + _, err := NewSnapshotter(root, WithFsverity(), WithDmverity()) + if err == nil { + t.Fatal("Expected error when enabling both fsverity and dmverity") + } + if err.Error() != "fsverity and dmverity cannot be enabled simultaneously" { + t.Fatalf("Unexpected error message: %v", err) + } +} + +func TestSetImmutableAndDmverityMutualExclusion(t *testing.T) { + testutil.RequiresRoot(t) + + root := t.TempDir() + + if !findErofs() { + t.Skip("check for erofs kernel support failed, skipping test") + } + + // Try to create snapshotter with both setImmutable and dmverity enabled + _, err := NewSnapshotter(root, WithImmutable(), WithDmverity()) + if err == nil { + t.Fatal("Expected error when enabling both setImmutable and dmverity") + } + if err.Error() != "setImmutable and dmverity cannot be enabled simultaneously" { + t.Fatalf("Unexpected error message: %v", err) + } +} diff --git a/plugins/snapshots/erofs/plugin/plugin_linux.go b/plugins/snapshots/erofs/plugin/plugin_linux.go index ff64f2d0ea32c..dd525496c4cb0 100644 --- a/plugins/snapshots/erofs/plugin/plugin_linux.go +++ b/plugins/snapshots/erofs/plugin/plugin_linux.go @@ -39,6 +39,9 @@ type Config struct { // If `SetImmutable` is enabled, IMMUTABLE_FL will be set on layer blobs. SetImmutable bool `toml:"set_immutable"` + + // EnableDmverity enables dmverity for EROFS layers + EnableDmverity bool `toml:"enable_dmverity"` } func init() { @@ -72,6 +75,10 @@ func init() { opts = append(opts, erofs.WithImmutable()) } + if config.EnableDmverity { + opts = append(opts, erofs.WithDmverity()) + } + ic.Meta.Exports[plugins.SnapshotterRootDir] = root return erofs.NewSnapshotter(root, opts...) },