Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions e2e/scenario_win_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -550,3 +550,30 @@ func Test_Windows2025Gen2_McrChinaCloud_Windows(t *testing.T) {
},
})
}

func Test_NetworkIsolatedCluster_Windows_WithEgress(t *testing.T) {
RunScenario(t, &Scenario{
Description: "Tests that Windows nodes in network isolated clusters configure containerd to use the bootstrap profile container registry for MCR images",
Tags: Tags{
NetworkIsolated: true,
NonAnonymousACR: true,
},
Config: Config{
Cluster: ClusterAzureBootstrapProfileCache,
VHD: config.VHDWindows2025Gen2,
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
PrivateEgress: &datamodel.PrivateEgress{
Enabled: true,
ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)),
},
}
},
Validator: func(ctx context.Context, s *Scenario) {
// Verify mcr.microsoft.com host config exist
ValidateFileExists(ctx, s, `C:\ProgramData\containerd\certs.d\mcr.microsoft.com\hosts.toml`)
ValidateFileDoesNotExist(ctx, s, `C:\ProgramData\containerd\certs.d\mcr.azk8s.cn\hosts.toml`)
},
},
})
}
4 changes: 4 additions & 0 deletions parts/windows/kuberneteswindowssetup.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,10 @@ $global:WindowsCiliumNetworkingConfiguration = "{{GetVariable "nextGenNetworking
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="{{GetBootstrapProfileContainerRegistryServer}}"
$global:MCR_REPOSITORY_BASE="{{GetMCRRepositoryBase}}"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
4 changes: 4 additions & 0 deletions pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
4 changes: 4 additions & 0 deletions pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
4 changes: 4 additions & 0 deletions pkg/agent/testdata/AKSWindows2019+K8S116/CustomData
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
4 changes: 4 additions & 0 deletions pkg/agent/testdata/AKSWindows2019+K8S117/CustomData
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
4 changes: 4 additions & 0 deletions pkg/agent/testdata/AKSWindows2019+K8S118/CustomData
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
4 changes: 4 additions & 0 deletions pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
4 changes: 4 additions & 0 deletions pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
4 changes: 4 additions & 0 deletions pkg/agent/testdata/AKSWindows2019+K8S119/CustomData
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "{}";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "{}";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ $global:WindowsCiliumNetworkingConfiguration = "";
$global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn'
$global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'

# Network isolated cluster
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER=""
$global:MCR_REPOSITORY_BASE="mcr.microsoft.com/"

# Extract cse helper script from ZIP
[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
try {
Expand Down
46 changes: 45 additions & 1 deletion staging/cse/windows/containerdfunc.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,11 @@ function ProcessAndWriteContainerdConfig {

# Set up registry mirrors
Set-ContainerdRegistryConfig -Registry "docker.io" -RegistryHost "registry-1.docker.io"
Set-ContainerdRegistryConfig -Registry "mcr.azk8s.cn" -RegistryHost "mcr.azure.cn"
if ((Test-Path variable:global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER) -and -not [string]::IsNullOrEmpty($global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER)) {
Set-BootstrapProfileRegistryContainerdHost
} else {
Set-ContainerdRegistryConfig -Registry "mcr.azk8s.cn" -RegistryHost "mcr.azure.cn"
}

if (([version]$ContainerdVersion).CompareTo([version]"1.7.9") -lt 0) {
# Remove annotations placeholders for older containerd versions
Expand Down Expand Up @@ -230,6 +234,46 @@ server = "https://$Registry"
Write-Log "Wrote containerd hosts config for registry '$Registry' to '$hostsTomlPath'"
}

function Set-BootstrapProfileRegistryContainerdHost {
$mcrRegistry = if ((Test-Path variable:global:MCR_REPOSITORY_BASE) -and
-not [string]::IsNullOrEmpty($global:MCR_REPOSITORY_BASE)) {
[string]$global:MCR_REPOSITORY_BASE
}
else {
"mcr.microsoft.com"
}
$rootRegistryPath = "C:\ProgramData\containerd\certs.d"
$mcrRegistryPath = Join-Path $rootRegistryPath $mcrRegistry
$hostsTomlPath = Join-Path $mcrRegistryPath "hosts.toml"

$registryHost = [string]$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER
$registryHost = ($registryHost -replace '^https?://', '').TrimEnd('/')

$registryHostParts = $registryHost.Split('/', 2)
$registryHostName = $registryHostParts[0]
$registryRepoPrefix = if ($registryHostParts.Length -gt 1) { $registryHostParts[1].Trim('/') } else { "" }

$registryHost = if ([string]::IsNullOrEmpty($registryRepoPrefix)) {
"$registryHostName/v2"
}
else {
"$registryHostName/v2/$registryRepoPrefix"
}

Create-Directory -FullPath $mcrRegistryPath -DirectoryUsage "storing containerd registry hosts config"

$content = @"
server = "https://$mcrRegistry"

[host."https://$registryHost"]
capabilities = ["pull", "resolve"]
override_path = true
"@

$content | Out-File -FilePath $hostsTomlPath -Encoding ascii
Write-Log "Wrote bootstrap profile container registry hosts config from '$mcrRegistry' to '$registryHost' at '$hostsTomlPath'"
}

function Install-Containerd {
Param(
[Parameter(Mandatory = $true)][string]
Expand Down
64 changes: 64 additions & 0 deletions staging/cse/windows/containerdfunc.tests.ps1
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
BeforeAll {
if (-not (Get-PSDrive -Name C -ErrorAction SilentlyContinue)) {
New-PSDrive -Name C -PSProvider FileSystem -Root ([System.IO.Path]::GetTempPath()) | Out-Null
}

# Define mock functions before loading the scripts
function Write-Log {
param($Message)
Expand Down Expand Up @@ -287,3 +291,63 @@ Describe "Set-ContainerdRegistryConfig" {
$script:capturedEncoding | Should -Be "ascii"
}
}

Describe "Set-BootstrapProfileRegistryContainerdHost" {
BeforeEach {
Mock Create-Directory -MockWith {
param($FullPath, $DirectoryUsage)
}

$script:capturedFilePath = $null
$script:capturedEncoding = $null
$script:capturedContent = $null
Mock Out-File -MockWith {
param($InputObject, $FilePath, $Encoding)
$script:capturedFilePath = $FilePath
$script:capturedEncoding = $Encoding
$script:capturedContent = $InputObject
}
}

It "Should write hosts.toml for default mcr.microsoft.com when MCR_REPOSITORY_BASE is not set" {
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER = "myacr.azurecr.io"
if (Test-Path variable:global:MCR_REPOSITORY_BASE) {
Remove-Variable -Name MCR_REPOSITORY_BASE -Scope Global
}

Set-BootstrapProfileRegistryContainerdHost

Assert-MockCalled -CommandName 'Create-Directory' -Exactly -Times 1 -ParameterFilter {
$FullPath -eq "C:\ProgramData\containerd\certs.d\mcr.microsoft.com" -and
$DirectoryUsage -eq "storing containerd registry hosts config"
}
$script:capturedFilePath | Should -Be "C:\ProgramData\containerd\certs.d\mcr.microsoft.com\hosts.toml"
$script:capturedEncoding | Should -Be "ascii"
$script:capturedContent | Should -Match 'server = "https://mcr.microsoft.com"'
$script:capturedContent | Should -Match '\[host\."https://myacr.azurecr.io/v2"\]'
$script:capturedContent | Should -Match 'override_path = true'
}

It "Should sanitize bootstrap profile host and use custom mcr repository base" {
$global:MCR_REPOSITORY_BASE = "my.mcr.mirror"
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER = "https://myacr.azurecr.io/some/path/"

Set-BootstrapProfileRegistryContainerdHost

Assert-MockCalled -CommandName 'Create-Directory' -Exactly -Times 1 -ParameterFilter {
$FullPath -eq "C:\ProgramData\containerd\certs.d\my.mcr.mirror"
}
$script:capturedFilePath | Should -Be "C:\ProgramData\containerd\certs.d\my.mcr.mirror\hosts.toml"
$script:capturedContent | Should -Match 'server = "https://my.mcr.mirror"'
$script:capturedContent | Should -Match '\[host\."https://myacr.azurecr.io/v2/some/path"\]'
}

It "Should map host with repository prefix to v2 path" {
$global:MCR_REPOSITORY_BASE = "mcr.microsoft.com"
$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER = "myacr.azurecr.io/aaa"

Set-BootstrapProfileRegistryContainerdHost

$script:capturedContent | Should -Match '\[host\."https://myacr.azurecr.io/v2/aaa"\]'
}
}
Loading