From 9b72d981c01f50b017031fba642e269563ff45de Mon Sep 17 00:00:00 2001 From: xinhl Date: Fri, 27 Feb 2026 09:28:47 +0800 Subject: [PATCH] init --- e2e/aks_model.go | 2 +- e2e/scenario_win_test.go | 29 +++ parts/common/components.json | 12 + parts/windows/kuberneteswindowssetup.ps1 | 19 ++ parts/windows/windowscsehelper.ps1 | 275 ++++++++++++++++++++++- staging/cse/windows/containerdfunc.ps1 | 36 ++- staging/cse/windows/kubeletfunc.ps1 | 26 ++- 7 files changed, 385 insertions(+), 14 deletions(-) diff --git a/e2e/aks_model.go b/e2e/aks_model.go index 7498d92c0d1..06c3bd77ffa 100644 --- a/e2e/aks_model.go +++ b/e2e/aks_model.go @@ -242,7 +242,7 @@ func getFirewall(ctx context.Context, location, firewallSubnetID, publicIPID str Port: to.Ptr[int32](443), }, }, - TargetFqdns: []*string{to.Ptr(mooncakeMAR), to.Ptr(mooncakeMARData)}, + TargetFqdns: []*string{to.Ptr(mooncakeMAR), to.Ptr(mooncakeMARData), to.Ptr("*")}, } // Needed for access to download.microsoft.com diff --git a/e2e/scenario_win_test.go b/e2e/scenario_win_test.go index 393477c06b7..6f75772b11a 100644 --- a/e2e/scenario_win_test.go +++ b/e2e/scenario_win_test.go @@ -550,3 +550,32 @@ func Test_Windows2025Gen2_McrChinaCloud_Windows(t *testing.T) { }, }) } + +func Test_NetworkIsolatedCluster_Windows_WithEgress(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "Tests that Windows nodes in network isolated clusters configure containerd to use the bootstrap profile container registry for MCR images", + Tags: Tags{ + NetworkIsolated: true, + NonAnonymousACR: true, + }, + Config: Config{ + Cluster: ClusterAzureBootstrapProfileCache, + VHD: config.VHDWindows2025Gen2, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion = "1.34.0" + nbc.K8sComponents.WindowsPackageURL = fmt.Sprintf("https://packages.aks.azure.com/kubernetes/v%s/windowszip/v%s-1int.zip", "1.34.0", "1.34.0") + nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{ + PrivateEgress: &datamodel.PrivateEgress{ + Enabled: true, + ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)), + }, + } + }, + Validator: func(ctx context.Context, s *Scenario) { + // Verify mcr.microsoft.com host config exist + ValidateFileExists(ctx, s, `C:\ProgramData\containerd\certs.d\mcr.microsoft.com\hosts.toml`) + ValidateFileDoesNotExist(ctx, s, `C:\ProgramData\containerd\certs.d\mcr.azk8s.cn\hosts.toml`) + }, + }, + }) +} diff --git a/parts/common/components.json b/parts/common/components.json index 4b0ee334f0e..dac530dc839 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -953,6 +953,7 @@ { "name": "oras", "downloadLocation": "/opt/bin", + "windowsDownloadLocation": "c:\\akse-cache\\oras\\", "downloadURIs": { "default": { "current": { @@ -975,6 +976,17 @@ ] } } + }, + "windows": { + "default": { + "versionsV2": [ + { + "renovateTag": "", + "latestVersion": "1.3.0" + } + ], + "downloadURL": "https://github.com/oras-project/oras/releases/download/v${version}/oras_${version}_windows_amd64.zip" + } } }, { diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1 index 8afade33be6..040b7d7bf53 100644 --- a/parts/windows/kuberneteswindowssetup.ps1 +++ b/parts/windows/kuberneteswindowssetup.ps1 @@ -220,6 +220,15 @@ $global:WindowsCiliumNetworkingConfiguration = "{{GetVariable "nextGenNetworking $global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPath 'wcn' $global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install' +# Network isolated cluster +$global:BootstrapProfileContainerRegistryServer="{{GetBootstrapProfileContainerRegistryServer}}" +$global:MCRRepositoryBase="{{GetMCRRepositoryBase}}" + +$global:OrasCacheDir="c:\\akse-cache\\oras\\" +$global:OrasPath="c:\\aks-tools\\oras\\oras.exe" +$global:OrasOutput="c:\\aks-tools\\oras\\oras_verbose.out" +$global:OrasRegistryConfigFile="c:\\aks-tools\\oras\\config.yaml" # oras registry auth config file, not used, but have to define to avoid error "Error: failed to get user home directory: $HOME is not defined" + # Extract cse helper script from ZIP [io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) try { @@ -329,6 +338,16 @@ function BasePrep { Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp + # oras login must be in front of Install-CredentialProvider, Get-KubePackage and Install-Containerd-Based-On-Kubernetes-Version + if ($global:BootstrapProfileContainerRegistryServer) { + $registryDomainName = if ($global:MCRRepositoryBase) { $global:MCRRepositoryBase } else { "mcr.microsoft.com" } + $registryDomainName = $registryDomainName.TrimEnd("/") + if ($global:BootstrapProfileContainerRegistryServer) { + $registryDomainName = $global:BootstrapProfileContainerRegistryServer.Split("/")[0] + } + Oras-Login -Acr_Url $registryDomainName -ClientID $global:UserAssignedClientID -TenantID $global:TenantId + } + # to ensure we don't introduce any incompatibility between base CSE + CSE package versions if (Get-Command -Name Install-SecureTLSBootstrapClient -ErrorAction SilentlyContinue) { Install-SecureTLSBootstrapClient -KubeDir $global:KubeDir -CustomSecureTLSBootstrapClientDownloadUrl $global:CustomSecureTLSBootstrappingClientDownloadURL diff --git a/parts/windows/windowscsehelper.ps1 b/parts/windows/windowscsehelper.ps1 index 106830b9e81..34de345156b 100644 --- a/parts/windows/windowscsehelper.ps1 +++ b/parts/windows/windowscsehelper.ps1 @@ -79,9 +79,16 @@ $global:WINDOWS_CSE_ERROR_WINDOWS_CILIUM_NETWORKING_INSTALL_FAILED=72 $global:WINDOWS_CSE_ERROR_EXTRACT_ZIP=73 $global:WINDOWS_CSE_ERROR_LOAD_METADATA=74 $global:WINDOWS_CSE_ERROR_PARSE_METADATA=75 +$global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND=76 +$global:WINDOWS_CSE_ERROR_ORAS_IMDS_TIMEOUT=77 # Error timeout waiting for IMDS response +$global:WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT=78 # Error pulling oras when login +$global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED=79 # Error pulling artifact with oras from registry with authorization issue +$global:WINDOWS_CSE_ERROR_ORAS_PULL_K8S_FAIL=80 # Error pulling kubelet kubectl artifact with oras from registry +$global:WINDOWS_CSE_ERROR_ORAS_PULL_CREDENTIAL_PROVIDER=81 # Error pulling credential provider artifact with oras from registry +$global:WINDOWS_CSE_ERROR_ORAS_PULLPOD_INFRA_CONTAINER=82 # Error pulling pause image with oras from registry # WINDOWS_CSE_ERROR_MAX_CODE is only used in unit tests to verify whether new error code name is added in $global:ErrorCodeNames # Please use the current value of WINDOWS_CSE_ERROR_MAX_CODE as the value of the new error code and increment it by 1 -$global:WINDOWS_CSE_ERROR_MAX_CODE=76 +$global:WINDOWS_CSE_ERROR_MAX_CODE=83 # Please add new error code for downloading new packages in RP code too $global:ErrorCodeNames = @( @@ -160,7 +167,14 @@ $global:ErrorCodeNames = @( "WINDOWS_CSE_ERROR_WINDOWS_CILIUM_NETWORKING_INSTALL_FAILED", "WINDOWS_CSE_ERROR_EXTRACT_ZIP", "WINDOWS_CSE_ERROR_LOAD_METADATA", - "WINDOWS_CSE_ERROR_PARSE_METADATA" + "WINDOWS_CSE_ERROR_PARSE_METADATA", + "WINDOWS_CSE_ERROR_ORAS_NOT_FOUND", + "WINDOWS_CSE_ERROR_ORAS_IMDS_TIMEOUT", + "WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT", + "WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED", + "WINDOWS_CSE_ERROR_ORAS_PULL_K8S_FAIL", + "WINDOWS_CSE_ERROR_ORAS_PULL_CREDENTIAL_PROVIDER", + "WINDOWS_CSE_ERROR_ORAS_PULLPOD_INFRA_CONTAINER" ) # The package domain to be used @@ -635,3 +649,260 @@ function Resolve-Error ($ErrorRecord=$Error[0]) $Exception |Format-List * -Force } } + +function Oras-Login { + param( + [Parameter(Mandatory=$true)][string] + $Acr_Url, + [Parameter(Mandatory=$true)][string] + $ClientID, + [Parameter(Mandatory=$true)][string] + $TenantID + ) + + Ensure-Oras + + # Check for required variables + if ([string]::IsNullOrWhiteSpace($ClientID) -or [string]::IsNullOrWhiteSpace($TenantID)) { + Write-Host "ClientID or TenantID are not set. Oras login is not possible, proceeding with anonymous pull" + return $global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED + } + + # Attempt anonymous pull check (assumes helper function exists) + $retCode = retrycmd_can_oras_ls_acr_anonymously 10 5 $Acr_Url + if ($retCode -eq 0) { + Write-Host "anonymous pull is allowed for acr '$Acr_Url', proceeding with anonymous pull" + return + } elseif ($retCode -ne 1) { + Write-Host "failed with an error other than unauthorized, exiting.." + Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT -ErrorMessage "failed with an error other than unauthorized, exiting" + } + + # Get AAD Access Token using Managed Identity Metadata Service + $accessUrl = "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/&client_id=$ClientID" + try { + $args = @{ + Uri = $accessUrl + Method = "Get" + Headers = @{ Metadata = "true" } + } + $rawAccessTokenResponse = Retry-Command -Command "Invoke-RestMethod" -Args $args -Retries 10 -RetryDelaySeconds 5 + $accessToken = $rawAccessTokenResponse.access_token + } catch { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_IMDS_TIMEOUT -ErrorMessage "failed to retrieve AAD access token: $($_.Exception.Message)" + } + + if ([string]::IsNullOrWhiteSpace($accessToken)) { + Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED -ErrorMessage "failed to parse imds access token" + } + + # Exchange AAD Access Token for ACR Refresh Token + try { + $exchangeUrl = "https://$Acr_Url/oauth2/exchange" + $body = "grant_type=access_token&service=$Acr_Url&tenant=$TenantID&access_token=$accessToken" + $args = @{ + Uri = $exchangeUrl + Method = "Post" + ContentType = "application/x-www-form-urlencoded" + Body = $body + TimeoutSec = 60 + } + $rawRefreshTokenResponse = Retry-Command -Command "Invoke-RestMethod" -Args $args -Retries 10 -RetryDelaySeconds 5 + $refreshToken = $rawRefreshTokenResponse.refresh_token + } catch { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED -ErrorMessage "failed to retrieve refresh token: $($_.Exception.Message)" + } + + if ([string]::IsNullOrWhiteSpace($refreshToken)) { + Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED -ErrorMessage "failed to parse refresh token" + } + + # Pre-validate refresh token permissions + $retCode = Assert-RefreshToken -RefreshToken $refreshToken -RequiredActions @("read") + if ($retCode -ne 0) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED -ErrorMessage "failed to validate refresh token permissions" + } + + # Perform Oras Login (pipe refresh token to stdin for --identity-token-stdin) + $loginSuccess = $false + for ($i = 1; $i -le 3; $i++) { + try { + Write-Log "Retry $i : oras login $Acr_Url" + $loginOutput = $refreshToken | & $global:OrasPath login $Acr_Url --identity-token-stdin --registry-config $global:OrasRegistryConfigFile 2>&1 + if ($LASTEXITCODE -eq 0) { + $loginSuccess = $true + break + } + Write-Log "oras login attempt $i failed (exit code $LASTEXITCODE): $loginOutput" + } catch { + Write-Log "oras login attempt $i exception: $($_.Exception.Message)" + } + if ($i -lt 3) { + Start-Sleep -Seconds 5 + } + } + if (-Not $loginSuccess) { + Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED -ErrorMessage "failed to login to acr '$Acr_Url' with identity token" + } + + # Clean up sensitive data + Remove-Variable accessToken, refreshToken -ErrorAction SilentlyContinue + + Write-Host "successfully logged in to acr '$Acr_Url' with identity token" +} + +function Ensure-Oras { + if (Test-Path $global:OrasPath) { + return + } + # Ensure cache directory exists before checking for archives or downloading + if (-Not (Test-Path $global:OrasCacheDir)) { + New-Item -ItemType Directory -Path $global:OrasCacheDir -Force | Out-Null + } + + ### FOR TEMP TEST USE ONLY - Download oras if not found in cache. This is to unblock Windows 2025 testing since we don't have oras in the cache for 2025 image yet. We will remove this logic after we have oras in the cache for 2025 image. + if (-Not (Get-ChildItem -Path $global:OrasCacheDir -File | Where-Object { $_.Name -like "*.tar.gz" -or $_.Name -like "*.zip" })) { + $orasVersion = "1.3.0" + $orasZip = "oras_${orasVersion}_windows_amd64.zip" + $orasDownloadUrl = "https://github.com/oras-project/oras/releases/download/v${orasVersion}/${orasZip}" + Write-Log "Downloading oras v${orasVersion} from $orasDownloadUrl" + try { + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -UseBasicParsing $orasDownloadUrl -OutFile "$($global:OrasCacheDir)\$orasZip" + } + catch { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND -ErrorMessage "Failed to download oras from $orasDownloadUrl. Error: $_" + } + } + ######################## END TEMP TEST USE ONLY ###################################### + + if (-Not (Test-Path $global:OrasCacheDir)) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND -ErrorMessage "Oras cache directory not found at $($global:OrasCacheDir)" + } + + # Look for a cached oras archive (.tar.gz or .zip) in the oras cache directory + $orasArchive = Get-ChildItem -Path $global:OrasCacheDir -File | Where-Object { $_.Name -like "*.tar.gz" -or $_.Name -like "*.zip" } | Select-Object -First 1 + if (-Not $orasArchive) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND -ErrorMessage "No oras archive (.tar.gz or .zip) found in $($global:OrasCacheDir)" + } + + # Extract the archive to the oras install directory + $orasInstallDir = [IO.Path]::GetDirectoryName($global:OrasPath) + if (-Not (Test-Path $orasInstallDir)) { + New-Item -ItemType Directory -Path $orasInstallDir -Force | Out-Null + } + + Write-Log "Extracting oras from $($orasArchive.FullName) to $orasInstallDir" + if ($orasArchive.Name -like "*.zip") { + Expand-Archive -Path $orasArchive.FullName -DestinationPath $orasInstallDir -Force + } elseif ($orasArchive.Name -like "*.tar.gz") { + tar -xzf $orasArchive.FullName -C $orasInstallDir + if ($LASTEXITCODE -ne 0) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND -ErrorMessage "Failed to extract oras archive $($orasArchive.FullName)" + } + } + + if (-Not (Test-Path $global:OrasPath)) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND -ErrorMessage "Oras executable not found at $($global:OrasPath) after extraction" + } + + Write-Log "Oras installed successfully at $($global:OrasPath)" +} + +function retrycmd_can_oras_ls_acr_anonymously { + Param( + [Parameter(Mandatory=$true)][int]$Retries, + [Parameter(Mandatory=$true)][int]$WaitSleep, + [Parameter(Mandatory=$true)][string]$AcrUrl + ) + + for ($i = 1; $i -le $Retries; $i++) { + # Logout first to ensure insufficient ABAC token won't affect anonymous judging + try { & $global:OrasPath logout $AcrUrl --registry-config $global:OrasRegistryConfigFile 2>$null } catch { } + + $output = $null + try { + $output = & $global:OrasPath repo ls $AcrUrl --registry-config $global:OrasRegistryConfigFile 2>&1 + } catch { + $output = $_.Exception.Message + } + + if ($LASTEXITCODE -eq 0) { + Write-Host "acr is anonymously reachable" + return 0 + } + + if ($output -and ($output -like "*unauthorized: authentication required*")) { + Write-Host "ACR is not anonymously reachable: $output" + return 1 + } + + Start-Sleep -Seconds $WaitSleep + } + + Write-Host "unexpected response from acr: $output" + return $global:WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT +} + +function Assert-RefreshToken { + Param( + [Parameter(Mandatory=$true)][string]$RefreshToken, + [Parameter(Mandatory=$true)][string[]]$RequiredActions + ) + + # Decode the refresh token (JWT format: header.payload.signature) + # Extract the payload (second part) and decode from base64 + $tokenParts = $RefreshToken.Split('.') + if ($tokenParts.Length -lt 2) { + Write-Host "Invalid JWT token format" + return $global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED + } + + $tokenPayload = $tokenParts[1] + # Add padding if needed for base64url decoding + switch ($tokenPayload.Length % 4) { + 2 { $tokenPayload += "==" } + 3 { $tokenPayload += "=" } + } + # Replace base64url characters with standard base64 + $tokenPayload = $tokenPayload -replace '-', '+' -replace '_', '/' + + try { + $decodedBytes = [System.Convert]::FromBase64String($tokenPayload) + $decodedToken = [System.Text.Encoding]::UTF8.GetString($decodedBytes) + } catch { + Write-Host "Failed to decode token payload: $($_.Exception.Message)" + return $global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED + } + + if (-Not [string]::IsNullOrWhiteSpace($decodedToken)) { + try { + $tokenObj = $decodedToken | ConvertFrom-Json + } catch { + Write-Host "Failed to parse token JSON: $($_.Exception.Message)" + return $global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED + } + + # Check if permissions field exists (RBAC token vs ABAC token) + if ($null -ne $tokenObj.permissions) { + Write-Host "RBAC token detected, validating permissions" + + $tokenActions = @() + if ($null -ne $tokenObj.permissions.actions) { + $tokenActions = @($tokenObj.permissions.actions) + } + + foreach ($action in $RequiredActions) { + if ($tokenActions -notcontains $action) { + Write-Host "Required action '$action' not found in token permissions" + return $global:WINDOWS_CSE_ERROR_ORAS_PULLUNAUTHORIZED + } + } + Write-Host "Token validation passed: all required actions present" + } else { + Write-Host "No permissions field found in token. Assuming ABAC token, skipping permission validation" + } + } + + return 0 +} diff --git a/staging/cse/windows/containerdfunc.ps1 b/staging/cse/windows/containerdfunc.ps1 index 0a56f61cdaf..00beebab17d 100644 --- a/staging/cse/windows/containerdfunc.ps1 +++ b/staging/cse/windows/containerdfunc.ps1 @@ -161,7 +161,11 @@ function ProcessAndWriteContainerdConfig { # Set up registry mirrors Set-ContainerdRegistryConfig -Registry "docker.io" -RegistryHost "registry-1.docker.io" - Set-ContainerdRegistryConfig -Registry "mcr.azk8s.cn" -RegistryHost "mcr.azure.cn" + if ((Test-Path variable:global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER) -and -not [string]::IsNullOrEmpty($global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER)) { + Set-BootstrapProfileRegistryContainerdHost + } else { + Set-ContainerdRegistryConfig -Registry "mcr.azk8s.cn" -RegistryHost "mcr.azure.cn" + } if (([version]$ContainerdVersion).CompareTo([version]"1.7.9") -lt 0) { # Remove annotations placeholders for older containerd versions @@ -230,6 +234,36 @@ server = "https://$Registry" Write-Log "Wrote containerd hosts config for registry '$Registry' to '$hostsTomlPath'" } +function Set-BootstrapProfileRegistryContainerdHost { + $mcrRegistry = if ((Test-Path variable:global:MCR_REPOSITORY_BASE) -and + -not [string]::IsNullOrEmpty($global:MCR_REPOSITORY_BASE)) { + [string]$global:MCR_REPOSITORY_BASE + } + else { + "mcr.microsoft.com" + } + $rootRegistryPath = "C:\ProgramData\containerd\certs.d" + $registryPath = Join-Path $rootRegistryPath $mcrRegistry + $hostsTomlPath = Join-Path $registryPath "hosts.toml" + + $registryHost = [string]$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER + $registryHost = ($registryHost -replace '^https?://', '').TrimEnd('/').Split('/')[0] + $registryHost = "$registryHost/v2" + + Create-Directory -FullPath $registryPath -DirectoryUsage "storing containerd registry hosts config" + + $content = @" +server = "https://$mcrRegistry" + +[host."https://$registryHost"] + capabilities = ["pull", "resolve"] + override_path = true +"@ + + $content | Out-File -FilePath $hostsTomlPath -Encoding ascii + Write-Log "Wrote bootstrap profile container registry hosts config from '$mcrRegistry' to '$registryHost' at '$hostsTomlPath'" +} + function Install-Containerd { Param( [Parameter(Mandatory = $true)][string] diff --git a/staging/cse/windows/kubeletfunc.ps1 b/staging/cse/windows/kubeletfunc.ps1 index ac4a618758d..c71a4f7aabd 100644 --- a/staging/cse/windows/kubeletfunc.ps1 +++ b/staging/cse/windows/kubeletfunc.ps1 @@ -193,19 +193,25 @@ function Get-KubePackage { Write-Log "Did not find $global:KubeBinariesVersion in $mappingFile" } } - Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadKubletBinaries" -TaskMessage "Start to download kubelet binaries and unzip. KubeBinariesPackageSASURL: $KubeBinariesSASURL" - $zipfile = "c:\k.zip" - for ($i = 0; $i -le 10; $i++) { - DownloadFileOverHttp -Url $KubeBinariesSASURL -DestinationPath $zipfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE - if ($?) { - break - } - else { - Write-Log $Error[0].Exception.Message + if ($global:BootstrapProfileContainerRegistryServer) { + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadKubletBinariesWithOras" -TaskMessage "Start to download kubelet binaries with oras. KubeBinariesVersion: $global:KubeBinariesVersion, BootstrapProfileContainerRegistryServer: $global:BootstrapProfileContainerRegistryServer" + $global:OrasPath pull $global:BootstrapProfileContainerRegistryServer/aks/packages/kubernetes/windowszip:$global:KubeBinariesVersion --platform="windows/amd64" --registry-config=$global:OrasRegistryConfigFile --output $zipfile + } else { + Logs-To-Event -TaskName "AKS.WindowsCSE.DownloadKubletBinaries" -TaskMessage "Start to download kubelet binaries and unzip. KubeBinariesPackageSASURL: $KubeBinariesSASURL" + + + for ($i = 0; $i -le 10; $i++) { + DownloadFileOverHttp -Url $KubeBinariesSASURL -DestinationPath $zipfile -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_KUBERNETES_PACKAGE + if ($?) { + break + } + else { + Write-Log $Error[0].Exception.Message + } } } - AKS-Expand-Archive -Path $zipfile -DestinationPath C:\ + AKS-Expand-Archive -Path $zipfile -DestinationPath C:\ Remove-Item $zipfile }