feat: install waagent via wireserver

e2e/vmss.go

@@ -570,6 +570,7 @@ func extractLogsFromVMLinux(ctx context.Context, s *Scenario, vm *ScenarioVM) er
 		"aks-log-collector.log":            "sudo journalctl -u aks-log-collector",
 		"cluster-provision-cse-output.log": "sudo cat /var/log/azure/cluster-provision-cse-output.log",
 		"sysctl-out.log":                   "sudo sysctl -a",
+		"waagent.log":                      "sudo cat /var/log/waagent.log",
 		"aks-node-controller.log":          "sudo cat /var/log/azure/aks-node-controller.log",
 		"aks-node-controller-config.json":  "sudo cat /opt/azure/containers/aks-node-controller-config.json", // Only available in Scriptless.
 

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -296,6 +296,153 @@ downloadSecureTLSBootstrapClient() {
     echo "aks-secure-tls-bootstrap-client installed successfully"
 }
 
+# installWALinuxAgent queries the Azure wireserver to get the WALinuxAgent GAFamily
+# version and manifest, downloads the zip for that version, and installs it under
+# /var/lib/waagent/WALinuxAgent-<version>/.
+# GAFamily is the exact version the waagent daemon targets during auto-update.
+# Installing it during VHD build lets the daemon pick it up locally without
+# downloading from the network at provisioning time.
+installWALinuxAgent() {
+    local downloadDir=$1
+    local wireserverURL=$2
+
+    echo "Installing WALinuxAgent from wireserver GAFamily manifest..."
+
+    # Step 1: Get the goalstate to find the ExtensionsConfig URL
+    local goalstate
+    goalstate=$(retrycmd_if_failure 10 5 60 curl -sSf -H "x-ms-agent-name: WALinuxAgent" -H "x-ms-version: 2012-11-30" "${wireserverURL}/machine/?comp=goalstate") || {
+        echo "ERROR: Failed to fetch goalstate from wireserver"
+        return 1
+    }
+
+    # Step 2: Extract and decode the ExtensionsConfig URL
+    local extensions_config_url
+    extensions_config_url=$(echo "${goalstate}" | python3 -c "
+import sys, re
+content = sys.stdin.read()
+m = re.search(r'<ExtensionsConfig>([^<]+)</ExtensionsConfig>', content)
+if m:
+    print(m.group(1))
+else:
+    sys.exit(1)
+") || {
+        echo "ERROR: Failed to extract ExtensionsConfig URL from goalstate"
+        return 1
+    }
+    # URL-decode and fix XML-escaped ampersands
+    extensions_config_url=$(echo "${extensions_config_url}" | python3 -c "import sys, urllib.parse; print(urllib.parse.unquote(sys.stdin.read().strip()))" | sed 's/&amp;/\&/g')
+
+    # Step 3: Fetch the extensions config to find the GAFamily version and manifest URI
+    local extensions_config
+    extensions_config=$(retrycmd_if_failure 10 5 60 curl -sSf -H "x-ms-agent-name: WALinuxAgent" -H "x-ms-version: 2012-11-30" "${extensions_config_url}") || {
+        echo "ERROR: Failed to fetch extensions config"
+        return 1
+    }
+
+    # Step 4: Extract the GAFamily version and first manifest URI using python3 regex.
+    # We use python3 instead of grep because:
+    # - grep -A N is fragile when the number of <Uri> entries varies by region
+    # - grep -oP (PCRE \K) has portability issues across distros/configurations
+    # - The GAFamily block can span many lines; python3 re.DOTALL handles this cleanly
+    local ga_family_info
+    ga_family_info=$(echo "${extensions_config}" | python3 -c "
+import sys, re
+content = sys.stdin.read()
+# Extract version from the GAFamily block
+vm = re.search(r'<GAFamily>.*?<Version>([^<]+)</Version>', content, re.DOTALL)
+if not vm:
+    print('ERROR: No GAFamily version found', file=sys.stderr)
+    sys.exit(1)
+# Extract first URI from the GAFamily block
+um = re.search(r'<GAFamily>.*?<Uri>([^<]+)</Uri>', content, re.DOTALL)
+if not um:
+    print('ERROR: No GAFamily manifest URI found', file=sys.stderr)
+    sys.exit(1)
+print(vm.group(1))
+print(um.group(1))
+") || {
+        echo "ERROR: Failed to parse GAFamily from extensions config"
+        return 1
+    }
+
+    local version
+    version=$(echo "${ga_family_info}" | head -n 1)
+    echo "GAFamily version: ${version}"
+
+    local manifest_url
+    manifest_url=$(echo "${ga_family_info}" | tail -n 1)
+    # Fix XML-escaped ampersands in SAS query parameters (same issue as extensions config URL)
+    manifest_url=$(echo "${manifest_url}" | sed 's/&amp;/\&/g')
+
+    # Step 6: Fetch the manifest
+    # Use retrycmd_silent to avoid logging the full URL (contains SAS token).
+    local manifest
+    manifest=$(retrycmd_silent 10 5 60 curl -sSf "${manifest_url}") || {
+        echo "ERROR: Failed to fetch manifest from ${manifest_url%%\?*}"
+        return 1
+    }
+
+    # Step 7: Find the zip URL for the GAFamily version by parsing the manifest XML
+    local zip_url
+    zip_url=$(echo "${manifest}" | python3 -c "
+import sys
+import xml.etree.ElementTree as ET
+content = sys.stdin.read()
+root = ET.fromstring(content)
+target = '${version}'
+for plugin in root.findall('.//Plugin'):
+    ver = plugin.find('Version')
+    if ver is not None and ver.text == target:
+        uri = plugin.find('.//Uri')
+        if uri is not None:
+            print(uri.text)
+            sys.exit(0)
+sys.exit(1)
+") || {
+        echo "ERROR: Version ${version} not found in WALinuxAgent manifest"
+        return 1
+    }
+
+    if [ -z "${zip_url}" ]; then
+        echo "ERROR: No download URL found for WALinuxAgent version ${version}"
+        return 1
+    fi
+
+    # Log the URL without query parameters to avoid leaking SAS tokens in build logs
+    echo "Found WALinuxAgent ${version} zip at: ${zip_url%%\?*}"
+
+    # Step 8: Download the zip
+    # Use retrycmd_silent to avoid logging the full URL (contains SAS token).
+    local tmpDir
+    tmpDir=$(mktemp -d)
+    local zipFile="${tmpDir}/WALinuxAgent-${version}.zip"
+
+    retrycmd_silent 10 5 60 curl -sSf -o "${zipFile}" "${zip_url}" || {
+        echo "ERROR: Failed to download WALinuxAgent zip from ${zip_url%%\?*}"
+        rm -rf "${tmpDir}"
+        return 1
+    }
+
+    # Step 9: Install the agent zip under /var/lib/waagent/WALinuxAgent-<version>/
+    local installDir="/var/lib/waagent/WALinuxAgent-${version}"
+    mkdir -p "${installDir}"
+    # Use python3 zipfile module instead of unzip, which may not be installed on all build VMs
+    python3 -m zipfile -e "${zipFile}" "${installDir}" || {
+        echo "ERROR: Failed to extract WALinuxAgent zip to ${installDir}"
+        rm -rf "${tmpDir}"
+        return 1
+    }
+
+    echo "WALinuxAgent ${version} installed successfully to ${installDir}"
+
+    # Store the zip in the download directory for provenance tracking
+    mkdir -p "${downloadDir}"
+    cp "${zipFile}" "${downloadDir}/" 2>/dev/null || true
+
+    # Clean up temporary files
+    rm -rf "${tmpDir}"
+}
+
 evalPackageDownloadURL() {
     local url=${1:-}
     if [ -n "$url" ]; then

vhdbuilder/packer/cleanup-vhd.sh

@@ -13,3 +13,63 @@ chmod 644 /etc/machine-id
 rm -f /opt/azure/disk-usage.txt
 # Cleanup IMDS instance metadata cache file
 rm -f /opt/azure/containers/imds_instance_metadata_cache.json
+
+# Write post-deprovision WALinuxAgent install script.
+# The deprovision step (waagent -force -deprovision+user) clears /var/lib/waagent/,
+# so we install the latest agent AFTER deprovision. This script is called from the
+# packer inline block after the deprovision command completes.
+# Skip on Flatcar and AzureLinux OSGuard which use their OS-packaged version.
+OS_ID=$(. /etc/os-release 2>/dev/null && echo "${ID:-}" | tr '[:lower:]' '[:upper:]')
+OS_VARIANT_ID=$(. /etc/os-release 2>/dev/null && echo "${VARIANT_ID:-}" | tr '[:lower:]' '[:upper:]' | tr -d '"')
+if [ "$OS_ID" != "FLATCAR" ] && [ "$OS_VARIANT_ID" != "OSGUARD" ]; then
+    WALINUXAGENT_DOWNLOAD_DIR="/opt/walinuxagent/downloads"
+    WALINUXAGENT_WIRESERVER_URL="http://168.63.129.16:80"
+    cat > /opt/azure/containers/post-deprovision-walinuxagent.sh << WALINUXAGENT_SCRIPT
+#!/bin/bash -eu
+# Post-deprovision WALinuxAgent install script.
+# NOTE: -x is intentionally omitted to avoid leaking SAS tokens from
+# wireserver manifest/blob URLs in packer build logs.
+# Sources the provisioning helpers and installs the latest agent from wireserver,
+# then configures waagent.conf to use the pre-cached agent from disk.
+
+# DNS will be broken on AzLinux after deprovision because
+# 'waagent -deprovision' clears /etc/resolv.conf.
+# Temporarily restore Azure DNS for manifest download
+# and then remove before the VHD is finalized to keep the image clean.
+RESOLV_CONF_BAK=""
+if [ ! -s /etc/resolv.conf ] || ! grep -q nameserver /etc/resolv.conf; then
+    cp /etc/resolv.conf /etc/resolv.conf.bak 2>/dev/null || true
+    RESOLV_CONF_BAK="/etc/resolv.conf.bak"
+    echo "nameserver 168.63.129.16" > /etc/resolv.conf
+    echo "Temporarily set DNS to Azure DNS for manifest download"
+fi
+
+source /opt/azure/containers/provision_source.sh
+source /opt/azure/containers/provision_installs.sh
+
+# Install WALinuxAgent from wireserver GAFamily manifest
+installWALinuxAgent ${WALINUXAGENT_DOWNLOAD_DIR} ${WALINUXAGENT_WIRESERVER_URL}
+
+# Configure waagent.conf to pick up the pre-cached agent from disk:
+# - AutoUpdate.Enabled=y tells the daemon to look for newer agent versions on disk
+# - AutoUpdate.UpdateToLatestVersion=n prevents downloading updates from the network
+sed -i 's/AutoUpdate.Enabled=n/AutoUpdate.Enabled=y/g' /etc/waagent.conf
+if ! grep -q '^AutoUpdate.Enabled=' /etc/waagent.conf; then
+    echo 'AutoUpdate.Enabled=y' >> /etc/waagent.conf
+fi
+sed -i 's/AutoUpdate.UpdateToLatestVersion=y/AutoUpdate.UpdateToLatestVersion=n/g' /etc/waagent.conf
+if ! grep -q '^AutoUpdate.UpdateToLatestVersion=' /etc/waagent.conf; then
+    echo 'AutoUpdate.UpdateToLatestVersion=n' >> /etc/waagent.conf
+fi
+
+# Restore resolv.conf to its post-deprovision state so the VHD ships clean
+if [ -n "\${RESOLV_CONF_BAK}" ] && [ -f "\${RESOLV_CONF_BAK}" ]; then
+    mv "\${RESOLV_CONF_BAK}" /etc/resolv.conf
+    echo "Restored /etc/resolv.conf to post-deprovision state"
+fi
+
+echo "WALinuxAgent installed and waagent.conf configured post-deprovision"
+WALINUXAGENT_SCRIPT
+    chmod 755 /opt/azure/containers/post-deprovision-walinuxagent.sh
+    echo "Wrote post-deprovision WALinuxAgent install script"
+fi

vhdbuilder/packer/test/linux-vhd-content-test.sh

@@ -243,6 +243,7 @@ testPackagesInstalled() {
         testContainerNetworkingPluginsInstalled
         continue
         ;;
+
     esac
 
     resolve_packages_source_url
@@ -1502,6 +1503,63 @@ testBccTools () {
   return 0
 }
 
+# testWALinuxAgentInstalled verifies that the WALinuxAgent GAFamily version was
+# installed post-deprovision and that waagent.conf is configured to use it.
+# The test runs on a VM booted from the captured VHD image, so the post-deprovision
+# script has already executed and self-deleted. We verify its *results*:
+#   1. At least one WALinuxAgent-* directory exists under /var/lib/waagent/
+#   2. The directory contains the expected artifacts (bin/, HandlerManifest.json, manifest.xml)
+#   3. waagent.conf has AutoUpdate.Enabled=y and AutoUpdate.UpdateToLatestVersion=n
+testWALinuxAgentInstalled() {
+  local test="testWALinuxAgentInstalled"
+  echo "$test:Start"
+
+  # Check that at least one WALinuxAgent-* directory was installed
+  local -a dirs
+  mapfile -t dirs < <(find /var/lib/waagent -maxdepth 1 -type d -name "WALinuxAgent-*" 2>/dev/null | sort -V)
+  local dirCount=${#dirs[@]}
+  if [ "$dirCount" -lt 1 ]; then
+    err "$test" "Expected at least 1 WALinuxAgent directory under /var/lib/waagent/, found ${dirCount}"
+    return 1
+  fi
+  echo "$test: Found ${dirCount} WALinuxAgent directories: ${dirs[*]}"
+
+  # Validate the newest directory (highest version) has expected artifacts
+  local installDir="${dirs[-1]}"
+  echo "$test: Validating pre-cached agent directory ${installDir}"
+
+  local expectedFiles=("HandlerManifest.json" "manifest.xml")
+  for f in "${expectedFiles[@]}"; do
+    if [ ! -f "${installDir}/${f}" ]; then
+      err "$test" "Expected file ${f} not found in ${installDir}, contents: $(ls -al "${installDir}")"
+      return 1
+    fi
+    echo "$test: Found expected file ${installDir}/${f}"
+  done
+
+  if [ ! -d "${installDir}/bin" ]; then
+    err "$test" "bin/ directory not found in ${installDir}, contents: $(ls -al "${installDir}")"
+    return 1
+  fi
+  echo "$test: Found bin/ directory in ${installDir}"
+
+  # Verify waagent.conf has the expected AutoUpdate settings
+  if grep -q '^AutoUpdate.Enabled=y' /etc/waagent.conf; then
+    echo "$test: waagent.conf has AutoUpdate.Enabled=y"
+  else
+    err "$test" "waagent.conf missing AutoUpdate.Enabled=y"
+    return 1
+  fi
+  if grep -q '^AutoUpdate.UpdateToLatestVersion=n' /etc/waagent.conf; then
+    echo "$test: waagent.conf has AutoUpdate.UpdateToLatestVersion=n"
+  else
+    err "$test" "waagent.conf missing AutoUpdate.UpdateToLatestVersion=n"
+    return 1
+  fi
+
+  echo "$test:Finish"
+}
+
 testAKSNodeControllerBinary () {
   local test="testAKSNodeControllerBinary"
   local go_binary_path="/opt/azure/containers/aks-node-controller"
@@ -1908,6 +1966,11 @@ testBccTools $OS_SKU
 testVHDBuildLogsExist
 testCriticalTools
 testPackagesInstalled
+# WALinuxAgent is installed post-deprovision (not via components.json),
+# so test it separately. Skip on Flatcar and AzureLinuxOSGuard which use OS-packaged version.
+if [ "$OS_SKU" != "Flatcar" ] && [ "$OS_SKU" != "AzureLinuxOSGuard" ]; then
+  testWALinuxAgentInstalled
+fi
 testImagesPulled "$(cat $COMPONENTS_FILEPATH)"
 testImagesCompleted
 testPodSandboxImagePinned

vhdbuilder/packer/vhd-image-builder-arm64-gen2.json

@@ -786,7 +786,7 @@
       "inline": [
         "sudo /bin/bash -eux /home/packer/cis.sh",
         "sudo /bin/bash -eux /opt/azure/containers/cleanup-vhd.sh",
-        "sudo /usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync || exit 125"
+        "sudo /bin/bash -c '/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync && if [ -f /opt/azure/containers/post-deprovision-walinuxagent.sh ]; then /bin/bash -eu /opt/azure/containers/post-deprovision-walinuxagent.sh && rm -f /opt/azure/containers/post-deprovision-walinuxagent.sh; fi' || exit 125"
       ]
     }
   ],

vhdbuilder/packer/vhd-image-builder-base.json

@@ -784,7 +784,7 @@
       "inline": [
         "sudo /bin/bash -eux /home/packer/cis.sh",
         "sudo /bin/bash -eux /opt/azure/containers/cleanup-vhd.sh",
-        "sudo /usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync || exit 125"
+        "sudo /bin/bash -c '/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync && if [ -f /opt/azure/containers/post-deprovision-walinuxagent.sh ]; then /bin/bash -eu /opt/azure/containers/post-deprovision-walinuxagent.sh && rm -f /opt/azure/containers/post-deprovision-walinuxagent.sh; fi' || exit 125"
       ]
     }
   ],

vhdbuilder/packer/vhd-image-builder-cvm.json

@@ -788,7 +788,7 @@
       "inline": [
         "sudo /bin/bash -eux /home/packer/cis.sh",
         "sudo /bin/bash -eux /opt/azure/containers/cleanup-vhd.sh",
-        "sudo /usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync || exit 125"
+        "sudo /bin/bash -c '/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync && if [ -f /opt/azure/containers/post-deprovision-walinuxagent.sh ]; then /bin/bash -eu /opt/azure/containers/post-deprovision-walinuxagent.sh && rm -f /opt/azure/containers/post-deprovision-walinuxagent.sh; fi' || exit 125"
       ]
     }
   ],

vhdbuilder/packer/vhd-image-builder-mariner-arm64.json

@@ -750,7 +750,7 @@
       "inline": [
         "sudo /bin/bash -eux /home/packer/cis.sh",
         "sudo /bin/bash -eux /opt/azure/containers/cleanup-vhd.sh",
-        "sudo waagent -force -deprovision+user && export HISTSIZE=0 && sync || exit 125"
+        "sudo /bin/bash -c 'waagent -force -deprovision+user && export HISTSIZE=0 && sync && if [ -f /opt/azure/containers/post-deprovision-walinuxagent.sh ]; then /bin/bash -eu /opt/azure/containers/post-deprovision-walinuxagent.sh && rm -f /opt/azure/containers/post-deprovision-walinuxagent.sh; fi' || exit 125"
       ]
     }
   ],

vhdbuilder/packer/vhd-image-builder-mariner-cvm.json

@@ -751,7 +751,7 @@
       "inline": [
         "sudo /bin/bash -eux /home/packer/cis.sh",
         "sudo /bin/bash -eux /opt/azure/containers/cleanup-vhd.sh",
-        "sudo waagent -force -deprovision+user && export HISTSIZE=0 && sync || exit 125"
+        "sudo /bin/bash -c 'waagent -force -deprovision+user && export HISTSIZE=0 && sync && if [ -f /opt/azure/containers/post-deprovision-walinuxagent.sh ]; then /bin/bash -eu /opt/azure/containers/post-deprovision-walinuxagent.sh && rm -f /opt/azure/containers/post-deprovision-walinuxagent.sh; fi' || exit 125"
       ]
     }
   ],

vhdbuilder/packer/vhd-image-builder-mariner.json

@@ -752,7 +752,7 @@
       "inline": [
         "sudo /bin/bash -eux /home/packer/cis.sh",
         "sudo /bin/bash -eux /opt/azure/containers/cleanup-vhd.sh",
-        "sudo waagent -force -deprovision+user && export HISTSIZE=0 && sync || exit 125"
+        "sudo /bin/bash -c 'waagent -force -deprovision+user && export HISTSIZE=0 && sync && if [ -f /opt/azure/containers/post-deprovision-walinuxagent.sh ]; then /bin/bash -eu /opt/azure/containers/post-deprovision-walinuxagent.sh && rm -f /opt/azure/containers/post-deprovision-walinuxagent.sh; fi' || exit 125"
       ]
     }
   ],