add WSL to CI build

Author: CyanoHao

.github/workflows/build.yml

@@ -34,6 +34,9 @@ jobs:
             '32_486-msvcrt_win98',
           ]
 
+          wsl_branch = ['16', '15']
+          wsl_profile = ['64_v2-mcf', '64_v2-ucrt']
+
           sat_group = [
             {
               'name': '64-nt6x',
@@ -101,6 +104,8 @@ jobs:
 
           print(f'branch={json.dumps(branch)}')
           print(f'profile={json.dumps(profile)}')
+          print(f'wsl_branch={json.dumps(wsl_branch)}')
+          print(f'wsl_profile={json.dumps(wsl_profile)}')
           print(f'release={json.dumps(release)}')
           print(f'prerelease={json.dumps(prerelease)}')
           print(f'release_xmake={json.dumps(release_xmake)}')
@@ -110,6 +115,8 @@ jobs:
     outputs:
       branch: ${{ steps.param.outputs.branch }}
       profile: ${{ steps.param.outputs.profile }}
+      wsl_branch: ${{ steps.param.outputs.wsl_branch }}
+      wsl_profile: ${{ steps.param.outputs.wsl_profile }}
       release: ${{ steps.param.outputs.release }}
       prerelease: ${{ steps.param.outputs.prerelease }}
       release_xmake: ${{ steps.param.outputs.release_xmake }}
@@ -168,6 +175,58 @@ jobs:
       branch: ${{ matrix.branch }}
       profile: ${{ matrix.profile }}
 
+  wsl_build_container:
+    name: Build container (WSL)
+    needs: [param]
+    if: ${{ ! fromJson(needs.param.outputs.release) }}
+    runs-on: windows-2025
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build container (WSL)
+        run: |
+          Copy-Item -Path support/wsl/.wslconfig -Destination "$env:USERPROFILE/.wslconfig"
+
+          Invoke-WebRequest `
+            -Uri https://cdimages.ubuntu.com/ubuntu-wsl/noble/daily-live/current/noble-wsl-amd64.wsl `
+            -OutFile noble-wsl-amd64.wsl
+
+          wsl --import mingw-lite-buildenv . noble-wsl-amd64.wsl
+
+          wsl --distribution mingw-lite-buildenv --exec `
+            apt update
+          wsl --distribution mingw-lite-buildenv --exec `
+            env DEBIAN_FRONTEND=noninteractive `
+              apt-get install --no-install-recommends -y `
+                autoconf automake build-essential cmake flex gettext libtool meson texinfo xmake `
+                7zip ca-certificates curl file gawk libarchive-tools python3 python3-packaging zstd
+
+          wsl --distribution mingw-lite-buildenv --exec `
+            sync
+          wsl --shutdown
+
+          mkdir assets
+          wsl --export mingw-lite-buildenv assets/buildenv.vhdx --format vhd
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wsl-buildenv
+          path: assets/buildenv.vhdx
+
+  wsl_build:
+    name: Build (WSL)
+    needs: [param, download_source, wsl_build_container]
+    if: ${{ ! fromJson(needs.param.outputs.release) }}
+    strategy:
+      fail-fast: false
+      matrix:
+        branch: ${{ fromJson(needs.param.outputs.wsl_branch) }}
+        profile: ${{ fromJson(needs.param.outputs.wsl_profile) }}
+    uses: ./.github/workflows/wsl_build_single.yml
+    with:
+      branch: ${{ matrix.branch }}
+      profile: ${{ matrix.profile }}
+
   release:
     name: Release
     needs: [param, build]

.github/workflows/build_single.yml

@@ -82,17 +82,26 @@ jobs:
 
       - uses: actions/checkout@v4
 
-      - uses: msys2/setup-msys2@v2
-        with:
-          msystem: ucrt64
-          pacboy: python:p python-packaging:p
-
       - uses: actions/download-artifact@v4
         with:
           name: mingw${{ inputs.profile }}-${{ inputs.branch }}
           path: dist
 
+      - name: Setup bsdtar
+        run: |
+          New-Item -ItemType Directory -Path assets -Force
+          Invoke-WebRequest `
+            -Uri https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-v1.5.7-win64.zip `
+            -OutFile assets/zstd-v1.5.7-win64.zip
+          Expand-Archive -Path assets/zstd-v1.5.7-win64.zip -DestinationPath assets
+          $system32 = [Environment]::SystemDirectory
+          Copy-Item -Path assets/zstd-v1.5.7-win64/zstd.exe -Destination "$system32/zstd.exe"
+          New-Item -ItemType SymbolicLink -Path "$system32/bsdtar.exe" -Target tar.exe
+
+      - name: Install Python Package
+        run: |
+          pip install packaging
+
       - name: Test
-        shell: msys2 {0}
         run: |
           python test.py --branch ${{ inputs.branch }} --profile ${{ inputs.profile }} --build x86_64-w64-mingw32 -v

.github/workflows/wsl_build_single.yml

@@ -0,0 +1,68 @@
+name: Build single (WSL)
+
+on:
+  workflow_call:
+    inputs:
+      profile:
+        required: true
+        type: string
+      branch:
+        required: true
+        type: string
+
+jobs:
+  build:
+    name: Build
+    runs-on: windows-2025
+
+    steps:
+      - name: Configure Git
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.symlinks true
+
+      - uses: actions/checkout@v4
+
+      - name: Download source
+        uses: actions/download-artifact@v4
+        with:
+          name: src-${{ inputs.branch }}
+          path: assets
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: wsl-buildenv
+          path: assets
+
+      - name: Setup bsdtar
+        run: |
+          New-Item -ItemType Directory -Path assets -Force
+          Invoke-WebRequest `
+            -Uri https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-v1.5.7-win64.zip `
+            -OutFile assets/zstd-v1.5.7-win64.zip
+          Expand-Archive -Path assets/zstd-v1.5.7-win64.zip -DestinationPath assets
+          $system32 = [Environment]::SystemDirectory
+          Copy-Item -Path assets/zstd-v1.5.7-win64/zstd.exe -Destination "$system32/zstd.exe"
+          New-Item -ItemType SymbolicLink -Path "$system32/bsdtar.exe" -Target tar.exe
+
+      - name: Install Python Package
+        run: |
+          pip install packaging
+
+      - name: Build
+        run: |
+          Copy-Item -Path support/wsl/.wslconfig -Destination "$env:USERPROFILE/.wslconfig"
+
+          wsl --import-in-place mingw-lite-buildenv assets/buildenv.vhdx
+          wsl --distribution mingw-lite-buildenv --exec `
+            env XMAKE_ROOT=y `
+              ./main.py --branch ${{ inputs.branch }} --profile ${{ inputs.profile }}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wsl${{ inputs.profile }}-${{ inputs.branch }}
+          path: dist
+
+      - name: Test
+        run: |
+          python test.py --branch ${{ inputs.branch }} --profile ${{ inputs.profile }} --build x86_64-w64-mingw32 -v

doc/wsl-buildenv.md

@@ -25,6 +25,10 @@ See [Microsoft Learn](https://learn.microsoft.com/en-us/windows/wsl/install).
      autoconf automake build-essential cmake flex gettext libtool meson texinfo xmake \
      7zip ca-certificates curl file gawk libarchive-tools python3 python3-packaging zstd
    ```
+3. Set environment variables:
+   ```bash
+   echo "export XMAKE_ROOT=y" >>~/.bashrc
+   ```
 
 ## Clone the Repository
 

module/ABB.py

@@ -10,7 +10,7 @@
 from module.debug import shell_here
 from module.path import ProjectPaths
 from module.profile import BranchProfile
-from module.util import XMAKE_ARCH_MAP, add_objects_to_static_lib, ensure, overlayfs_ro
+from module.util import XMAKE_ARCH_MAP, add_objects_to_static_lib, ensure, overlayfs_ro, remove_info_main_menu
 from module.util import cflags_B, configure, make_custom, make_default, make_destdir_install
 from module.util import meson_build, meson_config, meson_flags_B, meson_install
 from module.util import xmake_build, xmake_config, xmake_install
@@ -78,6 +78,8 @@ def _binutils(ver: BranchProfile, paths: ProjectPaths, config: argparse.Namespac
       'install',
     ], jobs = 1)
 
+  remove_info_main_menu(paths.layer_ABB.binutils)
+
   license_dir = paths.layer_ABB.binutils / 'share/licenses/binutils'
   ensure(license_dir)
   for file in ['COPYING', 'COPYING3', 'COPYING.LIB', 'COPYING3.LIB']:
@@ -368,6 +370,8 @@ def _gcc(ver: BranchProfile, paths: ProjectPaths, config: argparse.Namespace):
           atomic_objects.append(wrapper_obj)
       add_objects_to_static_lib(f'{ver.target}-ar', libgcc_a, atomic_objects)
 
+  remove_info_main_menu(paths.layer_ABB.gcc)
+
   license_dir = paths.layer_ABB.gcc / 'share/licenses/gcc'
   ensure(license_dir)
   for file in ['COPYING', 'COPYING3', 'COPYING.RUNTIME', 'COPYING.LIB', 'COPYING3.LIB']:
@@ -452,6 +456,8 @@ def _gdb(ver: BranchProfile, paths: ProjectPaths, config: argparse.Namespace):
       f.write('register_libstdcxx_printers(None)\n')
       f.write('end\n')
 
+  remove_info_main_menu(paths.layer_ABB.gdb)
+
   # collision with binutils
   for info_file in ['bfd.info', 'ctf-spec.info', 'sframe-spec.info']:
     os.unlink(paths.layer_ABB.gdb / 'share/info' / info_file)
@@ -497,6 +503,8 @@ def _gmake(ver: BranchProfile, paths: ProjectPaths, config: argparse.Namespace):
     make_default(build_dir, config.jobs)
     make_destdir_install(build_dir, paths.layer_ABB.make)
 
+  remove_info_main_menu(paths.layer_ABB.make)
+
   license_dir = paths.layer_ABB.make / 'share/licenses/make'
   ensure(license_dir)
   shutil.copy(paths.src_dir.make / 'COPYING', license_dir / 'COPYING')

module/util.py

@@ -5,7 +5,7 @@
 import shutil
 import subprocess
 from tempfile import TemporaryDirectory
-from typing import Iterable, List
+from typing import Iterable, List, Union
 
 from module.profile import ProfileInfo
 
@@ -174,7 +174,7 @@ def meson_install(
   )
 
 @contextmanager
-def overlayfs_ro(merged: Path | str, lower: list[Path]):
+def overlayfs_ro(merged: Union[Path, str], lower: list[Path]):
   try:
     if len(lower) == 1:
       subprocess.run([
@@ -197,8 +197,13 @@ def overlayfs_ro(merged: Path | str, lower: list[Path]):
   finally:
     subprocess.run(['umount', merged], check = False)
 
+def remove_info_main_menu(prefix: Path):
+  info_main_menu = prefix / 'share/info/dir'
+  if info_main_menu.exists():
+    info_main_menu.unlink()
+
 @contextmanager
-def temporary_rw_overlay(path: Path | str):
+def temporary_rw_overlay(path: Union[Path, str]):
   with TemporaryDirectory() as tmp:
     try:
       shutil.copytree(path, tmp, dirs_exist_ok = True)

support/wsl/.wslconfig

@@ -0,0 +1,22 @@
+############################################
+## WSL 2 configuration (GHA windows-2025) ##
+############################################
+
+[wsl2]
+
+# https://docs.github.com/en/actions/reference/runners/github-hosted-runners
+# memory: 12 GiB of 16 GiB
+memory=12GB
+
+swap=0
+
+guiApplications=false
+
+nestedVirtualization=false
+
+# storage: 12 GiB of 14 GiB
+defaultVhdSize=12GB
+
+[experimental]
+
+autoMemoryReclaim=gradual

test.py

@@ -39,6 +39,9 @@ def extract(path: Path, arx: Path):
     '-C', path.parent,
     '-xf', arx,
     '--no-same-owner',
+    # workaround zstd pipe error with Windows `tar.exe`
+    # https://github.com/libarchive/libarchive/issues/2512
+    '--ignore-zeros',
   ], check = True)
 
 def prepare_test_binary(ver: BranchProfile, paths: ProjectPaths):