Support building free-threaded CPython (#319)

* Support building free-threaded CPython Add support for Python's free threading build mode where the global interpreter lock is disabled. The packages are marked using a suffix on the architecture, like 'x64-freethreaded' or 'arm64-freethreaded'. * Match '-freethreaded' in arch * Use type 'string' instead of 'str' * On Linux, only delete Python installations with the same architecture. This matches the macOS behavior and allows users to install both the free-threading and default builds at the same time.
2025-04-03 22:09:38 +00:00 · 2025-02-20 09:42:21 -05:00
parent e550a75036
commit d456108e6b
14 changed files with 176 additions and 39 deletions
--- a/.github/workflows/build-python-packages.yml
+++ b/.github/workflows/build-python-packages.yml
@ -12,6 +12,11 @@ on:
        required: true
        type: boolean
        default: false
+      THREADING_BUILD_MODES:
+        description: 'CPython threading build modes'
+        required: true
+        type: string
+        default: 'default,freethreaded'
      PLATFORMS:
        description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
        required: true
@ -40,32 +45,42 @@ jobs:
        id: generate-matrix
        run: |
          [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64' }}".Split(",").Trim()
+          [String[]]$buildModes = "${{ inputs.threading_build_modes || 'default' }}".Split(",").Trim()
          $matrix = @()
          
          foreach ($configuration in $configurations) {
-            $parts = $configuration.Split("_")
-            $os = $parts[0]
-            $arch = if ($parts[1]) {$parts[1]} else {"x64"}
-            switch -wildcard ($os) {
-              "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
-              "*macos*" { $platform = 'darwin' }
-              "*windows*" { $platform = 'win32' }
-            }
-            
-            if ($configuration -eq "ubuntu-22.04_arm64") {
-              $os = "setup-actions-ubuntu-arm64-2-core"
-            }
-            elseif ($configuration -eq "ubuntu-24.04_arm64") {
-              $os = "setup-actions-ubuntu24-arm64-2-core"
-            }
-            elseif ($configuration -eq "windows-2019_arm64") {
-              $os = "setup-actions-windows-arm64-4-core"
-            }
-            
-            $matrix += @{
-              'platform' = $platform
-              'os' = $os
-              'arch' = $arch
+            foreach ($buildMode in $buildModes) {
+              $parts = $configuration.Split("_")
+              $os = $parts[0]
+              $arch = if ($parts[1]) {$parts[1]} else {"x64"}
+              switch -wildcard ($os) {
+                "*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
+                "*macos*" { $platform = 'darwin' }
+                "*windows*" { $platform = 'win32' }
+              }
+
+              if ($configuration -eq "ubuntu-22.04_arm64") {
+                $os = "setup-actions-ubuntu-arm64-2-core"
+              }
+              elseif ($configuration -eq "ubuntu-24.04_arm64") {
+                $os = "setup-actions-ubuntu24-arm64-2-core"
+              }
+              elseif ($configuration -eq "windows-2019_arm64") {
+                $os = "setup-actions-windows-arm64-4-core"
+              }
+
+              if ($buildMode -eq "freethreaded") {
+                if ([semver]"${{ inputs.VERSION }}" -lt [semver]"3.13.0") {
+                  continue;
+                }
+                $arch += "-freethreaded"
+              }
+
+              $matrix += @{
+                'platform' = $platform
+                'os' = $os
+                'arch' = $arch
+              }
            }
          }
          echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
@ -205,6 +220,9 @@ jobs:
          python-version: ${{ env.VERSION }}
          architecture: ${{ matrix.arch }}

+      - name: Python version
+        run: python -VVV
+
      - name: Verbose sysconfig dump
        if: runner.os == 'Linux' || runner.os == 'macOS'
        run: python ./sources/python-config-output.py
--- a/builders/macos-python-builder.psm1
+++ b/builders/macos-python-builder.psm1
@ -151,6 +151,37 @@ class macOSPythonBuilder : NixPythonBuilder {
        return $pkgLocation
    }

+    [string] GetFrameworkName() {
+        <#
+        .SYNOPSIS
+        Get the Python installation Package name.
+        #>
+
+        if ($this.IsFreeThreaded()) {
+            return "PythonT.framework"
+        } else {
+            return "Python.framework"
+        }
+    }
+
+    [string] GetPkgChoices() {
+        <#
+        .SYNOPSIS
+        Reads the configuration XML file for the Python installer
+        #>
+
+        $config = if ($this.IsFreeThreaded()) { "freethreaded" } else { "default" }
+        $choicesFile = Join-Path $PSScriptRoot "../config/macos-pkg-choices-$($config).xml"
+        $choicesTemplate = Get-Content -Path $choicesFile -Raw
+
+        $variablesToReplace = @{
+            "{{__VERSION_MAJOR_MINOR__}}" = "$($this.Version.Major).$($this.Version.Minor)";
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $choicesTemplate = $choicesTemplate.Replace($_, $variablesToReplace[$_]) }
+        return $choicesTemplate
+    }
+
    [void] CreateInstallationScriptPkg() {
        <#
        .SYNOPSIS
@ -165,6 +196,8 @@ class macOSPythonBuilder : NixPythonBuilder {
            "{{__VERSION_FULL__}}" = $this.Version;
            "{{__PKG_NAME__}}" = $this.GetPkgName();
            "{{__ARCH__}}" = $this.Architecture;
+            "{{__FRAMEWORK_NAME__}}" = $this.GetFrameworkName();
+            "{{__PKG_CHOICES__}}" = $this.GetPkgChoices();
        }

        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
--- a/builders/nix-python-builder.psm1
+++ b/builders/nix-python-builder.psm1
@ -115,7 +115,7 @@ class NixPythonBuilder : PythonBuilder {
        Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)"
        $buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File
        
-        Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue	
+        Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue
        Execute-Command -Command "make install" -ErrorAction Continue

        Write-Debug "Done; Make log location: $buildOutputLocation"
--- a/builders/python-builder.psm1
+++ b/builders/python-builder.psm1
@ -94,6 +94,24 @@ class PythonBuilder {
        return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)"
    }

+    [string] GetHardwareArchitecture() {
+        <#
+        .SYNOPSIS
+        The hardware architecture (x64, arm64) without any Python free threading suffix.
+        #>
+
+        return $this.Architecture.Replace("-freethreaded", "")
+    }
+
+    [bool] IsFreeThreaded() {
+        <#
+        .SYNOPSIS
+        Check if Python version is free threaded.
+        #>
+
+        return $this.Architecture.EndsWith("-freethreaded")
+    }
+
    [void] PreparePythonToolcacheLocation() {
        <#
        .SYNOPSIS
--- a/builders/ubuntu-python-builder.psm1
+++ b/builders/ubuntu-python-builder.psm1
@ -37,6 +37,14 @@ class UbuntuPythonBuilder : NixPythonBuilder {
        $configureString += " --enable-shared"
        $configureString += " --enable-optimizations"

+        if ($this.IsFreeThreaded()) {
+            if ($this.Version -lt "3.13.0") {
+                Write-Host "Python versions lower than 3.13.0 do not support free threading"
+                exit 1
+            }
+            $configureString += " --disable-gil"
+        }
+
        ### Compile with support of loadable sqlite extensions.
        ### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
        $configureString += " --enable-loadable-sqlite-extensions"
--- a/builders/win-python-builder.psm1
+++ b/builders/win-python-builder.psm1
@ -54,13 +54,13 @@ class WinPythonBuilder : PythonBuilder {
        #>

        $ArchitectureExtension = ""
-        if ($this.Architecture -eq "x64") {
+        if ($this.GetHardwareArchitecture() -eq "x64") {
            if ($this.Version -ge "3.5") {
                $ArchitectureExtension = "-amd64"
            } else {
                $ArchitectureExtension = ".amd64"
            }
-        }elseif ($this.Architecture -eq "arm64") {
+        } elseif ($this.GetHardwareArchitecture() -eq "arm64") {
                $ArchitectureExtension = "-arm64"
        }

@ -113,6 +113,7 @@ class WinPythonBuilder : PythonBuilder {

        $variablesToReplace = @{
            "{{__ARCHITECTURE__}}" = $this.Architecture;
+            "{{__HARDWARE_ARCHITECTURE__}}" = $this.GetHardwareArchitecture();
            "{{__VERSION__}}" = $this.Version;
            "{{__PYTHON_EXEC_NAME__}}" = $pythonExecName
        }
--- a/config/macos-pkg-choices-default.xml
+++ b/config/macos-pkg-choices-default.xml
@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<array>
+    <dict>
+   </dict>
+</array>
+</plist>
--- a/config/macos-pkg-choices-freethreaded.xml
+++ b/config/macos-pkg-choices-freethreaded.xml
@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<array>
+    <dict>
+        <key>attributeSetting</key>
+        <integer>1</integer>
+        <key>choiceAttribute</key>
+        <string>selected</string>
+        <key>choiceIdentifier</key>
+        <string>org.python.Python.PythonTFramework-{{__VERSION_MAJOR_MINOR__}}</string>
+    </dict>
+</array>
+</plist>
--- a/config/python-manifest-config.json
+++ b/config/python-manifest-config.json
@ -1,5 +1,5 @@
 {
-    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
+    "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+(-freethreaded)?)",
    "groups": {
        "arch": 4,
        "platform": 2,
--- a/installers/macos-pkg-setup-template.sh
+++ b/installers/macos-pkg-setup-template.sh
@ -2,6 +2,11 @@ set -e

 PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
 PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+PYTHON_FRAMEWORK_NAME="{{__FRAMEWORK_NAME__}}"
+PYTHON_PKG_CHOICES=$(cat << 'EOF'
+{{__PKG_CHOICES__}}
+EOF
+)
 ARCH="{{__ARCH__}}"
 MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
 MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
@ -20,7 +25,7 @@ fi
 PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
 PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
 PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
-PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
+PYTHON_FRAMEWORK_PATH="/Library/Frameworks/${PYTHON_FRAMEWORK_NAME}/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
 PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"

 echo "Check if Python hostedtoolcache folder exist..."
@ -38,8 +43,11 @@ else
    done
 fi

+PYTHON_PKG_CHOICES_FILES=$(mktemp)
+echo "$PYTHON_PKG_CHOICES" > $PYTHON_PKG_CHOICES_FILES
+
 echo "Install Python binaries from prebuilt package"
-sudo installer -pkg $PYTHON_PKG_NAME -target /
+sudo installer -pkg $PYTHON_PKG_NAME -applyChoiceChangesXML $PYTHON_PKG_CHOICES_FILES -target /

 echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
 echo "Create Python $PYTHON_FULL_VERSION folder"
@ -53,7 +61,9 @@ ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib

 echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
 ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
+chmod +x python

+# Note that bin is a symlink so referencing .. from bin will not work as expected
 cd bin/

 # This symlink already exists if Python version with the same major.minor version is installed,
@ -62,11 +72,15 @@ if [ ! -f $PYTHON_MAJOR_MINOR ]; then
    ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
 fi

+if [ ! -f $PYTHON_MAJOR ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR
+fi
+
 if [ ! -f python ]; then
    ln -s $PYTHON_MAJOR_DOT_MINOR python
 fi

-chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
+chmod +x $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python

 echo "Upgrading pip..."
 export PIP_ROOT_USER_ACTION=ignore
--- a/installers/nix-setup-template.sh
+++ b/installers/nix-setup-template.sh
@ -24,9 +24,9 @@ echo "Check if Python hostedtoolcache folder exist..."
 if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
    echo "Creating Python hostedtoolcache folder..."
    mkdir -p $PYTHON_TOOLCACHE_PATH
-elif [ -d $PYTHON_TOOLCACHE_VERSION_PATH ]; then
-    echo "Deleting Python $PYTHON_FULL_VERSION"
-    rm -rf $PYTHON_TOOLCACHE_VERSION_PATH
+elif [ -d $PYTHON_TOOLCACHE_VERSION_ARCH_PATH ]; then
+    echo "Deleting Python $PYTHON_FULL_VERSION ($ARCH)"
+    rm -rf $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
 fi

 echo "Create Python $PYTHON_FULL_VERSION folder"
--- a/installers/win-setup-template.ps1
+++ b/installers/win-setup-template.ps1
@ -1,4 +1,5 @@
 [String] $Architecture = "{{__ARCHITECTURE__}}"
+[String] $HardwareArchitecture = "{{__HARDWARE_ARCHITECTURE__}}"
 [String] $Version = "{{__VERSION__}}"
 [String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}"

@ -25,7 +26,7 @@ function Remove-RegistryEntries {
        [Parameter(Mandatory)][Int32] $MinorVersion
    )

-    $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
+    $versionFilter = Get-RegistryVersionFilter -Architecture $HardwareArchitecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion

    $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
    if (Test-Path -Path Registry::$regPath) {
@ -61,13 +62,15 @@ function Remove-RegistryEntries {
 function Get-ExecParams {
    param(
        [Parameter(Mandatory)][Boolean] $IsMSI,
+        [Parameter(Mandatory)][Boolean] $IsFreeThreaded,
        [Parameter(Mandatory)][String] $PythonArchPath
    )

    if ($IsMSI) {
        "TARGETDIR=$PythonArchPath ALLUSERS=1"
    } else {
-        "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1"
+        $Include_freethreaded = if ($IsFreeThreaded) { "Include_freethreaded=1" } else { "" }
+        "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1 $Include_freethreaded"
    }
 }

@ -81,6 +84,7 @@ $PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version
 $PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture

 $IsMSI = $PythonExecName -match "msi"
+$IsFreeThreaded = $Architecture -match "-freethreaded"

 $MajorVersion = $Version.Split('.')[0]
 $MinorVersion = $Version.Split('.')[1]
@ -120,13 +124,24 @@ Write-Host "Copy Python binaries to $PythonArchPath"
 Copy-Item -Path ./$PythonExecName -Destination $PythonArchPath | Out-Null

 Write-Host "Install Python $Version in $PythonToolcachePath..."
-$ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath
+$ExecParams = Get-ExecParams -IsMSI $IsMSI -IsFreeThreaded $IsFreeThreaded -PythonArchPath $PythonArchPath

 cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet"
 if ($LASTEXITCODE -ne 0) {
    Throw "Error happened during Python installation"
 }

+# print out all files in $PythonArchPath
+Write-Host "Files in $PythonArchPath"
+$files = Get-ChildItem -Path $PythonArchPath -File -Recurse
+Write-Output $files
+
+if ($IsFreeThreaded) {
+    # Delete python.exe and create a symlink to free-threaded exe
+    Remove-Item -Path "$PythonArchPath\python.exe" -Force
+    New-Item -Path "$PythonArchPath\python.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python${MajorVersion}.${MinorVersion}t.exe"
+}
+
 Write-Host "Create `python3` symlink"
 if ($MajorVersion -ne "2") {
    New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe"
--- a/tests/python-tests.ps1
+++ b/tests/python-tests.ps1
@ -7,6 +7,8 @@ param (
    $Architecture
 )

+$HardwareArchitecture = $Architecture -replace "-freethreaded", ""
+
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
 Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
 Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")
@ -58,7 +60,7 @@ Describe "Tests" {
    #     }
    # }

-    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
+    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($HardwareArchitecture -ne "arm64"))) {
        It "Check if sqlite3 module is installed" {
            "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
        }
--- a/tests/sources/python-config-test.py
+++ b/tests/sources/python-config-test.py
@ -8,25 +8,31 @@ os_type = platform.system()
 version = sys.argv[1]
 nativeVersion = sys.argv[2]
 architecture = sys.argv[3]
+hw_architecture = architecture.replace('-freethreaded', '')

 versions=version.split(".")
 version_major=int(versions[0])
 version_minor=int(versions[1])

-pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64"))
+pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (hw_architecture == "arm64"))

 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')

 is_shared = sysconfig.get_config_var('Py_ENABLE_SHARED')
 have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
+is_free_threaded = sysconfig.get_config_var('Py_GIL_DISABLED')

 ### Define expected variables
 if os_type == 'Linux': expected_ld_library_extension = 'so'
 if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
+if is_free_threaded:
+    framework_name = 'PythonT.framework'
+else:
+    framework_name = 'Python.framework'

 if pkg_installer:
-    expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
+    expected_lib_dir_path = f'/Library/Frameworks/{framework_name}/Versions/{version_major}.{version_minor}/lib'
 else:
    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/{architecture}/lib'