From fb2e761b1b40c2eab1902caf9f48c77de5ca1c03 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 11 Dec 2022 16:35:04 +0100 Subject: [PATCH] feat: build macOS arm64 packages --- .github/workflows/build-python-packages.yml | 5 +++-- builders/macos-python-builder.psm1 | 5 +++-- installers/macos-pkg-setup-template.sh | 15 ++++++++------- tests/python-tests.ps1 | 8 +++++--- tests/sources/python-config-test.py | 9 +++++---- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-python-packages.yml b/.github/workflows/build-python-packages.yml index bce7349..6b6ef3a 100644 --- a/.github/workflows/build-python-packages.yml +++ b/.github/workflows/build-python-packages.yml @@ -15,7 +15,7 @@ on: PLATFORMS: description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)' required: true - default: 'ubuntu-20.04,ubuntu-22.04,macos-11,windows-2019_x64,windows-2019_x86' + default: 'ubuntu-20.04,ubuntu-22.04,macos-11_x64,macos-11_arm64,windows-2019_x64,windows-2019_x86' pull_request: paths-ignore: - 'versions-manifest.json' @@ -39,7 +39,7 @@ jobs: - name: Generate execution matrix id: generate-matrix run: | - [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,macos-11,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim() + [String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,macos-11,macos-11_arm64,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim() $matrix = @() foreach ($configuration in $configurations) { @@ -155,6 +155,7 @@ jobs: $pesterContainer = New-PesterContainer -Path './python-tests.ps1' -Data @{ Version="${{ env.VERSION }}"; Platform="${{ matrix.platform }}"; + Architecture="${{ matrix.arch }}"; } $Result = Invoke-Pester -Container $pesterContainer -PassThru if ($Result.FailedCount -gt 0) { diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 089f2e7..cdefd9f 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -7,7 +7,7 @@ class macOSPythonBuilder : NixPythonBuilder { .DESCRIPTION Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder. - + While python.org provides precompiled binaries for macOS, switching to them risks breaking existing customers. If we wanted to start using the official binaries instead of building from source, we should avoid changing previous versions so we remain backwards compatible. @@ -151,6 +151,7 @@ class macOSPythonBuilder : NixPythonBuilder { $variablesToReplace = @{ "{{__VERSION_FULL__}}" = $this.Version; "{{__PKG_NAME__}}" = $this.GetPkgName(); + "{{__ARCH__}}" = $this.Architecture; } $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) } @@ -166,7 +167,7 @@ class macOSPythonBuilder : NixPythonBuilder { $PkgVersion = [semver]"3.11.0-beta.1" - if ($this.Version -ge $PkgVersion) { + if (($this.Version -ge $PkgVersion) -or ($this.Architecture -eq "arm64")) { Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..." $this.DownloadPkg() diff --git a/installers/macos-pkg-setup-template.sh b/installers/macos-pkg-setup-template.sh index 8206a3f..9fa7b6d 100644 --- a/installers/macos-pkg-setup-template.sh +++ b/installers/macos-pkg-setup-template.sh @@ -2,6 +2,7 @@ set -e PYTHON_FULL_VERSION="{{__VERSION_FULL__}}" PYTHON_PKG_NAME="{{__PKG_NAME__}}" +ARCH="{{__ARCH__}}" MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1) MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2) @@ -18,7 +19,7 @@ fi PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION -PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64 +PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}" PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}" @@ -29,10 +30,10 @@ if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then else # remove ALL other directories for same major.minor python versions find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do - python_version_x64="$python_version/x64" - if [ -e "$python_version_x64" ];then - echo "Deleting Python $python_version_x64" - rm -rf "$python_version_x64" + python_version_arch="$python_version/$ARCH" + if [ -e "$python_version_arch" ];then + echo "Deleting Python $python_version_arch" + rm -rf "$python_version_arch" fi done fi @@ -55,7 +56,7 @@ ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python cd bin/ -# This symlink already exists if Python version with the same major.minor version is installed, +# This symlink already exists if Python version with the same major.minor version is installed, # since we do not remove the framework folder if [ ! -f $PYTHON_MAJOR_MINOR ]; then ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR @@ -75,4 +76,4 @@ echo "Install OpenSSL certificates" sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command" echo "Create complete file" -touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete +touch $PYTHON_TOOLCACHE_VERSION_PATH/${ARCH}.complete diff --git a/tests/python-tests.ps1 b/tests/python-tests.ps1 index 8ef4eb2..dac2235 100644 --- a/tests/python-tests.ps1 +++ b/tests/python-tests.ps1 @@ -2,7 +2,9 @@ param ( [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] $Version, [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] - $Platform + $Platform, + [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] + $Architecture ) Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1") @@ -56,7 +58,7 @@ Describe "Tests" { # } # } - if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0")) { + if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) { It "Check if sqlite3 module is installed" { "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode } @@ -80,7 +82,7 @@ Describe "Tests" { It "Check if python configuration is correct" { $nativeVersion = Convert-Version -version $Version - "python ./sources/python-config-test.py $Version $nativeVersion" | Should -ReturnZeroExitCode + "python ./sources/python-config-test.py $Version $nativeVersion $Architecture" | Should -ReturnZeroExitCode } It "Check if shared libraries are linked correctly" { diff --git a/tests/sources/python-config-test.py b/tests/sources/python-config-test.py index de8a61d..55b78f4 100644 --- a/tests/sources/python-config-test.py +++ b/tests/sources/python-config-test.py @@ -9,12 +9,13 @@ import os os_type = platform.system() version = sys.argv[1] nativeVersion = sys.argv[2] +architecture = sys.argv[3] versions=version.split(".") version_major=int(versions[0]) version_minor=int(versions[1]) -pkg_installer = os_type == 'Darwin' and (version_major == 3 and version_minor >= 11) +pkg_installer = os_type == 'Darwin' and ((version_major == 3 and version_minor >= 11) or (architecture == "arm64")) lib_dir_path = sysconfig.get_config_var('LIBDIR') ld_library_name = sysconfig.get_config_var('LDLIBRARY') @@ -40,7 +41,7 @@ if lib_dir_path != expected_lib_dir_path: ### Validate shared libraries if is_shared: print('%s was built with shared extensions' % ld_library_name) - + ### Validate libpython extension ld_library_extension = ld_library_name.split('.')[-1] if ld_library_extension != expected_ld_library_extension: @@ -64,7 +65,7 @@ if os_type == 'Darwin': else: expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include' expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib' - + openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES') openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS') @@ -81,4 +82,4 @@ if os_type == 'Darwin': if sys.version_info < (3, 12): if not have_libreadline: print('Missing libreadline') - exit(1) \ No newline at end of file + exit(1)