From 3b05f538075a46adb29da494d6731f4d5538cc2f Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Thu, 15 Sep 2022 10:31:47 +0200 Subject: [PATCH] Instal osx python from prebuilt packagese (#184) * Download osx packages and setup script * install from pkg for 3.11 only * More debug * More debug * fix version check * New-Item build_output.txt * installationTemplateLocation * fix version * fix beta version * fix building from source * fix building from source * fix pkg name * fix setup.sh * fix test * Fix config test with semver * Fix PYTHON_MAJOR_MINOR * migrate from .format to interpolation * add PYTHON_FRAMEWORK_PATH variable * improve pkg condition --- builders/macos-python-builder.psm1 | 88 ++++++++++++++++++++++++++ installers/macos-pkg-setup-template.sh | 68 ++++++++++++++++++++ tests/sources/python-config-test.py | 23 +++++-- 3 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 installers/macos-pkg-setup-template.sh diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index 2358881..f31d38c 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -83,4 +83,92 @@ class macOSPythonBuilder : NixPythonBuilder { Execute-Command -Command $configureString } + + [string] GetPkgName() { + <# + .SYNOPSIS + Return Python installation Package. + #> + + $nativeVersion = Convert-Version -version $this.Version + $architecture = "-macos11" + $extension = ".pkg" + + $pkg = "python-${nativeVersion}${architecture}${extension}" + + return $pkg + } + + [uri] GetPkgUri() { + <# + .SYNOPSIS + Get base Python URI and return complete URI for Python installation package. + #> + + $base = $this.GetBaseUri() + $versionName = $this.GetBaseVersion() + $pkg = $this.GetPkgName() + + $uri = "${base}/${versionName}/${pkg}" + + return $uri + } + + [string] DownloadPkg() { + <# + .SYNOPSIS + Download Python installation executable into artifact location. + #> + + $pkgUri = $this.GetPkgUri() + + Write-Host "Sources URI: $pkgUri" + $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation + Write-Debug "Done; Package location: $pkgLocation" + + New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File + return $pkgLocation + } + + [void] CreateInstallationScriptPkg() { + <# + .SYNOPSIS + Create Python artifact installation script based on specified template. + #> + + $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath "macos-pkg-setup-template.sh" + $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw + $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File + + $variablesToReplace = @{ + "{{__VERSION_FULL__}}" = $this.Version; + "{{__PKG_NAME__}}" = $this.GetPkgName(); + } + + $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) } + $installationTemplateContent | Out-File -FilePath $installationScriptLocation + Write-Debug "Done; Installation script location: $installationScriptLocation)" + } + + [void] Build() { + <# + .SYNOPSIS + Generates Python artifact from downloaded Python installation executable. + #> + + $PkgVersion = [semver]"3.11.0-beta.1" + + if ($this.Version -ge $PkgVersion) { + Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..." + $this.DownloadPkg() + + Write-Host "Create installation script..." + $this.CreateInstallationScriptPkg() + } else { + ([NixPythonBuilder]$this).Build() + } + + Write-Host "Archive artifact" + $this.ArchiveArtifact() + } } diff --git a/installers/macos-pkg-setup-template.sh b/installers/macos-pkg-setup-template.sh new file mode 100644 index 0000000..67c57c8 --- /dev/null +++ b/installers/macos-pkg-setup-template.sh @@ -0,0 +1,68 @@ +set -e + +PYTHON_FULL_VERSION="{{__VERSION_FULL__}}" +PYTHON_PKG_NAME="{{__PKG_NAME__}}" +MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1) +MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2) + +PYTHON_MAJOR=python$MAJOR_VERSION +PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION +PYTHON_MAJOR_MINOR=python$MAJOR_VERSION$MINOR_VERSION + +if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then + # No AGENT_TOOLSDIRECTORY on GitHub images + TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE +else + TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY +fi + +PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python +PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION +PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64 +PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}" + +echo "Check if Python hostedtoolcache folder exist..." +if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then + echo "Creating Python hostedtoolcache folder..." + mkdir -p $PYTHON_TOOLCACHE_PATH +else + # remove ALL other directories for same major.minor python versions + find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do + python_version_x64="$python_version/x64" + if [ -e "$python_version_x64" ];then + echo "Deleting Python $python_version_x64" + rm -rf "$python_version_x64" + fi + done +fi + +echo "Install Python binaries from prebuilt package" +sudo installer -pkg $PYTHON_PKG_NAME -target / + +echo "Create hostedtoolcach symlinks (Required for the backward compatibility)" +echo "Create Python $PYTHON_FULL_VERSION folder" +mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH +cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH + +ln -s "${PYTHON_FRAMEWORK_PATH}/bin" bin +ln -s "${PYTHON_FRAMEWORK_PATH}/include" include +ln -s "${PYTHON_FRAMEWORK_PATH}/share" share +ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib + +echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)" +ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python + +cd bin/ +ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR +if [ ! -f python ]; then + ln -s $PYTHON_MAJOR_DOT_MINOR python +fi + +chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python + +echo "Upgrading pip..." +./python -m ensurepip +./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location + +echo "Create complete file" +touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete diff --git a/tests/sources/python-config-test.py b/tests/sources/python-config-test.py index a53ad6c..9ebcf24 100644 --- a/tests/sources/python-config-test.py +++ b/tests/sources/python-config-test.py @@ -10,6 +10,12 @@ os_type = platform.system() version = sys.argv[1] nativeVersion = sys.argv[2] +versions=version.split(".") +version_major=int(versions[0]) +version_minor=int(versions[1]) + +pkg_installer = os_type == 'Darwin' and (version_major == 3 and version_minor >= 11) + lib_dir_path = sysconfig.get_config_var('LIBDIR') ld_library_name = sysconfig.get_config_var('LDLIBRARY') @@ -19,7 +25,11 @@ have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE") ### Define expected variables if os_type == 'Linux': expected_ld_library_extension = 'so' if os_type == 'Darwin': expected_ld_library_extension = 'dylib' -expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version) + +if pkg_installer: + expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib' +else: + expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib' # Check modules ### Validate libraries path @@ -38,7 +48,8 @@ if is_shared: exit(1) else: print('%s was built without shared extensions' % ld_library_name) - exit(1) + if not pkg_installer: + exit(1) ### Validate macOS if os_type == 'Darwin': @@ -59,12 +70,14 @@ if os_type == 'Darwin': if openssl_includes != expected_openssl_includes: print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes)) - exit(1) + if not pkg_installer: + exit(1) if openssl_ldflags != expected_openssl_ldflags: print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags)) - exit(1) + if not pkg_installer: + exit(1) ### Validate libreadline if not have_libreadline: print('Missing libreadline') - exit(1) \ No newline at end of file + exit(1)