diff --git a/.github/workflows/manifest-config-validation.yml b/.github/workflows/manifest-config-validation.yml new file mode 100644 index 0000000..cb4ce65 --- /dev/null +++ b/.github/workflows/manifest-config-validation.yml @@ -0,0 +1,32 @@ +name: Manifest config tests +on: + pull_request: + branches: + - main + paths-ignore: + - 'versions-manifest.json' + +jobs: + RunTests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + + - name: Install Pester + shell: pwsh + run: | + Install-Module Pester -Force -Scope CurrentUser + + - name: Run tests + shell: pwsh + run: | + Import-Module Pester + Invoke-Pester -Configuration @{ + Run = @{ + Path = "tests/ManifestConfig.Tests.ps1" + Exit = $true + } + } \ No newline at end of file diff --git a/azure-pipelines/templates/build-job.yml b/azure-pipelines/templates/build-job.yml index 31b41e7..0d949fe 100644 --- a/azure-pipelines/templates/build-job.yml +++ b/azure-pipelines/templates/build-job.yml @@ -14,6 +14,7 @@ jobs: targetType: filePath filePath: './builders/build-python.ps1' arguments: '-Version $(VERSION) -Platform $(Platform) -Architecture $(Architecture)' + pwsh: true - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: Python $(VERSION)' diff --git a/azure-pipelines/templates/test-job.yml b/azure-pipelines/templates/test-job.yml index e642c12..22e7ac2 100644 --- a/azure-pipelines/templates/test-job.yml +++ b/azure-pipelines/templates/test-job.yml @@ -3,6 +3,8 @@ jobs: pool: name: Azure Pipelines vmImage: $(VmImage) + variables: + TestRunTitle: 'python-$(Platform)-$(Architecture)' steps: - checkout: self submodules: true diff --git a/builders/build-python.ps1 b/builders/build-python.ps1 index b2cc672..a14f020 100644 --- a/builders/build-python.ps1 +++ b/builders/build-python.ps1 @@ -21,7 +21,7 @@ Required parameter. The platform for which Python will be built. #> param( - [Parameter (Mandatory=$true)][Version] $Version, + [Parameter (Mandatory=$true)][semver] $Version, [Parameter (Mandatory=$true)][string] $Platform, [string] $Architecture = "x64" ) @@ -29,6 +29,7 @@ param( Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "common-helpers.psm1") -DisableNameChecking Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "nix-helpers.psm1") -DisableNameChecking Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "win-helpers.psm1") -DisableNameChecking +Import-Module (Join-Path $PSScriptRoot "python-version.psm1") -DisableNameChecking function Get-PythonBuilder { <# @@ -49,13 +50,12 @@ function Get-PythonBuilder { #> - param ( - [version] $Version, + param( + [semver] $Version, [string] $Architecture, [string] $Platform ) - $Platform = $Platform.ToLower() if ($Platform -match 'win32') { $builder = [WinPythonBuilder]::New($Version, $Architecture, $Platform) } elseif ($Platform -match 'linux') { @@ -71,5 +71,5 @@ function Get-PythonBuilder { } ### Create Python builder instance, and build artifact -$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform +$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform $Builder.Build() diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 index e78ff0b..228aba3 100644 --- a/builders/macos-python-builder.psm1 +++ b/builders/macos-python-builder.psm1 @@ -17,7 +17,7 @@ class macOSPythonBuilder : NixPythonBuilder { #> macOSPythonBuilder( - [version] $version, + [semver] $version, [string] $architecture, [string] $platform ) : Base($version, $architecture, $platform) { } diff --git a/builders/nix-python-builder.psm1 b/builders/nix-python-builder.psm1 index f486697..a90e131 100644 --- a/builders/nix-python-builder.psm1 +++ b/builders/nix-python-builder.psm1 @@ -33,13 +33,12 @@ class NixPythonBuilder : PythonBuilder { [string] $OutputArtifactName NixPythonBuilder( - [version] $version, + [semver] $version, [string] $architecture, [string] $platform ) : Base($version, $architecture, $platform) { $this.InstallationTemplateName = "nix-setup-template.sh" $this.InstallationScriptName = "setup.sh" - $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz" } @@ -50,8 +49,10 @@ class NixPythonBuilder : PythonBuilder { #> $base = $this.GetBaseUri() + $versionName = $this.GetBaseVersion() + $nativeVersion = Convert-Version -version $this.Version - return "${base}/$($this.Version)/Python-$($this.Version).tgz" + return "${base}/${versionName}/Python-${nativeVersion}.tgz" } [string] GetPythonBinary() { @@ -95,9 +96,7 @@ class NixPythonBuilder : PythonBuilder { $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw $variablesToReplace = @{ - "{{__VERSION_MAJOR__}}" = $this.Version.Major; - "{{__VERSION_MINOR__}}" = $this.Version.Minor; - "{{__VERSION_BUILD__}}" = $this.Version.Build; + "{{__VERSION_FULL__}}" = $this.Version; } $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) } diff --git a/builders/python-builder.psm1 b/builders/python-builder.psm1 index d19644a..bb44e56 100644 --- a/builders/python-builder.psm1 +++ b/builders/python-builder.psm1 @@ -29,7 +29,7 @@ class PythonBuilder { #> - [version] $Version + [semver] $Version [string] $Architecture [string] $Platform [string] $HostedToolcacheLocation @@ -38,17 +38,17 @@ class PythonBuilder { [string] $ArtifactFolderLocation [string] $InstallationTemplatesLocation - PythonBuilder ([version] $version, [string] $architecture, [string] $platform) { - $this.Version = $version - $this.Architecture = $architecture - $this.Platform = $platform + PythonBuilder ([semver] $version, [string] $architecture, [string] $platform) { + $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers" $this.HostedToolcacheLocation = $env:AGENT_TOOLSDIRECTORY $this.TempFolderLocation = $env:BUILD_SOURCESDIRECTORY $this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY $this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY - $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers" + $this.Version = $version + $this.Architecture = $architecture + $this.Platform = $platform } [uri] GetBaseUri() { @@ -79,11 +79,21 @@ class PythonBuilder { return "$pythonToolcacheLocation/$($this.Version)/$($this.Architecture)" } + [string] GetBaseVersion() { + <# + .SYNOPSIS + Return Major.Minor.Patch version string. + #> + + return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)" + } + [void] PreparePythonToolcacheLocation() { <# .SYNOPSIS Prepare system hostedtoolcache folder for new Python version. #> + $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation() if (Test-Path $pythonBinariesLocation) { diff --git a/builders/python-version.psm1 b/builders/python-version.psm1 new file mode 100644 index 0000000..62edeea --- /dev/null +++ b/builders/python-version.psm1 @@ -0,0 +1,45 @@ +function Convert-Label() { + <# + .SYNOPSIS + Convert generic semver label to native Python label. + #> + + param( + [Parameter(Mandatory)] + [string] $label + ) + + switch ($label) { + "alpha" { return "a" } + "beta" { return "b" } + "rc" { return "rc" } + default { throw "Invalid version label '$label'" } + } +} + +function Convert-Version { + <# + .SYNOPSIS + Convert generic semver version to native Python version. + #> + + param( + [Parameter(Mandatory)] + [semver] $version, + [char] $delimiter = "." + ) + + $nativeVersion = "{0}.{1}.{2}" -f $version.Major, $version.Minor, $version.Patch + + if ($version.PreReleaseLabel) + { + $preReleaseLabel = $version.PreReleaseLabel.Split($delimiter) + + $preReleaseLabelName = Convert-Label -Label $preReleaseLabel[0] + $preReleaseLabelVersion = $preReleaseLabel[1] + + $nativeVersion += "${preReleaseLabelName}${preReleaseLabelVersion}" + } + + return $nativeVersion +} \ No newline at end of file diff --git a/builders/ubuntu-python-builder.psm1 b/builders/ubuntu-python-builder.psm1 index ac46277..a674168 100644 --- a/builders/ubuntu-python-builder.psm1 +++ b/builders/ubuntu-python-builder.psm1 @@ -17,7 +17,7 @@ class UbuntuPythonBuilder : NixPythonBuilder { #> UbuntuPythonBuilder( - [version] $version, + [semver] $version, [string] $architecture, [string] $platform ) : Base($version, $architecture, $platform) { } @@ -68,6 +68,7 @@ class UbuntuPythonBuilder : NixPythonBuilder { } else { $tkinterInstallString = "sudo apt install -y python-tk tk-dev" } + Execute-Command -Command $tkinterInstallString ### Install dependent packages diff --git a/builders/win-python-builder.psm1 b/builders/win-python-builder.psm1 index 1350e88..f7b7461 100644 --- a/builders/win-python-builder.psm1 +++ b/builders/win-python-builder.psm1 @@ -27,7 +27,7 @@ class WinPythonBuilder : PythonBuilder { [string] $OutputArtifactName WinPythonBuilder( - [version] $version, + [semver] $version, [string] $architecture, [string] $platform ) : Base($version, $architecture, $platform) { @@ -72,10 +72,12 @@ class WinPythonBuilder : PythonBuilder { #> $base = $this.GetBaseUri() + $versionName = $this.GetBaseVersion() + $nativeVersion = Convert-Version -version $this.Version $architecture = $this.GetArchitectureExtension() $extension = $this.GetPythonExtension() - $uri = "${base}/$($this.Version)/python-$($this.Version)${architecture}${extension}" + $uri = "${base}/${versionName}/python-${nativeVersion}${architecture}${extension}" return $uri } diff --git a/config/python-manifest-config.json b/config/python-manifest-config.json index daa5781..676da2c 100644 --- a/config/python-manifest-config.json +++ b/config/python-manifest-config.json @@ -1,8 +1,8 @@ { - "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+)-([\\w\\.]+)?-?(x\\d+)", + "regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?(x\\d+)", "groups": { - "arch": 3, - "platform": 1, - "platform_version": 2 + "arch": 4, + "platform": 2, + "platform_version": 3 } } \ No newline at end of file diff --git a/installers/nix-setup-template.sh b/installers/nix-setup-template.sh index f3ff955..21d72b8 100644 --- a/installers/nix-setup-template.sh +++ b/installers/nix-setup-template.sh @@ -1,13 +1,12 @@ set -e -MAJOR_VERSION="{{__VERSION_MAJOR__}}" -MINOR_VERSION="{{__VERSION_MINOR__}}" -BUILD_VERSION="{{__VERSION_BUILD__}}" +PYTHON_FULL_VERSION="{{__VERSION_FULL__}}" +MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1) +MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2) PYTHON_MAJOR=python$MAJOR_VERSION PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION PYTHON_MAJORMINOR=python$MAJOR_VERSION$MINOR_VERSION -PYTHON_FULL_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$BUILD_VERSION if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then # No AGENT_TOOLSDIRECTORY on GitHub images diff --git a/installers/win-setup-template.ps1 b/installers/win-setup-template.ps1 index 4a3f835..a194541 100644 --- a/installers/win-setup-template.ps1 +++ b/installers/win-setup-template.ps1 @@ -1,10 +1,9 @@ [String] $Architecture = "{{__ARCHITECTURE__}}" -[Version] $Version = "{{__VERSION__}}" +[String] $Version = "{{__VERSION__}}" [String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}" function Get-RegistryVersionFilter { - param - ( + param( [Parameter(Mandatory)][String] $Architecture, [Parameter(Mandatory)][Int32] $MajorVersion, [Parameter(Mandatory)][Int32] $MinorVersion @@ -12,20 +11,15 @@ function Get-RegistryVersionFilter { $archFilter = if ($Architecture -eq 'x86') { "32-bit" } else { "64-bit" } ### Python 2.7 x86 have no architecture postfix - if (($Architecture -eq "x86") -and ($MajorVersion -eq 2)) - { + if (($Architecture -eq "x86") -and ($MajorVersion -eq 2)) { "Python $MajorVersion.$MinorVersion.\d+$" - } - else - { + } else { "Python $MajorVersion.$MinorVersion.*($archFilter)" } } -function Remove-RegistryEntries -{ - param - ( +function Remove-RegistryEntries { + param( [Parameter(Mandatory)][String] $Architecture, [Parameter(Mandatory)][Int32] $MajorVersion, [Parameter(Mandatory)][Int32] $MinorVersion @@ -35,10 +29,8 @@ function Remove-RegistryEntries $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products" $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName - foreach ($key in $regKeys) - { - if ($key.getValue("DisplayName") -match $versionFilter) - { + foreach ($key in $regKeys) { + if ($key.getValue("DisplayName") -match $versionFilter) { Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose } } @@ -63,40 +55,34 @@ function Remove-RegistryEntries } function Get-ExecParams { - param - ( + param( [Parameter(Mandatory)][Boolean] $IsMSI, [Parameter(Mandatory)][String] $PythonArchPath ) - if ($IsMSI) - { + if ($IsMSI) { "TARGETDIR=$PythonArchPath ALLUSERS=1" - } - else - { + } else { "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1" } } $ToolcacheRoot = $env:AGENT_TOOLSDIRECTORY -if ([string]::IsNullOrEmpty($ToolcacheRoot)) -{ +if ([string]::IsNullOrEmpty($ToolcacheRoot)) { # GitHub images don't have `AGENT_TOOLSDIRECTORY` variable $ToolcacheRoot = $env:RUNNER_TOOL_CACHE } $PythonToolcachePath = Join-Path -Path $ToolcacheRoot -ChildPath "Python" -$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version.ToString() +$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version $PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture $IsMSI = $PythonExecName -match "msi" -$MajorVersion = $Version.Major -$MinorVersion = $Version.Minor +$MajorVersion = $Version.Split('.')[0] +$MinorVersion = $Version.Split('.')[1] Write-Host "Check if Python hostedtoolcache folder exist..." -if (-Not (Test-Path $PythonToolcachePath)) -{ +if (-Not (Test-Path $PythonToolcachePath)) { Write-Host "Create Python toolcache folder" New-Item -ItemType Directory -Path $PythonToolcachePath | Out-Null } @@ -104,22 +90,17 @@ if (-Not (Test-Path $PythonToolcachePath)) Write-Host "Check if current Python version is installed..." $InstalledVersions = Get-Item "$PythonToolcachePath\$MajorVersion.$MinorVersion.*\$Architecture" -if ($null -ne $InstalledVersions) -{ +if ($null -ne $InstalledVersions) { Write-Host "Python$MajorVersion.$MinorVersion ($Architecture) was found in $PythonToolcachePath..." - foreach ($InstalledVersion in $InstalledVersions) - { - if (Test-Path -Path $InstalledVersion) - { + foreach ($InstalledVersion in $InstalledVersions) { + if (Test-Path -Path $InstalledVersion) { Write-Host "Deleting $InstalledVersion..." Remove-Item -Path $InstalledVersion -Recurse -Force Remove-Item -Path "$($InstalledVersion.Parent.FullName)/${Architecture}.complete" -Force -Verbose } } -} -else -{ +} else { Write-Host "No Python$MajorVersion.$MinorVersion.* found" } @@ -136,8 +117,7 @@ Write-Host "Install Python $Version in $PythonToolcachePath..." $ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet" -if ($LASTEXITCODE -ne 0) -{ +if ($LASTEXITCODE -ne 0) { Throw "Error happened during Python installation" } diff --git a/tests/ManifestConfig.Tests.ps1 b/tests/ManifestConfig.Tests.ps1 new file mode 100644 index 0000000..d04d81d --- /dev/null +++ b/tests/ManifestConfig.Tests.ps1 @@ -0,0 +1,44 @@ +Import-Module (Join-Path $PSScriptRoot "../helpers/packages-generation/manifest-utils.psm1") + +$ConfigurationFile = Join-Path $PSScriptRoot "../config/python-manifest-config.json" +$Configuration = Read-ConfigurationFile -Filepath $ConfigurationFile + +$stableTestCases = @( + @{ ReleaseName = "python-3.8.3-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} }, + @{ ReleaseName = "python-3.8.3-linux-16.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "16.04"; arch = "x64"} }, + @{ ReleaseName = "python-3.8.3-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} }, + @{ ReleaseName = "python-3.8.3-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} }, + @{ ReleaseName = "python-3.8.3-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} }, + @{ ReleaseName = "python-3.8.3-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} } +) | ForEach-Object { $_.Configuration = $Configuration; $_ } + +$unstableTestCases = @( + @{ ReleaseName = "python-3.9.0-alpha.2-darwin-x64.tar.gz"; ExpectedResult = @{ platform = "darwin"; platform_version = $null; arch = "x64"} }, + @{ ReleaseName = "python-3.9.0-beta.1-linux-16.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "16.04"; arch = "x64"} }, + @{ ReleaseName = "python-3.9.0-rc.4-linux-18.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "18.04"; arch = "x64"} }, + @{ ReleaseName = "python-3.9.0-beta.2-linux-20.04-x64.tar.gz"; ExpectedResult = @{ platform = "linux"; platform_version = "20.04"; arch = "x64"} }, + @{ ReleaseName = "python-3.9.0-beta.2-win32-x64.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x64"} }, + @{ ReleaseName = "python-3.9.0-beta.2-win32-x86.zip"; ExpectedResult = @{ platform = "win32"; platform_version = $null; arch = "x86"} } +) | ForEach-Object { $_.Configuration = $Configuration; $_ } + +Describe "Python manifest config" { + Context "Stable versions" { + It "" -TestCases $stableTestCases { + $Release = @{ name = $ReleaseName } + $asset = New-AssetItem -ReleaseAsset $Release -Configuration $Configuration + $asset.platform | Should -Be $ExpectedResult.platform + $asset.platform_version | Should -Be $ExpectedResult.platform_version + $asset.arch | Should -Be $ExpectedResult.arch + } + } + + Context "Prerelease versions" { + It "" -TestCases $unstableTestCases { + $Release = @{ name = $ReleaseName } + $asset = New-AssetItem -ReleaseAsset $Release -Configuration $Configuration + $asset.platform | Should -Be $ExpectedResult.platform + $asset.platform_version | Should -Be $ExpectedResult.platform_version + $asset.arch | Should -Be $ExpectedResult.arch + } + } +} \ No newline at end of file diff --git a/tests/python-tests.ps1 b/tests/python-tests.ps1 index ad5f37f..550ea4e 100644 --- a/tests/python-tests.ps1 +++ b/tests/python-tests.ps1 @@ -1,7 +1,7 @@ param ( - [Version] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] + [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] $Version, - [String] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] + [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] $Platform ) @@ -59,7 +59,7 @@ Describe "Tests" { } It "Check if python configuration is correct" { - "python ./sources/python-config-test.py" | Should -ReturnZeroExitCode + "python ./sources/python-config-test.py $Version" | Should -ReturnZeroExitCode } It "Check if shared libraries are linked correctly" { diff --git a/tests/sources/python-config-test.py b/tests/sources/python-config-test.py index fe4d852..ddc8602 100644 --- a/tests/sources/python-config-test.py +++ b/tests/sources/python-config-test.py @@ -6,7 +6,7 @@ import os # Define variables os_type = platform.system() -version = sys.version.split(" ")[0] +version = sys.argv[1] lib_dir_path = sysconfig.get_config_var('LIBDIR') ld_library_name = sysconfig.get_config_var('LDLIBRARY') diff --git a/tests/sources/python-modules.py b/tests/sources/python-modules.py index 76408b7..5dc9197 100644 --- a/tests/sources/python-modules.py +++ b/tests/sources/python-modules.py @@ -251,6 +251,10 @@ if sys.version_info >= (3, 7): if sys.version_info > (3, 7): standard_library.remove('macpath') +# 'dummy_threading' module has been removed from Python 3.9 +if sys.version_info > (3, 8): + standard_library.remove('dummy_threading') + # Remove tkinter and Easter eggs excluded_modules = [ 'antigravity',