Revert 1 padil2842 patch 1 (#3)

* Update versions-manifest based on build from 20200721.2 (#43) Co-authored-by: Service account <no-reply@microsoft.com> * Update versions-manifest based on build from 20200721.2 (#44) Co-authored-by: Service account <no-reply@microsoft.com> * Update versions-manifest.json * Update Git submodule (#45) Co-authored-by: MaksimZhukov <v-mazhuk@microsoft.com> * Update versions-manifest based on build from 20200812.1 * Update versions-manifest based on build from 20200818.1 * Move release and PR creation to the GitHub Actions * Add link to the created PR * Update versions-manifest * Update versions-manifest * Update versions-manifest * Add 3.9.0 (#55) Co-authored-by: Service account <no-reply@microsoft.com> * add new exclude * add version compare * Update versions-manifest * add new openssl * add fix to confirm uninstalling * change connection * change to static path * change links * revert test changes * remove debug lines * skip test for python 2 (#60) Co-authored-by: Dmitry Shibanov <v-dmshib@microsoft.com> * Update versions-manifest * Update versions-manifest * Update versions-manifest * Fix PowerShell issue with relative paths * Update overwriteExistingFiles input * Update python-config-test.py * Update versions-manifest * Update the list of modules in tests for Python 3.10 * Update versions-manifest * Update versions-manifest * Add a new workflow to test Python versions * Update the brew upgrade command * Remove temporary step * Update versions-manifest * Update versions-manifest (#74) Co-authored-by: Service account <no-reply@microsoft.com> * add flag to skip warnings of location of pip * Update versions-manifest * Create CODEOWNERS (#75) * Create CODEOWNERS * Update CODEOWNERS * Create python3 symlink on Windows * Update manifest * Remove test changes in manifest * Update versions-manifest * Update versions-manifest * Update versions-manifest * Update versions-manifest * Update versions-manifest (#83) Co-authored-by: Service account <no-reply@microsoft.com> * Skip tkinter failed build for alpha.6 * Update versions-manifest (#85) Co-authored-by: Service account <no-reply@microsoft.com> * Update versions-manifest * Update versions-manifest * Update condition to skip all alpha and beta python versions for 3.10.0 * Fix condition * Update versions-manifest * Update versions-manifest (#90) Co-authored-by: Service account <no-reply@microsoft.com> * Update versions-manifest * Update versions-manifest * Update versions-manifest * Test registry path before invoking Get-ChildItem (#95) * Update versions-manifest * Update versions-manifest * Update versions-manifest * Update versions-manifest * Update versions-manifest * Update versions-manifest (#102) Co-authored-by: Service account <no-reply@microsoft.com> * Readme to readme * Revert "Readme to readme" Co-authored-by: MaksimZhukov <46996400+MaksimZhukov@users.noreply.github.com> Co-authored-by: Service account <no-reply@microsoft.com> Co-authored-by: Maxim Lobanov <maxim-lobanov@github.com> Co-authored-by: MaksimZhukov <v-mazhuk@microsoft.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Dmitry Shibanov <v-dmshib@microsoft.com> Co-authored-by: Dmitry Shibanov <shibanov-1997@inbox.ru> Co-authored-by: Dmitry Shibanov <dmitry-shibanov@github.com> Co-authored-by: Vladimir Safonkin <vsafonkin@github.com> Co-authored-by: Sergey Dolin <v-sedoli@micorosoft.com> Co-authored-by: Alena Sviridenko <alenasviridenko@github.com> Co-authored-by: Ray <ritchxu@github.com>
2021-07-30 21:28:36 +08:00 · 2021-07-30 21:28:36 +08:00 · c427560b3d
parent bbbf87f2cf
commit c427560b3d
20 changed files with 2061 additions and 196 deletions
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@ -0,0 +1 @@
+* @actions/virtual-environments-owners
--- a/.github/workflows/create-pr.yml
+++ b/.github/workflows/create-pr.yml
@ -0,0 +1,33 @@
+name: Create Pull Request
+on:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  create_pr:
+    name: Create Pull Request
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Create versions-manifest.json
+      run: |
+        ./helpers/packages-generation/manifest-generator.ps1 -RepositoryFullName "$env:GITHUB_REPOSITORY" `
+                                                             -GitHubAccessToken "${{secrets.GITHUB_TOKEN}}" `
+                                                             -OutputFile "./versions-manifest.json" `
+                                                             -ConfigurationFile "./config/python-manifest-config.json"
+    - name: Create GitHub PR
+      run: |
+        $formattedDate = Get-Date -Format "MM/dd/yyyy"
+        ./helpers/github/create-pull-request.ps1 `
+                            -RepositoryFullName "$env:GITHUB_REPOSITORY" `
+                            -AccessToken "${{secrets.GITHUB_TOKEN}}" `
+                            -BranchName "update-versions-manifest-file" `
+                            -CommitMessage "Update versions-manifest" `
+                            -PullRequestTitle "[versions-manifest] Update for release from ${formattedDate}" `
+                            -PullRequestBody "Update versions-manifest.json for release from ${formattedDate}"
--- a/.github/workflows/create-release.yml
+++ b/.github/workflows/create-release.yml
@ -0,0 +1,18 @@
+name: Create release
+on:
+  repository_dispatch:
+    types: [create-release]
+
+jobs:
+  create_release:
+    name: Create release ${{ github.event.client_payload.ToolVersion }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Create release for Python ${{ github.event.client_payload.ToolVersion }}
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.event.client_payload.TagName }}
+        release_name: ${{ github.event.client_payload.ToolVersion }}
+        body: ${{ github.event.client_payload.ReleaseBody }}
--- a/.github/workflows/test-python-version.yml
+++ b/.github/workflows/test-python-version.yml
@ -0,0 +1,34 @@
+name: Test Python version
+on: 
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Python version to build'
+        required: true
+      architecture:
+        description: 'The target architecture (x86, x64) of the Python'
+        required: false
+        default: 'x64'
+
+jobs:
+  test-python:
+    name: Test Python ${{ github.event.inputs.version }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04] 
+    steps:
+    - name: Setup Python ${{ github.event.inputs.version }}
+      uses: actions/setup-python@main
+      with:
+        python-version: ${{ github.event.inputs.version }}
+        architecture: ${{ github.event.inputs.architecture }}
+
+    - name: Validate version
+      run: |
+        python --version
+      shell: pwsh
+
+    - name: Run simple code
+      run: python -c 'import math; print(math.factorial(5))'
--- a/azure-pipelines/build-python-packages.yml
+++ b/azure-pipelines/build-python-packages.yml
@ -116,3 +116,17 @@ stages:
    Architecture: x86
  jobs:
  - template: /azure-pipelines/templates/test-job.yml
+
+- stage: Publish_Release
+  dependsOn: [Test_Python_MacOS, Test_Python_Ubuntu_1604, Test_Python_Ubuntu_1804, Test_Python_Ubuntu_2004, Test_Python_x64_Windows, Test_Python_x86_Windows]
+  jobs:
+  - deployment: Publish_Release
+    pool:
+      name: Azure Pipelines
+      vmImage: ubuntu-18.04
+    environment: 'Get Available Tools Versions - Publishing Approval'
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - template: /azure-pipelines/templates/publish-release-steps.yml
--- a/azure-pipelines/templates/build-job.yml
+++ b/azure-pipelines/templates/build-job.yml
@ -7,7 +7,7 @@ jobs:
  steps:
  - checkout: self
    submodules: true
-    
+
  - task: PowerShell@2
    displayName: 'Build Python $(VERSION)'
    inputs:
--- a/azure-pipelines/templates/publish-release-steps.yml
+++ b/azure-pipelines/templates/publish-release-steps.yml
@ -0,0 +1,47 @@
+steps:
+- download: none
+
+- checkout: self
+  submodules: true
+
+- task: DownloadPipelineArtifact@2
+  inputs:
+    source: 'current'
+    path: $(Build.BinariesDirectory)
+
+- task: PowerShell@2
+  displayName: 'Create release Python $(VERSION)'
+  inputs:
+    TargetType: inline
+    script: |
+      $tagName = "$(VERSION)-$(Build.BuildId)"
+      $releaseBody = "Python $(VERSION)"
+      ./helpers/github/create-release.ps1 -RepositoryFullName "$(Build.Repository.Name)" `
+                                          -AccessToken "$(GITHUB_TOKEN)" `
+                                          -ToolVersion "$(VERSION)" `
+                                          -TagName "$tagName" `
+                                          -ReleaseBody "$releaseBody" `
+                                          -EventType "$(EVENT_TYPE)"
+- task: GitHubRelease@1
+  displayName: 'Upload release assets'
+  inputs:
+    gitHubConnection: 'Github Connection'
+    action: edit
+    tag: '$(VERSION)-$(Build.BuildId)'
+    title: '$(VERSION)'
+    releaseNotesSource: inline
+    releaseNotesInline: '$(RELEASE_NOTES_CONTENT)'
+    assets: '$(Build.BinariesDirectory)/*/*'
+    assetUploadMode: replace
+    addChangeLog: false
+
+- task: PowerShell@2
+  displayName: 'Trigger "Create Pull Request" workflow'
+  inputs:
+    TargetType: inline
+    script: |
+      Import-Module (Join-Path (Get-Location).Path "github-api.psm1")
+      $gitHubApi = Get-GitHubApi -RepositoryFullName "$(Build.Repository.Name)" -AccessToken "$(GITHUB_TOKEN)"
+      $gitHubApi.CreateWorkflowDispatch("$(WORKFLOW_FILE_NAME)", "$(WORKFLOW_DISPATCH_REF)", "$(INPUTS)")
+      Write-Host "Please find created Pull request here: $(Build.Repository.Uri)/pulls"
+    workingDirectory: '$(Build.SourcesDirectory)/helpers/github'
--- a/azure-pipelines/templates/test-job.yml
+++ b/azure-pipelines/templates/test-job.yml
@ -27,6 +27,7 @@ jobs:
      archiveFilePatterns: '$(Build.BinariesDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).*'
      destinationFolder: $(Build.BinariesDirectory)
      cleanDestinationFolder: false
+      overwriteExistingFiles: true

  - task: PowerShell@2
    displayName: 'Apply build artifact to the local machines'
--- a/builders/build-python.ps1
+++ b/builders/build-python.ps1
@ -1,6 +1,6 @@
-using module "./builders/win-python-builder.psm1"
-using module "./builders/ubuntu-python-builder.psm1"
-using module "./builders/macos-python-builder.psm1"
+using module "./win-python-builder.psm1"
+using module "./ubuntu-python-builder.psm1"
+using module "./macos-python-builder.psm1"

 <#
 .SYNOPSIS
--- a/builders/macos-python-builder.psm1
+++ b/builders/macos-python-builder.psm1
@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"

 class macOSPythonBuilder : NixPythonBuilder {
    <#
@ -40,10 +40,10 @@ class macOSPythonBuilder : NixPythonBuilder {
        ### and then add the appropriate paths for the header and library files to configure command.
        ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
        if ($this.Version -lt "3.7.0") {
-            $env:LDFLAGS = "-L$(brew --prefix openssl)/lib"
-            $env:CFLAGS = "-I$(brew --prefix openssl)/include"
+            $env:LDFLAGS = "-L/usr/local/opt/openssl@1.1/lib"
+            $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include"
        } else {
-            $configureString += " --with-openssl=/usr/local/opt/openssl"
+            $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
        }

        ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
--- a/builders/nix-python-builder.psm1
+++ b/builders/nix-python-builder.psm1
@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"

 class NixPythonBuilder : PythonBuilder {
    <#
--- a/builders/ubuntu-python-builder.psm1
+++ b/builders/ubuntu-python-builder.psm1
@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"

 class UbuntuPythonBuilder : NixPythonBuilder {
    <#
--- a/builders/win-python-builder.psm1
+++ b/builders/win-python-builder.psm1
@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"

 class WinPythonBuilder : PythonBuilder {
    <#
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 1069e7a4ece355bd321909454fc1f9f7244eb03b
+Subproject commit f8f76caff1878d249468db3d55f045c0b707493b
--- a/installers/nix-setup-template.sh
+++ b/installers/nix-setup-template.sh
@ -50,7 +50,7 @@ chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR pyth

 echo "Upgrading PIP..."
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip
+./python -m pip install --ignore-installed pip --no-warn-script-location

 echo "Create complete file"
 touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete
--- a/installers/win-setup-template.ps1
+++ b/installers/win-setup-template.ps1
@ -28,16 +28,20 @@ function Remove-RegistryEntries {
    $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion

    $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
-    $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
-    foreach ($key in $regKeys) {
-        if ($key.getValue("DisplayName") -match $versionFilter) {
-            Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+    if (Test-Path -Path Registry::$regPath) {
+        $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
+        foreach ($key in $regKeys) {
+            if ($key.getValue("DisplayName") -match $versionFilter) {
+                Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+            }
        }
    }

    $regPath = "HKEY_CLASSES_ROOT\Installer\Products"
-    Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
-        Remove-Item Registry::$_ -Recurse -Force -Verbose
+    if (Test-Path -Path Registry::$regPath) {
+        Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
+            Remove-Item Registry::$_ -Recurse -Force -Verbose
+        }
    }

    $uninstallRegistrySections = @(
@ -121,9 +125,14 @@ if ($LASTEXITCODE -ne 0) {
    Throw "Error happened during Python installation"
 }

+Write-Host "Create `python3` symlink"
+if ($MajorVersion -ne "2") {
+    New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe"
+}
+
 Write-Host "Install and upgrade Pip"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location"

 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null
--- a/tests/python-tests.ps1
+++ b/tests/python-tests.ps1
@ -7,6 +7,7 @@ param (

 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
 Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
+Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")

 function Analyze-MissingModules([string] $buildOutputLocation) {
    $searchStringStart = "Failed to build these modules:"
@ -20,9 +21,14 @@ function Analyze-MissingModules([string] $buildOutputLocation) {
    $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern)
    if ($regexMatch.Success)
    {
+        $module = $regexMatch.Groups[1].Value.Trim()
        Write-Host "Failed missing modules:"
-        Write-Host $regexMatch.Groups[1].Value
-        return 1
+        Write-Host $module
+        if ( ($module -eq "_tkinter") -and ( ($Version -like "3.10.0-beta*") -or ($Version -like "3.10.0-alpha*") ) ) {
+          Write-Host "$module $Version ignored"
+        } else {
+          return 1
+        }
    }

    return 0
@ -47,6 +53,11 @@ Describe "Tests" {
        }
    }

+    It "Run pip" {
+        "pip install requests" | Should -ReturnZeroExitCode
+        "pip uninstall requests -y" | Should -ReturnZeroExitCode
+    }
+
    if (IsNixPlatform $Platform) {

        It "Check for failed modules in build_output" {
@ -59,7 +70,8 @@ Describe "Tests" {
        }

        It "Check if python configuration is correct" {
-            "python ./sources/python-config-test.py $Version" | Should -ReturnZeroExitCode
+            $nativeVersion = Convert-Version -version $Version
+            "python ./sources/python-config-test.py $Version $nativeVersion" | Should -ReturnZeroExitCode
        }

        It "Check if shared libraries are linked correctly" {
@ -68,11 +80,11 @@ Describe "Tests" {
    }

    # Pyinstaller 3.5 does not support Python 3.8.0. Check issue https://github.com/pyinstaller/pyinstaller/issues/4311
-    if ($Version -lt "3.8.0") {
+    if ($Version -lt "3.8.0" -and $Version.Major -ne "2") {
        It "Validate Pyinstaller" {
            "pip install pyinstaller" | Should -ReturnZeroExitCode
            "pyinstaller --onefile ./sources/simple-test.py" | Should -ReturnZeroExitCode
            "./dist/simple-test" | Should -ReturnZeroExitCode
        }
    }
-}
+}
--- a/tests/sources/python-config-test.py
+++ b/tests/sources/python-config-test.py
@ -1,4 +1,5 @@
 import distutils.sysconfig
+from distutils.version import LooseVersion
 import sysconfig
 import sys
 import platform
@ -7,6 +8,7 @@ import os
 # Define variables
 os_type = platform.system()
 version = sys.argv[1]
+nativeVersion = sys.argv[2]

 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
@ -41,7 +43,7 @@ else:
 ### Validate macOS
 if os_type == 'Darwin':
    ### Validate openssl links
-    if version < "3.7.0":
+    if LooseVersion(nativeVersion) < LooseVersion("3.7.0"):
        expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib'
        ldflags = sysconfig.get_config_var('LDFLAGS')

@ -49,8 +51,8 @@ if os_type == 'Darwin':
            print('Invalid ldflags: %s; Expected: %s' % (ldflags, expected_ldflags))
            exit(1)
    else:
-        expected_openssl_includes = '-I/usr/local/opt/openssl/include'
-        expected_openssl_ldflags ='-L/usr/local/opt/openssl/lib'
+        expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
+        expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
        
        openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
        openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')
--- a/tests/sources/python-modules.py
+++ b/tests/sources/python-modules.py
@ -139,6 +139,7 @@ standard_library = [
    'sre_constants',
    'sre_parse',
    'ssl',
+    '_ssl',
    'stat',
    'string',
    'stringprep',
@ -255,6 +256,11 @@ if sys.version_info > (3, 7):
 if sys.version_info > (3, 8):
    standard_library.remove('dummy_threading')

+# 'symbol' and 'formatter' modules have been removed from Python 3.10
+if sys.version_info >= (3, 10):
+    standard_library.remove('symbol')
+    standard_library.remove('formatter')
+
 # Remove tkinter and Easter eggs
 excluded_modules = [
    'antigravity',
@ -265,7 +271,6 @@ excluded_modules = [
 def check_missing_modules(expected_modules):
    missing = []
    for module in expected_modules:
-        print('Try to import module ', module)
        try:
            importlib.import_module(module)
        except:
--- a/versions-manifest.json
+++ b/versions-manifest.json