From 59e54b2d8b2a7f024050f575fdf1be37bcceeed7 Mon Sep 17 00:00:00 2001 From: Mikhail Koliada <88318005+mikhailkoliada@users.noreply.github.com> Date: Thu, 23 Dec 2021 17:34:17 +0300 Subject: [PATCH] Migrate python-versions pipeline to GH Actions --- .github/workflows/create-pr.yml | 2 +- .github/workflows/python-builder.yml | 220 +++++++++++++++++++++++++++ builders/python-builder.psm1 | 11 +- tests/python-tests.ps1 | 4 +- 4 files changed, 230 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/python-builder.yml diff --git a/.github/workflows/create-pr.yml b/.github/workflows/create-pr.yml index 5a4f5a7..b1d5ef7 100644 --- a/.github/workflows/create-pr.yml +++ b/.github/workflows/create-pr.yml @@ -30,4 +30,4 @@ jobs: -BranchName "update-versions-manifest-file" ` -CommitMessage "Update versions-manifest" ` -PullRequestTitle "[versions-manifest] Update for release from ${formattedDate}" ` - -PullRequestBody "Update versions-manifest.json for release from ${formattedDate}" \ No newline at end of file + -PullRequestBody "Update versions-manifest.json for release from ${formattedDate}" diff --git a/.github/workflows/python-builder.yml b/.github/workflows/python-builder.yml new file mode 100644 index 0000000..2fdb91c --- /dev/null +++ b/.github/workflows/python-builder.yml @@ -0,0 +1,220 @@ +name: Build python package + +on: + workflow_dispatch: + inputs: + VERSION: + description: 'Python version to build and upload' + default: '3.9.6' + required: true + PUBLISH_RELEASES: + description: 'Whether to publish releases' + required: true + default: 'false' + pull_request: + paths-ignore: + - 'versions-manifest.json' + - 'LICENSE' + - '**.md' + branches: + - 'main' + +env: + VERSION: ${{ github.event.inputs.VERSION}} +defaults: + run: + shell: pwsh + +jobs: + build_python: + strategy: + fail-fast: false + matrix: + include: + - platform: 'linux-18.04' + os: 'ubuntu-18.04' + arch: 'x64' + - platform: 'linux-20.04' + os: 'ubuntu-20.04' + arch: 'x64' + - platform: 'darwin' + os: 'macos-10.15' + arch: 'x64' + - platform: 'win32' + os: 'windows-2019' + arch: 'x64' + - platform: 'win32' + os: 'windows-2019' + arch: 'x86' + runs-on: ${{ matrix.os }} + env: + ARTIFACT_NAME: python-${{ github.event.inputs.VERSION }}-${{ matrix.platform }}-${{ matrix.arch }} + steps: + + - name: Check out repository code + uses: actions/checkout@v2 + with: + submodules: true + + - name: Build Python ${{ env.VERSION }} + shell: pwsh + run: | + ./builders/build-python.ps1 -Version $env:VERSION ` + -Platform ${{ matrix.platform }} -Architecture ${{ matrix.arch }} + + - name: Publish artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ env.ARTIFACT_NAME }} + path: ${{ runner.temp }}/artifact + + test_python: + needs: build_python + strategy: + fail-fast: false + matrix: + include: + - platform: 'linux-18.04' + os: 'ubuntu-18.04' + arch: 'x64' + - platform: 'linux-20.04' + os: 'ubuntu-20.04' + arch: 'x64' + - platform: 'darwin' + os: 'macos-10.15' + arch: 'x64' + - platform: 'win32' + os: 'windows-2019' + arch: 'x64' + - platform: 'win32' + os: 'windows-2019' + arch: 'x86' + runs-on: ${{ matrix.os }} + env: + ARTIFACT_NAME: python-${{ github.event.inputs.VERSION }}-${{ matrix.platform }}-${{ matrix.arch }} + steps: + + - name: Check out repository code + uses: actions/checkout@v2 + with: + submodules: true + + - name: Fully cleanup the toolcache directory before testing + run: ./helpers/clean-toolcache.ps1 -ToolName "Python" + + - name: Download artifact + uses: actions/download-artifact@v2 + with: + path: ${{ runner.temp }} + + - name: Extract files + run: | + if ('${{ matrix.platform }}' -eq 'win32') { + $artifactName = "${{ env.ARTIFACT_NAME }}.zip" + 7z.exe x "$artifactName" -y | Out-Null + } else { + $artifactName = "${{ env.ARTIFACT_NAME }}.tar.gz" + tar -xzf $artifactName + } + working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }} + + - name: Apply build artifact to the local machine + run: | + if ('${{ matrix.platform }}' -eq 'win32') { powershell ./setup.ps1 } else { sh ./setup.sh } + if ('${{ matrix.platform }}' -ne 'win32') { + cp ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}/build_output.txt ${{ runner.temp }} + } + + working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }} + + - name: Setup Python ${{ env.VERSION }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.VERSION }} + + - name: Verbose sysconfig dump + run: | + if ('${{ matrix.platform }}' -ne 'win32') { python ./sources/python-config-output.py } + working-directory: ${{ github.workspace }}/tests + + - name: Verbose python binary links + run: | + $pythonLocation = which python + if ('${{ matrix.platform }}' -ne 'win32') { + if ('${{ matrix.platform }}' -eq 'darwin') { otool -L $pythonLocation } else { ldd $pythonLocation } + } + + - name: Run tests + shell: pwsh + run: | + Install-Module Pester -Force -Scope CurrentUser -RequiredVersion 4.10.1 + Import-Module Pester + $pesterParams = @{ + Path="./python-tests.ps1"; + Parameters=@{ + Version="${{ env.VERSION }}"; + Platform="${{ matrix.platform }}"; + } + } + $Result = Invoke-Pester -PassThru -Script $pesterParams -OutputFile "test_results.xml" -OutputFormat NUnitXml + if ($Result.FailedCount -gt 0) { + $host.SetShouldExit($Result.FailedCount) + exit $Result.FailedCount + } + + working-directory: ${{ github.workspace }}/tests + + publish_release: + name: Publish release + if: github.event_name == 'workflow_dispatch' && github.event.inputs.PUBLISH_RELEASES == 'true' + needs: test_python + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v2 + + - name: Publish Release ${{ env.VERSION }} + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ env.VERSION }}-${{ github.run_id }} + release_name: ${{ env.VERSION }} + body: | + Python ${{ env.VERSION }} + + - name: Upload release assets + uses: actions/github-script@v2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + for (let artifactDir of fs.readdirSync('.')) { + let artifactName = fs.readdirSync(`${artifactDir}`)[0]; + console.log(`Upload ${artifactName} asset`); + github.repos.uploadReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: ${{ steps.create_release.outputs.id }}, + name: artifactName, + data: fs.readFileSync(`./${artifactDir}/${artifactName}`) + }); + } + + trigger_pr: + name: Trigger "Create Pull Request" workflow + needs: publish_release + runs-on: ubuntu-latest + steps: + - name: Trigger "Create Pull Request" workflow + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.PERSONAL_TOKEN }} + script: | + github.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'create-pr.yml', + ref: 'main' + }); + diff --git a/builders/python-builder.psm1 b/builders/python-builder.psm1 index bb44e56..5c1922e 100644 --- a/builders/python-builder.psm1 +++ b/builders/python-builder.psm1 @@ -41,10 +41,13 @@ class PythonBuilder { PythonBuilder ([semver] $version, [string] $architecture, [string] $platform) { $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers" - $this.HostedToolcacheLocation = $env:AGENT_TOOLSDIRECTORY - $this.TempFolderLocation = $env:BUILD_SOURCESDIRECTORY - $this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY - $this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY + New-Item -Force -Type Directory (Join-Path $env:RUNNER_TEMP "artifact") + New-Item -Force -Type Directory (Join-Path $env:RUNNER_TEMP "work") + + $this.HostedToolcacheLocation = $env:RUNNER_TOOL_CACHE + $this.TempFolderLocation = $env:RUNNER_TEMP + $this.WorkFolderLocation = Join-Path $env:RUNNER_TEMP "work" + $this.ArtifactFolderLocation = Join-Path $env:RUNNER_TEMP "artifact" $this.Version = $version $this.Architecture = $architecture diff --git a/tests/python-tests.ps1 b/tests/python-tests.ps1 index 7ffd59d..d18c7be 100644 --- a/tests/python-tests.ps1 +++ b/tests/python-tests.ps1 @@ -39,7 +39,7 @@ Describe "Tests" { "python --version" | Should -ReturnZeroExitCode $pythonLocation = (Get-Command "python").Path $pythonLocation | Should -Not -BeNullOrEmpty - $expectedPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Python" + $expectedPath = Join-Path -Path $env:RUNNER_TOOL_CACHE -ChildPath "Python" $pythonLocation.startsWith($expectedPath) | Should -BeTrue } @@ -61,7 +61,7 @@ Describe "Tests" { if (IsNixPlatform $Platform) { It "Check for failed modules in build_output" { - $buildOutputLocation = Join-Path $env:BUILD_BINARIESDIRECTORY "build_output.txt" + $buildOutputLocation = Join-Path $env:RUNNER_TEMP "build_output.txt" Analyze-MissingModules $buildOutputLocation | Should -Be 0 }