From c641695f6a07526c18f10e374e503e649fef9427 Mon Sep 17 00:00:00 2001 From: Maxim Lobanov Date: Wed, 29 Apr 2020 10:57:27 +0300 Subject: [PATCH] mplement script to build and prepare packages for Python --- .gitignore | 10 + CODE_OF_CONDUCT.md | 76 ++++++ CONTRIBUTING.md | 52 ++++ LICENSE | 21 ++ README.md | 16 ++ SECURITY.md | 3 + azure-pipelines/build-python-packages.yml | 99 ++++++++ azure-pipelines/run-ci-builds.yml | 33 +++ azure-pipelines/templates/build-job.yml | 22 ++ azure-pipelines/templates/test-job.yml | 80 +++++++ builders/build-python.ps1 | 75 ++++++ builders/macos-python-builder.psm1 | 58 +++++ builders/nix-python-builder.psm1 | 162 +++++++++++++ builders/python-builder.psm1 | 97 ++++++++ builders/ubuntu-python-builder.psm1 | 85 +++++++ builders/win-python-builder.psm1 | 141 +++++++++++ helpers/azure-devops/azure-devops-api.ps1 | 89 +++++++ helpers/azure-devops/build-info.ps1 | 44 ++++ helpers/azure-devops/run-ci-builds.ps1 | 94 ++++++++ helpers/common-helpers.psm1 | 80 +++++++ helpers/generate-versions-manifest.ps1 | 158 +++++++++++++ helpers/github/create-pull-request.ps1 | 106 +++++++++ helpers/github/git.psm1 | 81 +++++++ helpers/github/github-api.psm1 | 109 +++++++++ helpers/nix-helpers.psm1 | 50 ++++ helpers/pester-extensions.psm1 | 33 +++ helpers/win-helpers.psm1 | 18 ++ installers/map_python_platforms.json | 29 +++ installers/nix-setup-template.sh | 50 ++++ installers/win-setup-template.ps1 | 138 +++++++++++ tests/clean-toolcache.ps1 | 11 + tests/python-tests.ps1 | 72 ++++++ tests/sources/psutil-install-test.sh | 5 + tests/sources/python-config-output.py | 6 + tests/sources/python-config-test.py | 68 ++++++ tests/sources/python-modules.py | 276 ++++++++++++++++++++++ tests/sources/simple-test.py | 11 + 37 files changed, 2558 insertions(+) create mode 100644 .gitignore create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 azure-pipelines/build-python-packages.yml create mode 100644 azure-pipelines/run-ci-builds.yml create mode 100644 azure-pipelines/templates/build-job.yml create mode 100644 azure-pipelines/templates/test-job.yml create mode 100644 builders/build-python.ps1 create mode 100644 builders/macos-python-builder.psm1 create mode 100644 builders/nix-python-builder.psm1 create mode 100644 builders/python-builder.psm1 create mode 100644 builders/ubuntu-python-builder.psm1 create mode 100644 builders/win-python-builder.psm1 create mode 100644 helpers/azure-devops/azure-devops-api.ps1 create mode 100644 helpers/azure-devops/build-info.ps1 create mode 100644 helpers/azure-devops/run-ci-builds.ps1 create mode 100644 helpers/common-helpers.psm1 create mode 100644 helpers/generate-versions-manifest.ps1 create mode 100644 helpers/github/create-pull-request.ps1 create mode 100644 helpers/github/git.psm1 create mode 100644 helpers/github/github-api.psm1 create mode 100644 helpers/nix-helpers.psm1 create mode 100644 helpers/pester-extensions.psm1 create mode 100644 helpers/win-helpers.psm1 create mode 100644 installers/map_python_platforms.json create mode 100644 installers/nix-setup-template.sh create mode 100644 installers/win-setup-template.ps1 create mode 100644 tests/clean-toolcache.ps1 create mode 100644 tests/python-tests.ps1 create mode 100644 tests/sources/psutil-install-test.sh create mode 100644 tests/sources/python-config-output.py create mode 100644 tests/sources/python-config-test.py create mode 100644 tests/sources/python-modules.py create mode 100644 tests/sources/simple-test.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d720a53 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Visual studio / VS Code cache +.vs/ +.vscode/ + +# Desktop Service Store +.DS_Store \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..517657b --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at opensource@github.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..73fe6e3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,52 @@ +## Contributing + +[fork]: https://github.com/actions/python-versions/fork +[pr]: https://github.com/actions/python-versions/compare +[code-of-conduct]: CODE_OF_CONDUCT.md + +Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great. + +Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [MIT](LICENSE.md). + +Please note that this project is released with a [Contributor Code of Conduct][code-of-conduct]. By participating in this project you agree to abide by its terms. + +## Submitting a pull request + +1. [Fork][fork] and clone the repository +1. Create a new branch: `git checkout -b my-branch-name` +1. Make your changes +1. Push to your fork and [submit a pull request][pr] +1. Make sure that checks in your pull request are green + +Here are a few things you can do that will increase the likelihood of your pull request being accepted: + +- Please include a summary of the change and which issue is fixed. Also include relevant motivation and context. +- Follow the style guide for [PowerShell](https://github.com/PoshCode/PowerShellPracticeAndStyle). +- Write [good commit messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). + +## Code structure + +### Directory structure +``` +TOOLCACHE-PYTHON-GENERATION +├── azure-pipelines/ +| └──templates/ +├── builders/ +├── helpers/ +├── installers/ +└── tests/ + └──sources/ +``` +- `azure-pipelines*` - contains global YAML definitions for build pipelines. Reusable templates for specific jobs are located in `templates` subfolder. +- `builders` - contains Python builder classes and functions. +- `helpers` - contains global helper functions and functions. +- `installers` - contains installation script templates. +- `tests` - contains test scripts. Required tests sources are located in `sources` subfolder. + +\* _We use Azure Pipelines because there are a few features that Actions is still missing, we'll move to Actions as soon as possible_. + +## Resources + +- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) +- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) +- [GitHub Help](https://help.github.com) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f42f5ab --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 GitHub + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e00e77c --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# Python for Actions +This repository contains the code and scripts that we use to build Python packages used in [virtual-environments](https://github.com/actions/virtual-environments) and accessible through the [setup-python](https://github.com/actions/setup-python) Action. +File [versions-manifest.json](./versions-manifest.json) contains the list of available and released versions. + +> Caution: this is prepared for and only permitted for use by actions `virtual-environments` and `setup-python` action. + +**Status**: Currently under development and in use for beta and preview actions. This repo is undergoing rapid changes. + +Some versions are pre-installed on [virtual-environments](https://github.com/actions/virtual-environments) images. +More versions will (soon!) be available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action. + +## Adding new versions +We are trying to build and release new versions of Python as soon as they are released. Please open an issue if any versions are missing. + +## Contribution +Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..f0b196f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +If you discover a security issue in this repo, please submit it through the [GitHub Security Bug Bounty](https://hackerone.com/github) + +Thanks for helping make GitHub Actions safe for everyone. diff --git a/azure-pipelines/build-python-packages.yml b/azure-pipelines/build-python-packages.yml new file mode 100644 index 0000000..eb75aeb --- /dev/null +++ b/azure-pipelines/build-python-packages.yml @@ -0,0 +1,99 @@ +name: $(date:yyyyMMdd)$(rev:.r)-Python-$(VERSION) +trigger: none +pr: none + +stages: +- stage: Build_Python_MacOS + dependsOn: [] + variables: + VmImage: 'macOS-10.14' + Platform: macos-1014 + Architecture: x64 + jobs: + - template: /azure-pipelines/templates/build-job.yml + +- stage: Test_Python_MacOS + condition: succeeded() + dependsOn: Build_Python_MacOS + variables: + VmImage: 'macOS-10.14' + Platform: macos-1014 + Architecture: x64 + jobs: + - template: /azure-pipelines/templates/test-job.yml + +- stage: Build_Python_Ubuntu_1604 + dependsOn: [] + variables: + VmImage: 'ubuntu-16.04' + Platform: ubuntu-1604 + Architecture: x64 + jobs: + - template: /azure-pipelines/templates/build-job.yml + +- stage: Test_Python_Ubuntu_1604 + condition: succeeded() + dependsOn: Build_Python_Ubuntu_1604 + variables: + VmImage: 'ubuntu-16.04' + Platform: ubuntu-1604 + Architecture: x64 + jobs: + - template: /azure-pipelines/templates/test-job.yml + +- stage: Build_Python_Ubuntu_1804 + dependsOn: [] + variables: + VmImage: 'ubuntu-18.04' + Platform: ubuntu-1804 + Architecture: x64 + jobs: + - template: /azure-pipelines/templates/build-job.yml + +- stage: Test_Python_Ubuntu_1804 + condition: succeeded() + dependsOn: Build_Python_Ubuntu_1804 + variables: + VmImage: 'ubuntu-18.04' + Platform: ubuntu-1804 + Architecture: x64 + jobs: + - template: /azure-pipelines/templates/test-job.yml + +- stage: Build_Python_X64_Windows + dependsOn: [] + variables: + VmImage: 'vs2017-win2016' + Platform: windows-2016 + Architecture: x64 + jobs: + - template: /azure-pipelines/templates/build-job.yml + +- stage: Test_Python_x64_Windows + condition: succeeded() + dependsOn: Build_Python_X64_Windows + variables: + VmImage: 'vs2017-win2016' + Platform: windows-2016 + Architecture: x64 + jobs: + - template: /azure-pipelines/templates/test-job.yml + +- stage: Build_Python_x86_Windows + dependsOn: [] + variables: + VmImage: 'vs2017-win2016' + Platform: windows-2016 + Architecture: x86 + jobs: + - template: /azure-pipelines/templates/build-job.yml + +- stage: Test_Python_x86_Windows + condition: succeeded() + dependsOn: Build_Python_x86_Windows + variables: + VmImage: 'vs2017-win2016' + Platform: windows-2016 + Architecture: x86 + jobs: + - template: /azure-pipelines/templates/test-job.yml diff --git a/azure-pipelines/run-ci-builds.yml b/azure-pipelines/run-ci-builds.yml new file mode 100644 index 0000000..d7ae36b --- /dev/null +++ b/azure-pipelines/run-ci-builds.yml @@ -0,0 +1,33 @@ +trigger: none +pr: + autoCancel: true + branches: + include: + - master + paths: + exclude: + - versions-manifest.json + +jobs: +- job: Run_Builds + pool: + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + steps: + - checkout: self + submodules: true + + - task: PowerShell@2 + displayName: 'Run build' + inputs: + targetType: filePath + filePath: './helpers/azure-devops/run-ci-builds.ps1 ' + arguments: | + -TeamFoundationCollectionUri $(System.TeamFoundationCollectionUri) ` + -AzureDevOpsProjectName $(System.TeamProject) ` + -AzureDevOpsAccessToken $(System.AccessToken) ` + -SourceBranch $(Build.SourceBranch) ` + -SourceVersion $(Build.SourceVersion) ` + -ToolVersions "$(PYTHON_VERSIONS)" ` + -DefinitionId $(DEFINITION_ID) \ No newline at end of file diff --git a/azure-pipelines/templates/build-job.yml b/azure-pipelines/templates/build-job.yml new file mode 100644 index 0000000..31b41e7 --- /dev/null +++ b/azure-pipelines/templates/build-job.yml @@ -0,0 +1,22 @@ +jobs: +- job: Build_Python + timeoutInMinutes: 90 + pool: + name: Azure Pipelines + vmImage: $(VmImage) + steps: + - checkout: self + submodules: true + + - task: PowerShell@2 + displayName: 'Build Python $(VERSION)' + inputs: + targetType: filePath + filePath: './builders/build-python.ps1' + arguments: '-Version $(VERSION) -Platform $(Platform) -Architecture $(Architecture)' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: Python $(VERSION)' + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)' + artifactName: 'python-$(VERSION)-$(Platform)-$(Architecture)' \ No newline at end of file diff --git a/azure-pipelines/templates/test-job.yml b/azure-pipelines/templates/test-job.yml new file mode 100644 index 0000000..f3a1074 --- /dev/null +++ b/azure-pipelines/templates/test-job.yml @@ -0,0 +1,80 @@ +jobs: +- job: Test_Python + pool: + name: Azure Pipelines + vmImage: $(VmImage) + steps: + - checkout: self + submodules: true + + - task: DownloadPipelineArtifact@2 + inputs: + source: 'current' + artifact: 'python-$(VERSION)-$(Platform)-$(Architecture)' + path: $(Build.BinariesDirectory) + + - task: ExtractFiles@1 + inputs: + archiveFilePatterns: '$(Build.BinariesDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).*' + destinationFolder: $(Build.BinariesDirectory) + cleanDestinationFolder: false + + - task: PowerShell@2 + displayName: 'Apply build artifact to the local machines' + inputs: + TargetType: inline + script: | + if ($env:PLATFORM -match 'windows') { powershell ./setup.ps1 } else { sh ./setup.sh } + workingDirectory: '$(Build.BinariesDirectory)' + + - task: UsePythonVersion@0 + displayName: 'Use Python $(VERSION)' + inputs: + versionSpec: '$(VERSION)' + architecture: '$(Architecture)' + + - task: PowerShell@2 + displayName: 'Verbose sysconfig dump' + inputs: + TargetType: inline + script: | + Invoke-Expression "python ./sources/python-config-output.py" + workingDirectory: '$(Build.SourcesDirectory)/tests' + condition: ne(variables['Platform'], 'windows-2016') + + - task: PowerShell@2 + displayName: 'Verbose python binary links' + inputs: + TargetType: inline + script: | + $pythonLocation = which python + if ($env:PLATFORM -match 'macos') { otool -L $pythonLocation } else { ldd $pythonLocation } + workingDirectory: '$(Build.BinariesDirectory)' + condition: ne(variables['Platform'], 'windows-2016') + + - task: PowerShell@2 + displayName: 'Run tests' + inputs: + TargetType: inline + script: | + Install-Module Pester -Force -Scope CurrentUser + Import-Module Pester + $pesterParams = @{ + Path="./python-tests.ps1"; + Parameters=@{ + Version="$(VERSION)"; + Platform="$(Platform)"; + } + } + Invoke-Pester -Script $pesterParams -OutputFile "test_results.xml" -OutputFormat NUnitXml + workingDirectory: '$(Build.SourcesDirectory)/tests' + + - task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testResultsFiles: '*.xml' + testResultsFormat: NUnit + searchFolder: 'tests' + failTaskOnFailedTests: true + testRunTitle: "$(TestRunTitle)" + condition: always() diff --git a/builders/build-python.ps1 b/builders/build-python.ps1 new file mode 100644 index 0000000..ccd0d09 --- /dev/null +++ b/builders/build-python.ps1 @@ -0,0 +1,75 @@ +using module "./builders/win-python-builder.psm1" +using module "./builders/ubuntu-python-builder.psm1" +using module "./builders/macos-python-builder.psm1" + +<# +.SYNOPSIS +Generate Python artifact. + +.DESCRIPTION +Main script that creates instance of PythonBuilder and builds of Python using specified parameters. + +.PARAMETER Version +Required parameter. The version with which Python will be built. + +.PARAMETER Architecture +Optional parameter. The architecture with which Python will be built. Using x64 by default. + +.PARAMETER Platform +Required parameter. The platform for which Python will be built. + +#> + +param( + [Parameter (Mandatory=$true)][Version] $Version, + [Parameter (Mandatory=$true)][string] $Platform, + [string] $Architecture = "x64" +) + +Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "common-helpers.psm1") -DisableNameChecking +Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "nix-helpers.psm1") -DisableNameChecking +Import-Module (Join-Path $PSScriptRoot "../helpers" | Join-Path -ChildPath "win-helpers.psm1") -DisableNameChecking + +function Get-PythonBuilder { + <# + .SYNOPSIS + Wrapper for class constructor to simplify importing PythonBuilder. + + .DESCRIPTION + Create instance of PythonBuilder with specified parameters. + + .PARAMETER Version + The version with which Python will be built. + + .PARAMETER Architecture + The architecture with which Python will be built. + + .PARAMETER Platform + The platform for which Python will be built. + + #> + + param ( + [version] $Version, + [string] $Architecture, + [string] $Platform + ) + + $Platform = $Platform.ToLower() + if ($Platform -match 'windows') { + $builder = [WinPythonBuilder]::New($Version, $Architecture, $Platform) + } elseif ($Platform -match 'ubuntu') { + $builder = [UbuntuPythonBuilder]::New($Version, $Architecture, $Platform) + } elseif ($Platform -match 'macos') { + $builder = [macOSPythonBuilder]::New($Version, $Architecture, $Platform) + } else { + Write-Host "##vso[task.logissue type=error;] Invalid platform: $Platform" + exit 1 + } + + return $builder +} + +### Create Python builder instance, and build artifact +$Builder = Get-PythonBuilder -Version $Version -Architecture $Architecture -Platform $Platform +$Builder.Build() diff --git a/builders/macos-python-builder.psm1 b/builders/macos-python-builder.psm1 new file mode 100644 index 0000000..9c6c9f4 --- /dev/null +++ b/builders/macos-python-builder.psm1 @@ -0,0 +1,58 @@ +using module "./builders/nix-python-builder.psm1" + +class macOSPythonBuilder : NixPythonBuilder { + <# + .SYNOPSIS + MacOS Python builder class. + + .DESCRIPTION + Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder. + + .PARAMETER platform + The full name of platform for which Python should be built. + + .PARAMETER version + The version of Python that should be built. + + #> + + macOSPythonBuilder( + [version] $version, + [string] $architecture, + [string] $platform + ) : Base($version, $architecture, $platform) { } + + [void] Configure() { + <# + .SYNOPSIS + Execute configure script with required parameters. + #> + + $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation() + $configureString = "./configure --prefix=$pythonBinariesLocation --enable-optimizations --enable-shared --with-lto" + + ### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL. + ### Solution is to install these libraries from a third-party package manager, + ### and then add the appropriate paths for the header and library files to configure command. + ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies) + if ($this.Version -lt "3.7.0") { + $env:LDFLAGS="-L$(brew --prefix openssl)/lib" + $env:CFLAGS="-I$(brew --prefix openssl)/include" + } else { + $configureString += " --with-openssl=/usr/local/opt/openssl" + } + + Execute-Command -Command $configureString + } + + [void] PrepareEnvironment() { + <# + .SYNOPSIS + Prepare system environment by installing dependencies and required packages. + #> + + ### reinstall header files to Avoid issue with X11 headers on Mojave + $pkgName = "/Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg" + Execute-Command -Command "sudo installer -pkg $pkgName -target /" + } +} diff --git a/builders/nix-python-builder.psm1 b/builders/nix-python-builder.psm1 new file mode 100644 index 0000000..ceb0cb3 --- /dev/null +++ b/builders/nix-python-builder.psm1 @@ -0,0 +1,162 @@ +using module "./builders/python-builder.psm1" + +class NixPythonBuilder : PythonBuilder { + <# + .SYNOPSIS + Base Python builder class for *Nix systems. + + .DESCRIPTION + Contains methods that required to build Python artifact for *nix systems. Inherited from base PythonBuilder class. + + .PARAMETER version + The version of Python that should be built. + + .PARAMETER Platform + The type of platform for which Python should be built. + + .PARAMETER PlatformVersion + The version of platform for which Python should be built. + + .PARAMETER InstallationTemplateName + The name of template that will be used to create installation script for generated Python artifact. + + .PARAMETER InstallationScriptName + The name of installation script that will be generated for Python artifact. + + .PARAMETER OutputArtifactName + The name of archive with Python binaries that will be generated as part of Python artifact. + + #> + + [string] $InstallationTemplateName + [string] $InstallationScriptName + [string] $OutputArtifactName + + NixPythonBuilder( + [version] $version, + [string] $architecture, + [string] $platform + ) : Base($version, $architecture, $platform) { + $this.InstallationTemplateName = "nix-setup-template.sh" + $this.InstallationScriptName = "setup.sh" + + $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.tar.gz" + } + + [uri] GetSourceUri() { + <# + .SYNOPSIS + Get base Python URI and return complete URI for Python sources. + #> + + $base = $this.GetBaseUri() + + return "${base}/$($this.Version)/Python-$($this.Version).tgz" + } + + [string] GetPythonBinary() { + <# + .SYNOPSIS + Return name of Python binary. + #> + + if ($this.Version.Major -eq 2) { $pythonBinary = "python" } else { $pythonBinary = "python3" } + return $pythonBinary + } + + [string] Download() { + <# + .SYNOPSIS + Download Python sources and extract them at temporary work folder. Returns expanded archive location path. + #> + + $sourceUri = $this.GetSourceUri() + Write-Host "Sources URI: $sourceUri" + + $archiveFilepath = Download-File -Uri $sourceUri -OutputFolder $this.WorkFolderLocation + $expandedSourceLocation = Join-Path -Path $this.TempFolderLocation -ChildPath "SourceCode" + New-Item -Path $expandedSourceLocation -ItemType Directory + + Extract-TarArchive -ArchivePath $archiveFilepath -OutputDirectory $expandedSourceLocation + Write-Debug "Done; Sources location: $expandedSourceLocation" + + return $expandedSourceLocation + } + + [void] CreateInstallationScript() { + <# + .SYNOPSIS + Create Python artifact installation script based on template specified in InstallationTemplateName property. + #> + + $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File + $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName + + $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw + $installationTemplateContent = $installationTemplateContent -f $this.Version.Major, $this.Version.Minor, $this.Version.Build + $installationTemplateContent | Out-File -FilePath $installationScriptLocation + + Write-Debug "Done; Installation script location: $installationScriptLocation)" + } + + [void] Make() { + <# + .SYNOPSIS + Executes "make" and "make install" commands for configured build sources. Make output will be writen in build_output.txt located in artifact location folder. + #> + + Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)" + $buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File + + Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue + Execute-Command -Command "make install" -ErrorAction Continue + + Write-Debug "Done; Make log location: $buildOutputLocation" + } + + [void] CopyBuildResults() { + $buildFolder = $this.GetFullPythonToolcacheLocation() + Move-Item -Path "$buildFolder/*" -Destination $this.WorkFolderLocation + } + + [void] ArchiveArtifact() { + $OutputPath = Join-Path $this.ArtifactFolderLocation $this.OutputArtifactName + Create-TarArchive -SourceFolder $this.WorkFolderLocation -ArchivePath $OutputPath + } + + [void] Build() { + <# + .SYNOPSIS + Build Python artifact from sources. + #> + + Write-Host "Prepare Python Hostedtoolcache location..." + $this.PreparePythonToolcacheLocation() + + Write-Host "Prepare system environment..." + $this.PrepareEnvironment() + + Write-Host "Download Python $($this.Version)[$($this.Architecture)] sources..." + $sourcesLocation = $this.Download() + + Push-Location -Path $sourcesLocation + Write-Host "Configure for $($this.Platform)..." + $this.Configure() + + Write-Host "Make for $($this.Platform)..." + $this.Make() + Pop-Location + + Write-Host "Generate structure dump" + New-ToolStructureDump -ToolPath $this.GetFullPythonToolcacheLocation() -OutputFolder $this.WorkFolderLocation + + Write-Host "Copying build results to destination location" + $this.CopyBuildResults() + + Write-Host "Create installation script..." + $this.CreateInstallationScript() + + Write-Host "Archive artifact..." + $this.ArchiveArtifact() + } +} diff --git a/builders/python-builder.psm1 b/builders/python-builder.psm1 new file mode 100644 index 0000000..d19644a --- /dev/null +++ b/builders/python-builder.psm1 @@ -0,0 +1,97 @@ +class PythonBuilder { + <# + .SYNOPSIS + Base Python builder class. + + .DESCRIPTION + Base Python builder class that contains general builder methods. + + .PARAMETER Version + The version of Python that should be built. + + .PARAMETER Architecture + The architecture with which Python should be built. + + .PARAMETER HostedToolcacheLocation + The location of hostedtoolcache artifacts. Using system AGENT_TOOLSDIRECTORY variable value. + + .PARAMETER TempFolderLocation + The location of temporary files that will be used during Python generation. Using system TEMP directory. + + .PARAMETER WorkFolderLocation + The location of generated Python artifact. Using system environment BUILD_STAGINGDIRECTORY variable value. + + .PARAMETER ArtifactFolderLocation + The location of generated Python artifact. Using system environment BUILD_BINARIESDIRECTORY variable value. + + .PARAMETER InstallationTemplatesLocation + The location of installation script template. Using "installers" folder from current repository. + + #> + + [version] $Version + [string] $Architecture + [string] $Platform + [string] $HostedToolcacheLocation + [string] $TempFolderLocation + [string] $WorkFolderLocation + [string] $ArtifactFolderLocation + [string] $InstallationTemplatesLocation + + PythonBuilder ([version] $version, [string] $architecture, [string] $platform) { + $this.Version = $version + $this.Architecture = $architecture + $this.Platform = $platform + + $this.HostedToolcacheLocation = $env:AGENT_TOOLSDIRECTORY + $this.TempFolderLocation = $env:BUILD_SOURCESDIRECTORY + $this.WorkFolderLocation = $env:BUILD_BINARIESDIRECTORY + $this.ArtifactFolderLocation = $env:BUILD_STAGINGDIRECTORY + + $this.InstallationTemplatesLocation = Join-Path -Path $PSScriptRoot -ChildPath "../installers" + } + + [uri] GetBaseUri() { + <# + .SYNOPSIS + Return base URI for Python build sources. + #> + + return "https://www.python.org/ftp/python" + } + + [string] GetPythonToolcacheLocation() { + <# + .SYNOPSIS + Return path to Python hostedtoolcache folder. + #> + + return "$($this.HostedToolcacheLocation)/Python" + } + + [string] GetFullPythonToolcacheLocation() { + <# + .SYNOPSIS + Return full path to hostedtoolcache Python folder. + #> + + $pythonToolcacheLocation = $this.GetPythonToolcacheLocation() + return "$pythonToolcacheLocation/$($this.Version)/$($this.Architecture)" + } + + [void] PreparePythonToolcacheLocation() { + <# + .SYNOPSIS + Prepare system hostedtoolcache folder for new Python version. + #> + $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation() + + if (Test-Path $pythonBinariesLocation) { + Write-Host "Purge $pythonBinariesLocation folder..." + Remove-Item $pythonBinariesLocation -Recurse -Force + } else { + Write-Host "Create $pythonBinariesLocation folder..." + New-Item -ItemType Directory -Path $pythonBinariesLocation + } + } +} diff --git a/builders/ubuntu-python-builder.psm1 b/builders/ubuntu-python-builder.psm1 new file mode 100644 index 0000000..5c080d7 --- /dev/null +++ b/builders/ubuntu-python-builder.psm1 @@ -0,0 +1,85 @@ +using module "./builders/nix-python-builder.psm1" + +class UbuntuPythonBuilder : NixPythonBuilder { + <# + .SYNOPSIS + Ubuntu Python builder class. + + .DESCRIPTION + Contains methods that required to build Ubuntu Python artifact from sources. Inherited from base NixPythonBuilder. + + .PARAMETER platform + The full name of platform for which Python should be built. + + .PARAMETER version + The version of Python that should be built. + + #> + + UbuntuPythonBuilder( + [version] $version, + [string] $architecture, + [string] $platform + ) : Base($version, $architecture, $platform) { } + + [void] Configure() { + <# + .SYNOPSIS + Execute configure script with required parameters. + #> + + $pythonBinariesLocation = $this.GetFullPythonToolcacheLocation() + + ### To build Python with SO we must pass full path to lib folder to the linker + $env:LDFLAGS="-Wl,--rpath=${pythonBinariesLocation}/lib" + $configureString = "./configure --prefix=$pythonBinariesLocation --enable-shared --enable-optimizations" + + if ($this.Version -lt "3.0.0") { + ### Compile with ucs4 for Python 2.x. On 3.x, ucs4 is enabled by default + $configureString += " --enable-unicode=ucs4" + } + + Execute-Command -Command $configureString + } + + [void] PrepareEnvironment() { + <# + .SYNOPSIS + Prepare system environment by installing dependencies and required packages. + #> + + if (($this.Version -gt "3.0.0") -and ($this.Version -lt "3.5.3")) { + Write-Host "Python3 versions lower than 3.5.3 are not supported" + exit 1 + } + + ### Compile with tkinter support + if ($this.Version -gt "3.0.0") { + $tkinterInstallString = "sudo apt-get install -y --allow-downgrades python3-tk tk-dev" + } else { + $tkinterInstallString = "sudo apt install -y python-tk tk-dev" + } + Execute-Command -Command $tkinterInstallString + + ### Install dependent packages + @( + "make", + "build-essential", + "libssl-dev", + "zlib1g-dev", + "libbz2-dev", + "libsqlite3-dev", + "libncursesw5-dev", + "libreadline-dev", + "libgdbm-dev" + ) | ForEach-Object { + Execute-Command -Command "sudo apt install -y $_" + } + + if ($this.Platform -ne "ubuntu-1604") { + ### On Ubuntu-1804, libgdbm-compat-dev has older modules that are no longer in libgdbm-dev + Execute-Command -Command "sudo apt install -y libgdbm-compat-dev" + } + + } +} diff --git a/builders/win-python-builder.psm1 b/builders/win-python-builder.psm1 new file mode 100644 index 0000000..1350e88 --- /dev/null +++ b/builders/win-python-builder.psm1 @@ -0,0 +1,141 @@ +using module "./builders/python-builder.psm1" + +class WinPythonBuilder : PythonBuilder { + <# + .SYNOPSIS + Base Python builder class for Windows systems. + + .DESCRIPTION + Contains methods required for build Windows Python artifact. Inherited from base PythonBuilder class. + + .PARAMETER version + The version of Python that should be built. + + .PARAMETER architecture + The architecture with which Python should be built. + + .PARAMETER InstallationTemplateName + The name of installation script template that will be used in generated artifact. + + .PARAMETER InstallationScriptName + The name of generated installation script. + + #> + + [string] $InstallationTemplateName + [string] $InstallationScriptName + [string] $OutputArtifactName + + WinPythonBuilder( + [version] $version, + [string] $architecture, + [string] $platform + ) : Base($version, $architecture, $platform) { + $this.InstallationTemplateName = "win-setup-template.ps1" + $this.InstallationScriptName = "setup.ps1" + $this.OutputArtifactName = "python-$Version-$Platform-$Architecture.zip" + } + + [string] GetPythonExtension() { + <# + .SYNOPSIS + Return extension for required version of Python executable. + #> + + $extension = if ($this.Version -lt "3.5" -and $this.Version -ge "2.5") { ".msi" } else { ".exe" } + + return $extension + } + + [string] GetArchitectureExtension() { + <# + .SYNOPSIS + Return architecture suffix for Python executable. + #> + + $ArchitectureExtension = "" + if ($this.Architecture -eq "x64") { + if ($this.Version -ge "3.5") { + $ArchitectureExtension = "-amd64" + } else { + $ArchitectureExtension = ".amd64" + } + } + + return $ArchitectureExtension + } + + [uri] GetSourceUri() { + <# + .SYNOPSIS + Get base Python URI and return complete URI for Python installation executable. + #> + + $base = $this.GetBaseUri() + $architecture = $this.GetArchitectureExtension() + $extension = $this.GetPythonExtension() + + $uri = "${base}/$($this.Version)/python-$($this.Version)${architecture}${extension}" + + return $uri + } + + [string] Download() { + <# + .SYNOPSIS + Download Python installation executable into artifact location. + #> + + $sourceUri = $this.GetSourceUri() + + Write-Host "Sources URI: $sourceUri" + $sourcesLocation = Download-File -Uri $sourceUri -OutputFolder $this.WorkFolderLocation + Write-Debug "Done; Sources location: $sourcesLocation" + + return $sourcesLocation + } + + [void] CreateInstallationScript() { + <# + .SYNOPSIS + Create Python artifact installation script based on specified template. + #> + + $sourceUri = $this.GetSourceUri() + $pythonExecName = [IO.path]::GetFileName($sourceUri.AbsoluteUri) + $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath $this.InstallationTemplateName + $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw + $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName -ItemType File + + $variablesToReplace = @{ + "{{__ARCHITECTURE__}}" = $this.Architecture; + "{{__VERSION__}}" = $this.Version; + "{{__PYTHON_EXEC_NAME__}}" = $pythonExecName + } + + $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) } + $installationTemplateContent | Out-File -FilePath $installationScriptLocation + Write-Debug "Done; Installation script location: $installationScriptLocation)" + } + + [void] ArchiveArtifact() { + $OutputPath = Join-Path $this.ArtifactFolderLocation $this.OutputArtifactName + Create-SevenZipArchive -SourceFolder $this.WorkFolderLocation -ArchivePath $OutputPath + } + + [void] Build() { + <# + .SYNOPSIS + Generates Python artifact from downloaded Python installation executable. + #> + + Write-Host "Download Python $($this.Version) [$($this.Architecture)] executable..." + $this.Download() + + Write-Host "Create installation script..." + $this.CreateInstallationScript() + + Write-Host "Archive artifact" + $this.ArchiveArtifact() + } +} diff --git a/helpers/azure-devops/azure-devops-api.ps1 b/helpers/azure-devops/azure-devops-api.ps1 new file mode 100644 index 0000000..dd2feba --- /dev/null +++ b/helpers/azure-devops/azure-devops-api.ps1 @@ -0,0 +1,89 @@ +class AzureDevOpsApi +{ + [string] $BaseUrl + [string] $RepoOwner + [object] $AuthHeader + + AzureDevOpsApi( + [string] $TeamFoundationCollectionUri, + [string] $ProjectName, + [string] $AccessToken + ) { + $this.BaseUrl = $this.BuildBaseUrl($TeamFoundationCollectionUri, $ProjectName) + $this.AuthHeader = $this.BuildAuth($AccessToken) + } + + [object] hidden BuildAuth([string]$AccessToken) { + if ([string]::IsNullOrEmpty($AccessToken)) { + return $null + } + return @{ + Authorization = "Bearer $AccessToken" + } + } + + [string] hidden BuildBaseUrl([string]$TeamFoundationCollectionUri, [string]$ProjectName) { + return "${TeamFoundationCollectionUri}/${ProjectName}/_apis" + } + + [object] QueueBuild([string]$ToolVersion, [string]$SourceBranch, [string]$SourceVersion, [UInt32]$DefinitionId){ + $url = "build/builds" + + # The content of parameters field should be a json string + $buildParameters = @{ VERSION = $ToolVersion } | ConvertTo-Json + + $body = @{ + definition = @{ + id = $DefinitionId + } + sourceBranch = $SourceBranch + sourceVersion = $SourceVersion + parameters = $buildParameters + } | ConvertTo-Json + + return $this.InvokeRestMethod($url, 'POST', $body) + } + + [object] GetBuildInfo([UInt32]$BuildId){ + $url = "build/builds/$BuildId" + + return $this.InvokeRestMethod($url, 'GET', $null) + } + + [string] hidden BuildUrl([string]$Url) { + return "$($this.BaseUrl)/${Url}/?api-version=5.1" + } + + [object] hidden InvokeRestMethod( + [string] $Url, + [string] $Method, + [string] $Body + ) { + $requestUrl = $this.BuildUrl($Url) + $params = @{ + Method = $Method + ContentType = "application/json" + Uri = $requestUrl + Headers = @{} + } + if ($this.AuthHeader) { + $params.Headers += $this.AuthHeader + } + if (![string]::IsNullOrEmpty($body)) { + $params.Body = $Body + } + + return Invoke-RestMethod @params + } + +} + +function Get-AzureDevOpsApi { + param ( + [string] $TeamFoundationCollectionUri, + [string] $ProjectName, + [string] $AccessToken + ) + + return [AzureDevOpsApi]::New($TeamFoundationCollectionUri, $ProjectName, $AccessToken) +} \ No newline at end of file diff --git a/helpers/azure-devops/build-info.ps1 b/helpers/azure-devops/build-info.ps1 new file mode 100644 index 0000000..9622d8c --- /dev/null +++ b/helpers/azure-devops/build-info.ps1 @@ -0,0 +1,44 @@ +Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1") + +class BuildInfo +{ + [AzureDevOpsApi] $AzureDevOpsApi + [String] $Name + [UInt32] $Id + [String] $Status + [String] $Result + [String] $Link + + BuildInfo([AzureDevOpsApi] $AzureDevOpsApi, [object] $Build) + { + $this.AzureDevOpsApi = $AzureDevOpsApi + $this.Id = $Build.id + $this.Name = $Build.buildNumber + $this.Link = $Build._links.web.href + $this.Status = $Build.status + $this.Result = $Build.result + } + + [boolean] IsFinished() { + return ($this.Status -eq "completed") -or ($this.Status -eq "cancelling") + } + + [boolean] IsSuccess() { + return $this.Result -eq "succeeded" + } + + [void] UpdateBuildInfo() { + $buildInfo = $this.AzureDevOpsApi.GetBuildInfo($this.Id) + $this.Status = $buildInfo.status + $this.Result = $buildInfo.result + } +} + +function Get-BuildInfo { + param ( + [AzureDevOpsApi] $AzureDevOpsApi, + [object] $Build + ) + + return [BuildInfo]::New($AzureDevOpsApi, $Build) +} \ No newline at end of file diff --git a/helpers/azure-devops/run-ci-builds.ps1 b/helpers/azure-devops/run-ci-builds.ps1 new file mode 100644 index 0000000..bf11295 --- /dev/null +++ b/helpers/azure-devops/run-ci-builds.ps1 @@ -0,0 +1,94 @@ +param ( + [Parameter(Mandatory)] [string] $TeamFoundationCollectionUri, + [Parameter(Mandatory)] [string] $AzureDevOpsProjectName, + [Parameter(Mandatory)] [string] $AzureDevOpsAccessToken, + [Parameter(Mandatory)] [string] $SourceBranch, + [Parameter(Mandatory)] [string] $ToolVersions, + [Parameter(Mandatory)] [UInt32] $DefinitionId, + [string] $SourceVersion +) + +Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1") +Import-Module (Join-Path $PSScriptRoot "build-info.ps1") + +function Queue-Builds { + param ( + [Parameter(Mandatory)] [AzureDevOpsApi] $AzureDevOpsApi, + [Parameter(Mandatory)] [string] $ToolVersions, + [Parameter(Mandatory)] [string] $SourceBranch, + [Parameter(Mandatory)] [string] $SourceVersion, + [Parameter(Mandatory)] [string] $DefinitionId + ) + + [BuildInfo[]]$queuedBuilds = @() + + $ToolVersions.Split(',') | ForEach-Object { + $version = $_.Trim() + Write-Host "Queue build for $version..." + $queuedBuild = $AzureDevOpsApi.QueueBuild($version, $SourceBranch, $SourceVersion, $DefinitionId) + $buildInfo = Get-BuildInfo -AzureDevOpsApi $AzureDevOpsApi -Build $queuedBuild + Write-Host "Queued build: $($buildInfo.Link)" + $queuedBuilds += $buildInfo + } + + return $queuedBuilds +} + +function Wait-Builds { + param ( + [Parameter(Mandatory)] [BuildInfo[]] $Builds + ) + + $timeoutBetweenRefreshSec = 30 + + do { + # If build is still running - refresh its status + foreach($build in $builds) { + if (!$build.IsFinished()) { + $build.UpdateBuildInfo() + + if ($build.IsFinished()) { + Write-Host "The $($build.Name) build was completed: $($build.Link)" + } + } + } + + $runningBuildsCount = ($builds | Where-Object { !$_.IsFinished() }).Length + + Start-Sleep -Seconds $timeoutBetweenRefreshSec + } while($runningBuildsCount -gt 0) +} + +function Make-BuildsOutput { + param ( + [Parameter(Mandatory)] [BuildInfo[]] $Builds + ) + + Write-Host "Builds info:" + $builds | Format-Table -AutoSize -Property Name,Id,Status,Result,Link | Out-String -Width 10000 + + # Return exit code based on status of builds + $failedBuilds = ($builds | Where-Object { !$_.IsSuccess() }) + if ($failedBuilds.Length -ne 0) { + Write-Host "##vso[task.logissue type=error;]Builds failed" + $failedBuilds | ForEach-Object -Process { Write-Host "##vso[task.logissue type=error;]Name: $($_.Name); Link: $($_.Link)" } + Write-Host "##vso[task.complete result=Failed]" + } else { + Write-host "##[section] All builds have been passed successfully" + } +} + +$azureDevOpsApi = Get-AzureDevOpsApi -TeamFoundationCollectionUri $TeamFoundationCollectionUri ` + -ProjectName $AzureDevOpsProjectName ` + -AccessToken $AzureDevOpsAccessToken + +$queuedBuilds = Queue-Builds -AzureDevOpsApi $azureDevOpsApi ` + -ToolVersions $ToolVersions ` + -SourceBranch $SourceBranch ` + -SourceVersion $SourceVersion ` + -DefinitionId $DefinitionId + +Write-Host "Waiting results of builds ..." +Wait-Builds -Builds $queuedBuilds + +Make-BuildsOutput -Builds $queuedBuilds diff --git a/helpers/common-helpers.psm1 b/helpers/common-helpers.psm1 new file mode 100644 index 0000000..8de58a7 --- /dev/null +++ b/helpers/common-helpers.psm1 @@ -0,0 +1,80 @@ +<# +.SYNOPSIS +The execute command and print all output to the logs +#> +function Execute-Command { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)][string] $Command + ) + + Write-Debug "Execute $Command" + + try { + Invoke-Expression $Command | ForEach-Object { Write-Host $_ } + } + catch { + Write-Host "Error happened during command execution: $Command" + Write-Host "##vso[task.logissue type=error;] $_" + } +} + +<# +.SYNOPSIS +Download file from url and return local path to file +#> +function Download-File { + param( + [Parameter(Mandatory=$true)] + [Uri]$Uri, + [Parameter(Mandatory=$true)] + [String]$OutputFolder + ) + + $targetFilename = [IO.Path]::GetFileName($Uri) + $targetFilepath = Join-Path $OutputFolder $targetFilename + + Write-Debug "Download source from $Uri to $OutFile" + try { + (New-Object System.Net.WebClient).DownloadFile($Uri, $targetFilepath) + return $targetFilepath + } catch { + Write-Host "Error during downloading file from '$Uri'" + "$_" + exit 1 + } +} + +<# +.SYNOPSIS +Generate file that contains the list of all files in particular directory +#> +function New-ToolStructureDump { + param( + [Parameter(Mandatory=$true)] + [String]$ToolPath, + [Parameter(Mandatory=$true)] + [String]$OutputFolder + ) + + $outputFile = Join-Path $OutputFolder "tools_structure.txt" + + $folderContent = Get-ChildItem -Path $ToolPath -Recurse | Sort-Object | Select-Object -Property FullName, Length + $folderContent | ForEach-Object { + $relativePath = $_.FullName.Replace($ToolPath, ""); + return "${relativePath}" + } | Out-File -FilePath $outputFile +} + +<# +.SYNOPSIS +Check if it is macOS / Ubuntu platform +#> +function IsNixPlatform { + param( + [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] + [String]$Platform + ) + + return ($Platform -match "macos") -or ($Platform -match "ubuntu") +} \ No newline at end of file diff --git a/helpers/generate-versions-manifest.ps1 b/helpers/generate-versions-manifest.ps1 new file mode 100644 index 0000000..176360a --- /dev/null +++ b/helpers/generate-versions-manifest.ps1 @@ -0,0 +1,158 @@ +<# +.SYNOPSIS +Generate versions manifest based on repository releases + +.DESCRIPTION +Versions manifest is needed to find the latest assets for particular version of tool +.PARAMETER GitHubRepositoryOwner +Required parameter. The organization which tool repository belongs +.PARAMETER GitHubRepositoryName +Optional parameter. The name of tool repository +.PARAMETER GitHubAccessToken +Required parameter. PAT Token to overcome GitHub API Rate limit +.PARAMETER OutputFile +Required parameter. File "*.json" where generated results will be saved +.PARAMETER PlatformMapFile +Optional parameter. Path to the json file with platform map +Structure example: +{ + "macos-1014": [ + { + "platform": "darwin", + "platform_version": "10.14" + }, ... + ], ... +} +#> + +param ( + [Parameter(Mandatory)] [string] $GitHubRepositoryOwner, + [Parameter(Mandatory)] [string] $GitHubRepositoryName, + [Parameter(Mandatory)] [string] $GitHubAccessToken, + [Parameter(Mandatory)] [string] $OutputFile, + [string] $PlatformMapFile +) + +Import-Module (Join-Path $PSScriptRoot "github/github-api.psm1") + +if ($PlatformMapFile -and (Test-Path $PlatformMapFile)) { + $PlatformMap = Get-Content $PlatformMapFile -Raw | ConvertFrom-Json -AsHashtable +} else { + $PlatformMap = @{} +} + +function Get-FileNameWithoutExtension { + param ( + [Parameter(Mandatory)][string]$Filename + ) + + if ($Filename.EndsWith(".tar.gz")) { + $Filename = [IO.path]::GetFileNameWithoutExtension($Filename) + } + + return [IO.path]::GetFileNameWithoutExtension($Filename) +} + +function New-AssetItem { + param ( + [Parameter(Mandatory)][string]$Filename, + [Parameter(Mandatory)][string]$DownloadUrl, + [Parameter(Mandatory)][string]$Arch, + [Parameter(Mandatory)][string]$Platform, + [string]$PlatformVersion + ) + $asset = New-Object PSObject + + $asset | Add-Member -Name "filename" -Value $Filename -MemberType NoteProperty + $asset | Add-Member -Name "arch" -Value $Arch -MemberType NoteProperty + $asset | Add-Member -Name "platform" -Value $Platform -MemberType NoteProperty + if ($PlatformVersion) { $asset | Add-Member -Name "platform_version" -Value $PlatformVersion -MemberType NoteProperty } + $asset | Add-Member -Name "download_url" -Value $DownloadUrl -MemberType NoteProperty + + return $asset +} + +function Build-AssetsList { + param ( + [AllowEmptyCollection()] + [Parameter(Mandatory)][array]$ReleaseAssets + ) + + + $assets = @() + foreach($releaseAsset in $ReleaseAssets) { + $filename = Get-FileNameWithoutExtension -Filename $releaseAsset.name + $parts = $filename.Split("-") + $arch = $parts[-1] + $buildPlatform = [string]::Join("-", $parts[2..($parts.Length-2)]) + + if ($PlatformMap[$buildPlatform]) { + $PlatformMap[$buildPlatform] | ForEach-Object { + $assets += New-AssetItem -Filename $releaseAsset.name ` + -DownloadUrl $releaseAsset.browser_download_url ` + -Arch $arch ` + -Platform $_.platform ` + -PlatformVersion $_.platform_version + } + + } else { + $assets += New-AssetItem -Filename $releaseAsset.name ` + -DownloadUrl $releaseAsset.browser_download_url ` + -Arch $arch ` + -Platform $buildPlatform + } + } + + return $assets +} + +function Get-VersionFromRelease { + param ( + [Parameter(Mandatory)][object]$Release + ) + # Release name can contain additional information after ':' so filter it + [string]$releaseName = $Release.name.Split(':')[0] + [Version]$version = $null + if (![Version]::TryParse($releaseName, [ref]$version)) { + throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )" + } + + return $version +} + +function Build-VersionsManifest { + param ( + [Parameter(Mandatory)][array]$Releases + ) + + $Releases = $Releases | Sort-Object -Property "published_at" -Descending + + $versionsHash = @{} + foreach ($release in $Releases) { + if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) { + continue + } + + [Version]$version = Get-VersionFromRelease $release + $versionKey = $version.ToString() + + if ($versionsHash.ContainsKey($versionKey)) { + continue + } + + $versionsHash.Add($versionKey, [PSCustomObject]@{ + version = $versionKey + stable = $true + release_url = $release.html_url + files = Build-AssetsList $release.assets + }) + } + + # Sort versions by descending + return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true } +} + +$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken +$releases = $gitHubApi.GetGitHubReleases() +$versionIndex = Build-VersionsManifest $releases +$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force diff --git a/helpers/github/create-pull-request.ps1 b/helpers/github/create-pull-request.ps1 new file mode 100644 index 0000000..9c01042 --- /dev/null +++ b/helpers/github/create-pull-request.ps1 @@ -0,0 +1,106 @@ +<# +.SYNOPSIS +Create commit with all unstaged changes in repository and create pull-request + +.PARAMETER RepositoryOwner +Required parameter. The organization which tool repository belongs +.PARAMETER RepositoryName +Optional parameter. The name of tool repository +.PARAMETER AccessToken +Required parameter. PAT Token to authorize +.PARAMETER BranchName +Required parameter. The name of branch where changes will be pushed +.PARAMETER CommitMessage +Required parameter. The commit message to push changes +.PARAMETER PullRequestTitle +Required parameter. The title of pull-request +.PARAMETER PullRequestBody +Required parameter. The description of pull-request +#> +param ( + [Parameter(Mandatory)] [string] $RepositoryOwner, + [Parameter(Mandatory)] [string] $RepositoryName, + [Parameter(Mandatory)] [string] $AccessToken, + [Parameter(Mandatory)] [string] $BranchName, + [Parameter(Mandatory)] [string] $CommitMessage, + [Parameter(Mandatory)] [string] $PullRequestTitle, + [Parameter(Mandatory)] [string] $PullRequestBody +) + +Import-Module (Join-Path $PSScriptRoot "github-api.psm1") +Import-Module (Join-Path $PSScriptRoot "git.psm1") + +function Update-PullRequest { + Param ( + [Parameter(Mandatory=$true)] + [object] $GitHubApi, + [Parameter(Mandatory=$true)] + [string] $Title, + [Parameter(Mandatory=$true)] + [string] $Body, + [Parameter(Mandatory=$true)] + [string] $BranchName, + [Parameter(Mandatory=$true)] + [object] $PullRequest + ) + + $updatedPullRequest = $GitHubApi.UpdatePullRequest($Title, $Body, $BranchName, $PullRequest.number) + + if (($updatedPullRequest -eq $null) -or ($updatedPullRequest.html_url -eq $null)) { + Write-Host "##vso[task.logissue type=error;] Unexpected error occurs while updating pull request." + exit 1 + } + Write-host "##[section] Pull request updated: $($updatedPullRequest.html_url)" +} + +function Create-PullRequest { + Param ( + [Parameter(Mandatory=$true)] + [object] $GitHubApi, + [Parameter(Mandatory=$true)] + [string] $Title, + [Parameter(Mandatory=$true)] + [string] $Body, + [Parameter(Mandatory=$true)] + [string] $BranchName + ) + + $createdPullRequest = $GitHubApi.CreateNewPullRequest($Title, $Body, $BranchName) + + if (($createdPullRequest -eq $null) -or ($createdPullRequest.html_url -eq $null)) { + Write-Host "##vso[task.logissue type=error;] Unexpected error occurs while creating pull request." + exit 1 + } + + Write-host "##[section] Pull request created: $($createdPullRequest.html_url)" +} + +Write-Host "Configure local git preferences" +Git-ConfigureUser -Name "Service account" -Email "no-reply@microsoft.com" + +Write-Host "Create branch: $BranchName" +Git-CreateBranch -Name $BranchName + +Write-Host "Create commit" +Git-CommitAllChanges -Message $CommitMessage + +Write-Host "Push branch: $BranchName" +Git-PushBranch -Name $BranchName -Force $true + +$gitHubApi = Get-GitHubApi -AccountName $RepositoryOwner -ProjectName $RepositoryName -AccessToken $AccessToken +$pullRequest = $gitHubApi.GetPullRequest($BranchName, $RepositoryOwner) + +if ($pullRequest.Count -gt 0) { + Write-Host "Update pull request" + Update-PullRequest -GitHubApi $gitHubApi ` + -Title $PullRequestTitle ` + -Body $PullRequestBody ` + -BranchName $BranchName ` + -PullRequest $pullRequest[0] +} else { + Write-Host "Create pull request" + Create-PullRequest -GitHubApi $gitHubApi ` + -Title $PullRequestTitle ` + -Body $PullRequestBody ` + -BranchName $BranchName +} diff --git a/helpers/github/git.psm1 b/helpers/github/git.psm1 new file mode 100644 index 0000000..2383a79 --- /dev/null +++ b/helpers/github/git.psm1 @@ -0,0 +1,81 @@ +<# +.SYNOPSIS +Configure git credentials to use with commits +#> +function Git-ConfigureUser { + Param ( + [Parameter(Mandatory=$true)] + [string] $Name, + [Parameter(Mandatory=$true)] + [string] $Email + ) + + git config --global user.name $Name | Out-Host + git config --global user.email $Email | Out-Host + + if ($LASTEXITCODE -ne 0) { + Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while configuring git preferences." + exit 1 + } +} + +<# +.SYNOPSIS +Create new branch +#> +function Git-CreateBranch { + Param ( + [Parameter(Mandatory=$true)] + [string] $Name + ) + + git checkout -b $Name | Out-Host + + if ($LASTEXITCODE -ne 0) { + Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while creating new branch: $Name." + exit 1 + } +} + +<# +.SYNOPSIS +Commit all staged and unstaged changes +#> +function Git-CommitAllChanges { + Param ( + [Parameter(Mandatory=$true)] + [string] $Message + ) + + git add -A | Out-Host + git commit -m "$Message" | Out-Host + + if ($LASTEXITCODE -ne 0) { + Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while commiting changes." + exit 1 + } +} + +<# +.SYNOPSIS +Push branch to remote repository +#> +function Git-PushBranch { + Param ( + [Parameter(Mandatory=$true)] + [string] $Name, + [Parameter(Mandatory=$true)] + [boolean] $Force + ) + + if ($Force) { + git push --set-upstream origin $Name --force | Out-Host + } else { + git push --set-upstream origin $Name | Out-Host + } + + if ($LASTEXITCODE -ne 0) { + Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while pushing changes." + exit 1 + } +} \ No newline at end of file diff --git a/helpers/github/github-api.psm1 b/helpers/github/github-api.psm1 new file mode 100644 index 0000000..fef64a5 --- /dev/null +++ b/helpers/github/github-api.psm1 @@ -0,0 +1,109 @@ +<# +.SYNOPSIS +The module that contains a bunch of methods to interact with GitHub API V3 +#> +class GitHubApi +{ + [string] $BaseUrl + [string] $RepoOwner + [object] $AuthHeader + + GitHubApi( + [string] $AccountName, + [string] $ProjectName, + [string] $AccessToken + ) { + $this.BaseUrl = $this.BuildBaseUrl($AccountName, $ProjectName) + $this.AuthHeader = $this.BuildAuth($AccessToken) + } + + [object] hidden BuildAuth([string]$AccessToken) { + if ([string]::IsNullOrEmpty($AccessToken)) { + return $null + } + $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}")) + return @{ + Authorization = "Basic ${base64AuthInfo}" + } + } + + [string] hidden BuildBaseUrl([string]$RepositoryOwner, [string]$RepositoryName) { + return "https://api.github.com/repos/$RepositoryOwner/$RepositoryName" + } + + [object] CreateNewPullRequest([string]$Title, [string]$Body, [string]$BranchName){ + $requestBody = @{ + title = $Title + body = $Body + head = $BranchName + base = "master" + } | ConvertTo-Json + + $url = "pulls" + return $this.InvokeRestMethod($url, 'Post', $null, $requestBody) + } + + [object] GetPullRequest([string]$BranchName, [string]$RepositoryOwner){ + $url = "pulls" + return $this.InvokeRestMethod($url, 'GET', "head=${RepositoryOwner}:$BranchName&base=master", $null) + } + + [object] UpdatePullRequest([string]$Title, [string]$Body, [string]$BranchName, [string]$PullRequestNumber){ + $requestBody = @{ + title = $Title + body = $Body + head = $BranchName + base = "master" + } | ConvertTo-Json + + $url = "pulls/$PullRequestNumber" + return $this.InvokeRestMethod($url, 'Post', $null, $requestBody) + } + + [object] GetGitHubReleases(){ + $url = "releases" + return $this.InvokeRestMethod($url, 'GET', $null, $null) + } + + [string] hidden BuildUrl([string]$Url, [string]$RequestParams) { + if ([string]::IsNullOrEmpty($RequestParams)) { + return "$($this.BaseUrl)/$($Url)" + } else { + return "$($this.BaseUrl)/$($Url)?$($RequestParams)" + } + } + + [object] hidden InvokeRestMethod( + [string] $Url, + [string] $Method, + [string] $RequestParams, + [string] $Body + ) { + $requestUrl = $this.BuildUrl($Url, $RequestParams) + $params = @{ + Method = $Method + ContentType = "application/json" + Uri = $requestUrl + Headers = @{} + } + if ($this.AuthHeader) { + $params.Headers += $this.AuthHeader + } + if (![string]::IsNullOrEmpty($Body)) { + $params.Body = $Body + } + + return Invoke-RestMethod @params + } + +} + +function Get-GitHubApi { + param ( + [string] $AccountName, + [string] $ProjectName, + [string] $AccessToken + ) + + return [GitHubApi]::New($AccountName, $ProjectName, $AccessToken) +} \ No newline at end of file diff --git a/helpers/nix-helpers.psm1 b/helpers/nix-helpers.psm1 new file mode 100644 index 0000000..9cbd49e --- /dev/null +++ b/helpers/nix-helpers.psm1 @@ -0,0 +1,50 @@ +<# +.SYNOPSIS +Pack folder to *.zip format +#> +function Pack-Zip { + param( + [Parameter(Mandatory=$true)] + [String]$PathToArchive, + [Parameter(Mandatory=$true)] + [String]$ToolZipFile + ) + + Write-Debug "Pack $PathToArchive to $ToolZipFile" + Push-Location -Path $PathToArchive + zip -q -r $ToolZipFile * | Out-Null + Pop-Location +} + +<# +.SYNOPSIS +Unpack *.tar file +#> +function Extract-TarArchive { + param( + [Parameter(Mandatory=$true)] + [String]$ArchivePath, + [Parameter(Mandatory=$true)] + [String]$OutputDirectory + ) + + Write-Debug "tar -C $OutputDirectory -xzf $ArchivePath --strip 1" + tar -C $OutputDirectory -xzf $ArchivePath --strip 1 +} + +function Create-TarArchive { + param( + [Parameter(Mandatory=$true)] + [String]$SourceFolder, + [Parameter(Mandatory=$true)] + [String]$ArchivePath, + [string]$CompressionType = "gz" + ) + + $CompressionTypeArgument = If ([string]::IsNullOrWhiteSpace($CompressionType)) { "" } else { "--${CompressionType}" } + + Push-Location $SourceFolder + Write-Debug "tar -c $CompressionTypeArgument -f $ArchivePath ." + tar -c $CompressionTypeArgument -f $ArchivePath . + Pop-Location +} \ No newline at end of file diff --git a/helpers/pester-extensions.psm1 b/helpers/pester-extensions.psm1 new file mode 100644 index 0000000..46d2a75 --- /dev/null +++ b/helpers/pester-extensions.psm1 @@ -0,0 +1,33 @@ +<# +.SYNOPSIS +Pester extension that allows to run command and validate exit code +.EXAMPLE +"python file.py" | Should -ReturnZeroExitCode +#> +function ShouldReturnZeroExitCode { + Param( + [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] + [String]$ActualValue, + [switch]$Negate + ) + + Write-Host "Run command '${ActualValue}'" + Invoke-Expression -Command $ActualValue | ForEach-Object { Write-Host $_ } + $actualExitCode = $LASTEXITCODE + + [bool]$succeeded = $actualExitCode -eq 0 + if ($Negate) { $succeeded = -not $succeeded } + + if (-not $succeeded) + { + $failureMessage = "Command '${ActualValue}' has finished with exit code ${actualExitCode}" + } + + return New-Object PSObject -Property @{ + Succeeded = $succeeded + FailureMessage = $failureMessage + } +} + +Add-AssertionOperator -Name ReturnZeroExitCode ` + -Test $function:ShouldReturnZeroExitCode diff --git a/helpers/win-helpers.psm1 b/helpers/win-helpers.psm1 new file mode 100644 index 0000000..b80145b --- /dev/null +++ b/helpers/win-helpers.psm1 @@ -0,0 +1,18 @@ +function Create-SevenZipArchive { + param( + [Parameter(Mandatory=$true)] + [String]$SourceFolder, + [Parameter(Mandatory=$true)] + [String]$ArchivePath, + [String]$ArchiveType = "zip", + [String]$CompressionLevel = 5 + ) + + $ArchiveTypeArgument = "-t${ArchiveType}" + $CompressionLevelArgument = "-mx=${CompressionLevel}" + + Push-Location $SourceFolder + Write-Debug "7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath @$SourceFolder" + 7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath $SourceFolder\* + Pop-Location +} \ No newline at end of file diff --git a/installers/map_python_platforms.json b/installers/map_python_platforms.json new file mode 100644 index 0000000..eb7ddd5 --- /dev/null +++ b/installers/map_python_platforms.json @@ -0,0 +1,29 @@ +{ + "macos-1014": [ + { + "platform": "darwin", + "platform_version": "10.14" + }, + { + "platform": "darwin", + "platform_version": "10.15" + } + ], + "ubuntu-1604": [ + { + "platform": "linux", + "platform_version": "16.04" + } + ], + "ubuntu-1804": [ + { + "platform": "linux", + "platform_version": "18.04" + } + ], + "windows-2016": [ + { + "platform": "win32" + } + ] +} \ No newline at end of file diff --git a/installers/nix-setup-template.sh b/installers/nix-setup-template.sh new file mode 100644 index 0000000..dd27788 --- /dev/null +++ b/installers/nix-setup-template.sh @@ -0,0 +1,50 @@ +set -e + +MAJOR_VERSION={0} +MINOR_VERSION={1} +BUILD_VERSION={2} + +PYTHON_MAJOR=python$MAJOR_VERSION +PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION +PYTHON_MAJORMINOR=python$MAJOR_VERSION$MINOR_VERSION +PYTHON_FULL_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$BUILD_VERSION + +PYTHON_TOOLCACHE_PATH=$AGENT_TOOLSDIRECTORY/Python +PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION +PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64 + +echo "Check if Python hostedtoolcache folder exist..." +if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then + echo "Creating Python hostedtoolcache folder..." + mkdir -p $PYTHON_TOOLCACHE_PATH +elif [ -d $PYTHON_TOOLCACHE_VERSION_PATH ]; then + echo "Deleting Python $PYTHON_FULL_VERSION" + rm -rf $PYTHON_TOOLCACHE_VERSION_PATH +fi + +echo "Create Python $PYTHON_FULL_VERSION folder" +mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH + +echo "Copy Python binaries to hostedtoolcache folder" +cp -R ./* $PYTHON_TOOLCACHE_VERSION_ARCH_PATH +rm $PYTHON_TOOLCACHE_VERSION_ARCH_PATH/setup.sh + +cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH + +echo "Create additional symlinks (Required for UsePythonVersion VSTS task)" +ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python + +cd bin/ +ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR +if [ ! -f python ]; then + ln -s $PYTHON_MAJOR_DOT_MINOR python +fi + +chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR python + +echo "Upgrading PIP..." +./python -m ensurepip +./python -m pip install --ignore-installed pip + +echo "Create complete file" +touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete \ No newline at end of file diff --git a/installers/win-setup-template.ps1 b/installers/win-setup-template.ps1 new file mode 100644 index 0000000..2868123 --- /dev/null +++ b/installers/win-setup-template.ps1 @@ -0,0 +1,138 @@ +[String] $Architecture = "{{__ARCHITECTURE__}}" +[Version] $Version = "{{__VERSION__}}" +[String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}" + +function Get-RegistryVersionFilter { + param + ( + [Parameter(Mandatory)][String] $Architecture, + [Parameter(Mandatory)][Int32] $MajorVersion, + [Parameter(Mandatory)][Int32] $MinorVersion + ) + + $archFilter = if ($Architecture -eq 'x86') { "32-bit" } else { "64-bit" } + ### Python 2.7 x86 have no architecture postfix + if (($Architecture -eq "x86") -and ($MajorVersion -eq 2)) + { + "Python $MajorVersion.$MinorVersion.\d+$" + } + else + { + "Python $MajorVersion.$MinorVersion.*($archFilter)" + } +} + +function Remove-RegistryEntries +{ + param + ( + [Parameter(Mandatory)][String] $Architecture, + [Parameter(Mandatory)][Int32] $MajorVersion, + [Parameter(Mandatory)][Int32] $MinorVersion + ) + + $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion + + $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products" + $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName + foreach ($key in $regKeys) + { + if ($key.getValue("DisplayName") -match $versionFilter) + { + Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose + } + } + + $regPath = "HKEY_CLASSES_ROOT\Installer\Products" + Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object { + Remove-Item Registry::$_ -Recurse -Force -Verbose + } + + $uninstallRegistrySections = @( + "HKEY_CURRENT_USER\Software\Microsoft\Windows\CurrentVersion\Uninstall", # current user, x64 + "HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion\Uninstall", # all users, x64 + "HKEY_CURRENT_USER\Software\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall", # current user, x86 + "HKEY_LOCAL_MACHINE\Software\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall" # all users, x86 + ) + + $uninstallRegistrySections | Where-Object { Test-Path -Path Registry::$_ } | ForEach-Object { + Get-ChildItem -Path Registry::$_ | Where-Object { $_.getValue("DisplayName") -match $versionFilter } | ForEach-Object { + Remove-Item Registry::$_ -Recurse -Force -Verbose + } + } +} + +function Get-ExecParams { + param + ( + [Parameter(Mandatory)][Boolean] $IsMSI, + [Parameter(Mandatory)][String] $PythonArchPath + ) + + if ($IsMSI) + { + "TARGETDIR=$PythonArchPath ALLUSERS=1" + } + else + { + "DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1" + } +} + +$ToolcacheRoot = $env:AGENT_TOOLSDIRECTORY +if ([string]::IsNullOrEmpty($ToolcacheRoot)) +{ + # GitHub images don't have `AGENT_TOOLSDIRECTORY` variable + $ToolcacheRoot = $env:RUNNER_TOOL_CACHE +} +$PythonToolcachePath = Join-Path -Path $ToolcacheRoot -ChildPath "Python" +$PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version.ToString() +$PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture + +$IsMSI = $PythonExecName -match "msi" + +$MajorVersion = $Version.Major +$MinorVersion = $Version.Minor + +Write-Host "Check if Python hostedtoolcache folder exist..." +if (-Not (Test-Path $PythonToolcachePath)) +{ + Write-Host "Create Python toolcache folder" + New-Item -ItemType Directory -Path $PythonToolcachePath | Out-Null +} + +Write-Host "Check if current Python version is installed..." +$InstalledVersion = Get-ChildItem -Path $PythonToolcachePath -Filter "$MajorVersion.$MinorVersion.*" + +Write-Host "Remove registry entries for Python ${MajorVersion}.${MinorVersion}(${Architecture})..." +Remove-RegistryEntries -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion + +if (($null -ne $InstalledVersion) -and (Test-Path -Path $InstalledVersion.FullName)) { + Write-Host "Python$MajorVersion.$MinorVersion was found in $PythonToolcachePath" + Write-Host "Deleting $($InstalledVersion.FullName)..." + Remove-Item -Path $InstalledVersion.FullName -Recurse -Force +} else { + Write-Host "No Python$MajorVersion.$MinorVersion.* found" +} + +Write-Host "Create Python $Version folder in $PythonToolcachePath" +New-Item -ItemType Directory -Path $PythonArchPath -Force | Out-Null + +Write-Host "Copy Python binaries to $PythonArchPath" +Copy-Item -Path ./$PythonExecName -Destination $PythonArchPath | Out-Null + +Write-Host "Install Python $Version in $PythonToolcachePath..." +$ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath + +cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet" +if ($LASTEXITCODE -ne 0) +{ + Throw "Error happened during Python installation" +} + +Write-Host "Install and upgrade Pip" +$PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe" +cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip" + +Write-Host "Create complete file" +New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null diff --git a/tests/clean-toolcache.ps1 b/tests/clean-toolcache.ps1 new file mode 100644 index 0000000..837c16b --- /dev/null +++ b/tests/clean-toolcache.ps1 @@ -0,0 +1,11 @@ +if ($env:PLATFORM -match 'windows') { + $PythonFilter = "Name like '%Python%'" + Get-WmiObject Win32_Product -Filter $PythonFilter | Foreach-Object { + Write-Host "Uninstalling $($_.Name) ..." + $_.Uninstall() | Out-Null + } +} + +$PythonToolcachePath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Python" +Write-Host "Removing Python toolcache directory ..." +Remove-Item -Path $PythonToolcachePath -Recurse -Force \ No newline at end of file diff --git a/tests/python-tests.ps1 b/tests/python-tests.ps1 new file mode 100644 index 0000000..8154182 --- /dev/null +++ b/tests/python-tests.ps1 @@ -0,0 +1,72 @@ +param ( + [Version] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] + $Version, + [String] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] + $Platform +) + +Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1") +Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1") + +function Analyze-MissingModules([string] $buildOutputLocation) { + $searchStringStart = "Failed to build these modules:" + $searchStringEnd = "running build_scripts" + $pattern = "$searchStringStart(.*?)$searchStringEnd" + + $buildContent = Get-Content -Path $buildOutputLocation + $splitBuiltOutput = $buildContent -split "\n"; + + ### Search for missing modules that are displayed between the search strings + $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern) + if ($regexMatch.Success) + { + Write-Host "Failed missing modules:" + Write-Host $regexMatch.Groups[1].Value + return 1 + } + + return 0 +} + +Describe "Tests" { + It "Python version" { + "python --version" | Should -ReturnZeroExitCode + $pythonLocation = (Get-Command "python").Path + $pythonLocation | Should -Not -BeNullOrEmpty + $expectedPath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "Python" + $pythonLocation.startsWith($expectedPath) | Should -BeTrue + } + + It "Run simple code" { + "python ./sources/simple-test.py" | Should -ReturnZeroExitCode + } + + if (IsNixPlatform $Platform) { + + It "Check for failed modules in build_output" { + $buildOutputLocation = Join-Path $env:BUILD_BINARIESDIRECTORY "build_output.txt" + Analyze-MissingModules $buildOutputLocation | Should -Be 0 + } + + It "Check if all required python modules are installed" { + "python ./sources/python-modules.py" | Should -ReturnZeroExitCode + } + + It "Check if python configuration is correct" { + "python ./sources/python-config-test.py" | Should -ReturnZeroExitCode + } + + It "Check if shared libraries are linked correctly" { + "bash ./sources/psutil-install-test.sh" | Should -ReturnZeroExitCode + } + } + + # Pyinstaller 3.5 does not support Python 3.8.0. Check issue https://github.com/pyinstaller/pyinstaller/issues/4311 + if ($Version -lt "3.8.0") { + It "Validate Pyinstaller" { + "pip install pyinstaller" | Should -ReturnZeroExitCode + "pyinstaller --onefile ./sources/simple-test.py" | Should -ReturnZeroExitCode + "./dist/simple-test" | Should -ReturnZeroExitCode + } + } +} \ No newline at end of file diff --git a/tests/sources/psutil-install-test.sh b/tests/sources/psutil-install-test.sh new file mode 100644 index 0000000..ddf0eb8 --- /dev/null +++ b/tests/sources/psutil-install-test.sh @@ -0,0 +1,5 @@ +# Check if shared libraries are linked correctly +python -m venv /tmp/aml-ve +source /tmp/aml-ve/bin/activate +easy_install --version +pip install psutil --verbose \ No newline at end of file diff --git a/tests/sources/python-config-output.py b/tests/sources/python-config-output.py new file mode 100644 index 0000000..02ba82b --- /dev/null +++ b/tests/sources/python-config-output.py @@ -0,0 +1,6 @@ +import distutils.sysconfig +import sysconfig + +from pprint import pprint +pprint(sysconfig.get_config_vars()) +pprint(distutils.sysconfig.get_config_vars()) \ No newline at end of file diff --git a/tests/sources/python-config-test.py b/tests/sources/python-config-test.py new file mode 100644 index 0000000..fe4d852 --- /dev/null +++ b/tests/sources/python-config-test.py @@ -0,0 +1,68 @@ +import distutils.sysconfig +import sysconfig +import sys +import platform +import os + +# Define variables +os_type = platform.system() +version = sys.version.split(" ")[0] + +lib_dir_path = sysconfig.get_config_var('LIBDIR') +ld_library_name = sysconfig.get_config_var('LDLIBRARY') + +is_shared = sysconfig.get_config_var('Py_ENABLE_SHARED') +have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE") + +### Define expected variables +if os_type == 'Linux': expected_ld_library_extension = 'so' +if os_type == 'Darwin': expected_ld_library_extension = 'dylib' +expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version) + +# Check modules +### Validate libraries path +if lib_dir_path != expected_lib_dir_path: + print('Invalid libraries location: %s; Expected: %s' % (lib_dir_path, expected_lib_dir_path)) + exit(1) + +### Validate shared libraries +if is_shared: + print('%s was built with shared extensions' % ld_library_name) + + ### Validate libpython extension + ld_library_extension = ld_library_name.split('.')[-1] + if ld_library_extension != expected_ld_library_extension: + print('Invalid extension: %s; Expected %s' % (ld_library_extension, expected_ld_library_extension)) + exit(1) +else: + print('%s was built without shared extensions' % ld_library_name) + exit(1) + +### Validate macOS +if os_type == 'Darwin': + ### Validate openssl links + if version < "3.7.0": + expected_ldflags = '-L/usr/local/opt/openssl@1.1/lib' + ldflags = sysconfig.get_config_var('LDFLAGS') + + if not expected_ldflags in ldflags: + print('Invalid ldflags: %s; Expected: %s' % (ldflags, expected_ldflags)) + exit(1) + else: + expected_openssl_includes = '-I/usr/local/opt/openssl/include' + expected_openssl_ldflags ='-L/usr/local/opt/openssl/lib' + + openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES') + openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS') + + if openssl_includes != expected_openssl_includes: + print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes)) + exit(1) + if openssl_ldflags != expected_openssl_ldflags: + print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags)) + exit(1) + +### Validate libreadline +if not have_libreadline: + print('Missing libreadline') + exit(1) \ No newline at end of file diff --git a/tests/sources/python-modules.py b/tests/sources/python-modules.py new file mode 100644 index 0000000..76408b7 --- /dev/null +++ b/tests/sources/python-modules.py @@ -0,0 +1,276 @@ +""" +Make sure all the optional modules are installed. +This is needed for Linux since we build from source. +""" + +from __future__ import print_function + +import importlib +import sys + +# The Python standard library as of Python 3.0 +standard_library = [ + 'abc', + 'aifc', + 'antigravity', + 'argparse', + 'ast', + 'asynchat', + 'asyncore', + 'base64', + 'bdb', + 'binhex', + 'bisect', + 'bz2', + 'cProfile', + 'calendar', + 'cgi', + 'cgitb', + 'chunk', + 'cmd', + 'code', + 'codecs', + 'codeop', + 'collections', + 'colorsys', + 'compileall', + 'configparser', + 'contextlib', + 'copy', + 'copyreg', + 'crypt', + 'csv', + 'ctypes', + 'curses', + 'datetime', + 'dbm', + 'decimal', + 'difflib', + 'dis', + 'distutils', + 'doctest', + 'dummy_threading', + 'email', + 'encodings', + 'filecmp', + 'fileinput', + 'fnmatch', + 'formatter', + 'fractions', + 'ftplib', + 'functools', + 'genericpath', + 'getopt', + 'getpass', + 'gettext', + 'glob', + 'gzip', + 'hashlib', + 'heapq', + 'hmac', + 'html', + 'http', + 'idlelib', + 'imaplib', + 'imghdr', + 'imp', + 'importlib', + 'inspect', + 'io', + 'json', + 'keyword', + 'lib2to3', + 'linecache', + 'locale', + 'logging', + 'macpath', + 'mailbox', + 'mailcap', + 'mimetypes', + 'modulefinder', + 'multiprocessing', + 'netrc', + 'nntplib', + 'ntpath', + 'nturl2path', + 'numbers', + 'opcode', + 'operator', + 'optparse', + 'os', + 'pdb', + 'pickle', + 'pickletools', + 'pipes', + 'pkgutil', + 'platform', + 'plistlib', + 'poplib', + 'posixpath', + 'pprint', + 'profile', + 'pstats', + 'pty', + 'py_compile', + 'pyclbr', + 'pydoc', + 'pydoc_data', + 'queue', + 'quopri', + 'random', + 're', + 'readline', + 'reprlib', + 'rlcompleter', + 'runpy', + 'sched', + 'shelve', + 'shlex', + 'shutil', + 'signal', + 'site', + 'smtpd', + 'smtplib', + 'sndhdr', + 'socket', + 'socketserver', + 'sqlite3', + 'sre_compile', + 'sre_constants', + 'sre_parse', + 'ssl', + 'stat', + 'string', + 'stringprep', + 'struct', + 'subprocess', + 'sunau', + 'symbol', + 'symtable', + 'sysconfig', + 'tabnanny', + 'tarfile', + 'telnetlib', + 'tempfile', + 'test', + 'textwrap', + 'this', + 'threading', + 'timeit', + 'tkinter', + 'token', + 'tokenize', + 'trace', + 'traceback', + 'tty', + 'turtle', + 'turtledemo', + 'types', + 'unittest', + 'urllib', + 'uu', + 'uuid', + 'warnings', + 'wave', + 'weakref', + 'webbrowser', + 'wsgiref', + 'xdrlib', + 'xml', + 'xmlrpc', + 'zipfile' +] + +# Modules that had different names in Python 2 +if sys.version_info.major == 2: + def replace(lst, old, new): + lst[lst.index(old)] = new + + # Keys are the Python 2 names + # Values are the Python 3 names + renames = { + 'ConfigParser': 'configparser', + 'copy_reg': 'copyreg', + 'HTMLParser': 'html', + 'httplib': 'http', + 'Queue': 'queue', + 'repr': 'reprlib', + 'SocketServer': 'socketserver', + 'xmlrpclib': 'xmlrpc', + 'Tkinter': 'tkinter' + } + + # All of the Python 3 names should be in the list + for python2name, python3name in renames.items(): + replace(standard_library, python3name, python2name) + +# Add new modules +# See https://docs.python.org/3/whatsnew/index.html +if sys.version_info >= (3, 2): + standard_library.extend([ + 'concurrent', + ]) + +if sys.version_info >= (3, 3): + standard_library.extend([ + 'ipaddress', + 'faulthandler', + 'lzma', + 'venv', + ]) + +if sys.version_info >= (3, 4): + standard_library.extend([ + 'asyncio', + 'ensurepip', + 'enum', + 'pathlib', + 'selectors', + 'statistics', + 'tracemalloc', + ]) + +if sys.version_info >= (3, 5): + standard_library.extend([ + 'typing', + 'zipapp', + ]) + +if sys.version_info >= (3, 6): + standard_library.extend([ + 'secrets', + ]) + +if sys.version_info >= (3, 7): + standard_library.extend([ + 'contextvars', + 'dataclasses', + ]) + +# 'macpath' module has been removed from Python 3.8 +if sys.version_info > (3, 7): + standard_library.remove('macpath') + +# Remove tkinter and Easter eggs +excluded_modules = [ + 'antigravity', + 'this', + 'turtledemo', +] + +def check_missing_modules(expected_modules): + missing = [] + for module in expected_modules: + print('Try to import module ', module) + try: + importlib.import_module(module) + except: + missing.append(module) + return missing + +missing_modules = check_missing_modules(x for x in standard_library if x not in excluded_modules) +if missing_modules: + print('The following modules are missing:') + for module in missing_modules: + print(' ', module) + exit(1) diff --git a/tests/sources/simple-test.py b/tests/sources/simple-test.py new file mode 100644 index 0000000..dc63846 --- /dev/null +++ b/tests/sources/simple-test.py @@ -0,0 +1,11 @@ +import sys +print(sys.version) +print(sys.prefix) + +# Python program to find the factorial of a number +num = 65 +factorial = 1 +print("Find the factorial of ", num) +for i in range(1, num + 1): + factorial = factorial*i +print("The factorial of ", num, " is ", factorial) \ No newline at end of file