Adding support for more PyPy versions and installing them on-flight (#168)

* add support to install pypy

* resolved comments, update readme, add e2e tests.

* resolve throw error

* Add pypy unit tests to cover code

* add tests

* Update test-pypy.yml

* Update test-python.yml

* Update test-python.yml

* Update README.md

* fixing tests

* change order

Co-authored-by: Maxim Lobanov <v-malob@microsoft.com>

* add pypy tests and fix issue with pypy-3-nightly

Co-authored-by: Maxim Lobanov <v-malob@microsoft.com>

src/find-pypy.ts

@@ -0,0 +1,131 @@
+import * as path from 'path';
+import * as pypyInstall from './install-pypy';
+import {
+  IS_WINDOWS,
+  validateVersion,
+  getPyPyVersionFromPath,
+  readExactPyPyVersionFile,
+  validatePythonVersionFormatForPyPy
+} from './utils';
+
+import * as semver from 'semver';
+import * as core from '@actions/core';
+import * as tc from '@actions/tool-cache';
+
+interface IPyPyVersionSpec {
+  pypyVersion: string;
+  pythonVersion: string;
+}
+
+export async function findPyPyVersion(
+  versionSpec: string,
+  architecture: string
+): Promise<{resolvedPyPyVersion: string; resolvedPythonVersion: string}> {
+  let resolvedPyPyVersion = '';
+  let resolvedPythonVersion = '';
+  let installDir: string | null;
+
+  const pypyVersionSpec = parsePyPyVersion(versionSpec);
+
+  // PyPy only precompiles binaries for x86, but the architecture parameter defaults to x64.
+  if (IS_WINDOWS && architecture === 'x64') {
+    architecture = 'x86';
+  }
+
+  ({installDir, resolvedPythonVersion, resolvedPyPyVersion} = findPyPyToolCache(
+    pypyVersionSpec.pythonVersion,
+    pypyVersionSpec.pypyVersion,
+    architecture
+  ));
+
+  if (!installDir) {
+    ({
+      installDir,
+      resolvedPythonVersion,
+      resolvedPyPyVersion
+    } = await pypyInstall.installPyPy(
+      pypyVersionSpec.pypyVersion,
+      pypyVersionSpec.pythonVersion,
+      architecture
+    ));
+  }
+
+  const pipDir = IS_WINDOWS ? 'Scripts' : 'bin';
+  const _binDir = path.join(installDir, pipDir);
+  const pythonLocation = pypyInstall.getPyPyBinaryPath(installDir);
+  core.exportVariable('pythonLocation', pythonLocation);
+  core.addPath(pythonLocation);
+  core.addPath(_binDir);
+
+  return {resolvedPyPyVersion, resolvedPythonVersion};
+}
+
+export function findPyPyToolCache(
+  pythonVersion: string,
+  pypyVersion: string,
+  architecture: string
+) {
+  let resolvedPyPyVersion = '';
+  let resolvedPythonVersion = '';
+  let installDir: string | null = tc.find('PyPy', pythonVersion, architecture);
+
+  if (installDir) {
+    // 'tc.find' finds tool based on Python version but we also need to check
+    // whether PyPy version satisfies requested version.
+    resolvedPythonVersion = getPyPyVersionFromPath(installDir);
+    resolvedPyPyVersion = readExactPyPyVersionFile(installDir);
+
+    const isPyPyVersionSatisfies = semver.satisfies(
+      resolvedPyPyVersion,
+      pypyVersion
+    );
+    if (!isPyPyVersionSatisfies) {
+      installDir = null;
+      resolvedPyPyVersion = '';
+      resolvedPythonVersion = '';
+    }
+  }
+
+  if (!installDir) {
+    core.info(
+      `PyPy version ${pythonVersion} (${pypyVersion}) was not found in the local cache`
+    );
+  }
+
+  return {installDir, resolvedPythonVersion, resolvedPyPyVersion};
+}
+
+export function parsePyPyVersion(versionSpec: string): IPyPyVersionSpec {
+  const versions = versionSpec.split('-').filter(item => !!item);
+
+  if (versions.length < 2 || versions[0] != 'pypy') {
+    throw new Error(
+      "Invalid 'version' property for PyPy. PyPy version should be specified as 'pypy-<python-version>'. See README for examples and documentation."
+    );
+  }
+
+  const pythonVersion = versions[1];
+  let pypyVersion: string;
+  if (versions.length > 2) {
+    pypyVersion = pypyInstall.pypyVersionToSemantic(versions[2]);
+  } else {
+    pypyVersion = 'x';
+  }
+
+  if (!validateVersion(pythonVersion) || !validateVersion(pypyVersion)) {
+    throw new Error(
+      "Invalid 'version' property for PyPy. Both Python version and PyPy versions should satisfy SemVer notation. See README for examples and documentation."
+    );
+  }
+
+  if (!validatePythonVersionFormatForPyPy(pythonVersion)) {
+    throw new Error(
+      "Invalid format of Python version for PyPy. Python version should be specified in format 'x.y'. See README for examples and documentation."
+    );
+  }
+
+  return {
+    pypyVersion: pypyVersion,
+    pythonVersion: pythonVersion
+  };
+}

src/find-python.ts

@@ -1,5 +1,6 @@
 import * as os from 'os';
 import * as path from 'path';
+import {IS_WINDOWS, IS_LINUX} from './utils';
 
 import * as semver from 'semver';
 
@@ -8,9 +9,6 @@ import * as installer from './install-python';
 import * as core from '@actions/core';
 import * as tc from '@actions/tool-cache';
 
-const IS_WINDOWS = process.platform === 'win32';
-const IS_LINUX = process.platform === 'linux';
-
 // Python has "scripts" or "bin" directories where command-line tools that come with packages are installed.
 // This is where pip is, along with anything that pip installs.
 // There is a seperate directory for `pip install --user`.

src/install-pypy.ts

@@ -0,0 +1,193 @@
+import * as path from 'path';
+import * as core from '@actions/core';
+import * as tc from '@actions/tool-cache';
+import * as semver from 'semver';
+import * as httpm from '@actions/http-client';
+import * as exec from '@actions/exec';
+import fs from 'fs';
+
+import {
+  IS_WINDOWS,
+  IPyPyManifestRelease,
+  createSymlinkInFolder,
+  isNightlyKeyword,
+  writeExactPyPyVersionFile
+} from './utils';
+
+export async function installPyPy(
+  pypyVersion: string,
+  pythonVersion: string,
+  architecture: string
+) {
+  let downloadDir;
+
+  const releases = await getAvailablePyPyVersions();
+  if (!releases || releases.length === 0) {
+    throw new Error('No release was found in PyPy version.json');
+  }
+
+  const releaseData = findRelease(
+    releases,
+    pythonVersion,
+    pypyVersion,
+    architecture
+  );
+
+  if (!releaseData || !releaseData.foundAsset) {
+    throw new Error(
+      `PyPy version ${pythonVersion} (${pypyVersion}) with arch ${architecture} not found`
+    );
+  }
+
+  const {foundAsset, resolvedPythonVersion, resolvedPyPyVersion} = releaseData;
+  let downloadUrl = `${foundAsset.download_url}`;
+
+  core.info(`Downloading PyPy from "${downloadUrl}" ...`);
+  const pypyPath = await tc.downloadTool(downloadUrl);
+
+  core.info('Extracting downloaded archive...');
+  if (IS_WINDOWS) {
+    downloadDir = await tc.extractZip(pypyPath);
+  } else {
+    downloadDir = await tc.extractTar(pypyPath, undefined, 'x');
+  }
+
+  // root folder in archive can have unpredictable name so just take the first folder
+  // downloadDir is unique folder under TEMP and can't contain any other folders
+  const archiveName = fs.readdirSync(downloadDir)[0];
+
+  const toolDir = path.join(downloadDir, archiveName);
+  let installDir = toolDir;
+  if (!isNightlyKeyword(resolvedPyPyVersion)) {
+    installDir = await tc.cacheDir(
+      toolDir,
+      'PyPy',
+      resolvedPythonVersion,
+      architecture
+    );
+  }
+
+  writeExactPyPyVersionFile(installDir, resolvedPyPyVersion);
+
+  const binaryPath = getPyPyBinaryPath(installDir);
+  await createPyPySymlink(binaryPath, resolvedPythonVersion);
+  await installPip(binaryPath);
+
+  return {installDir, resolvedPythonVersion, resolvedPyPyVersion};
+}
+
+async function getAvailablePyPyVersions() {
+  const url = 'https://downloads.python.org/pypy/versions.json';
+  const http: httpm.HttpClient = new httpm.HttpClient('tool-cache');
+
+  const response = await http.getJson<IPyPyManifestRelease[]>(url);
+  if (!response.result) {
+    throw new Error(
+      `Unable to retrieve the list of available PyPy versions from '${url}'`
+    );
+  }
+
+  return response.result;
+}
+
+async function createPyPySymlink(
+  pypyBinaryPath: string,
+  pythonVersion: string
+) {
+  const version = semver.coerce(pythonVersion)!;
+  const pythonBinaryPostfix = semver.major(version);
+  const pypyBinaryPostfix = pythonBinaryPostfix === 2 ? '' : '3';
+  let binaryExtension = IS_WINDOWS ? '.exe' : '';
+
+  core.info('Creating symlinks...');
+  createSymlinkInFolder(
+    pypyBinaryPath,
+    `pypy${pypyBinaryPostfix}${binaryExtension}`,
+    `python${pythonBinaryPostfix}${binaryExtension}`,
+    true
+  );
+
+  createSymlinkInFolder(
+    pypyBinaryPath,
+    `pypy${pypyBinaryPostfix}${binaryExtension}`,
+    `python${binaryExtension}`,
+    true
+  );
+}
+
+async function installPip(pythonLocation: string) {
+  core.info('Installing and updating pip');
+  const pythonBinary = path.join(pythonLocation, 'python');
+  await exec.exec(`${pythonBinary} -m ensurepip`);
+
+  await exec.exec(
+    `${pythonLocation}/python -m pip install --ignore-installed pip`
+  );
+}
+
+export function findRelease(
+  releases: IPyPyManifestRelease[],
+  pythonVersion: string,
+  pypyVersion: string,
+  architecture: string
+) {
+  const filterReleases = releases.filter(item => {
+    const isPythonVersionSatisfied = semver.satisfies(
+      semver.coerce(item.python_version)!,
+      pythonVersion
+    );
+    const isPyPyNightly =
+      isNightlyKeyword(pypyVersion) && isNightlyKeyword(item.pypy_version);
+    const isPyPyVersionSatisfied =
+      isPyPyNightly ||
+      semver.satisfies(pypyVersionToSemantic(item.pypy_version), pypyVersion);
+    const isArchPresent =
+      item.files &&
+      item.files.some(
+        file => file.arch === architecture && file.platform === process.platform
+      );
+    return isPythonVersionSatisfied && isPyPyVersionSatisfied && isArchPresent;
+  });
+
+  if (filterReleases.length === 0) {
+    return null;
+  }
+
+  const sortedReleases = filterReleases.sort((previous, current) => {
+    return (
+      semver.compare(
+        semver.coerce(pypyVersionToSemantic(current.pypy_version))!,
+        semver.coerce(pypyVersionToSemantic(previous.pypy_version))!
+      ) ||
+      semver.compare(
+        semver.coerce(current.python_version)!,
+        semver.coerce(previous.python_version)!
+      )
+    );
+  });
+
+  const foundRelease = sortedReleases[0];
+  const foundAsset = foundRelease.files.find(
+    item => item.arch === architecture && item.platform === process.platform
+  );
+
+  return {
+    foundAsset,
+    resolvedPythonVersion: foundRelease.python_version,
+    resolvedPyPyVersion: foundRelease.pypy_version
+  };
+}
+
+/** Get PyPy binary location from the tool of installation directory
+ *  - On Linux and macOS, the Python interpreter is in 'bin'.
+ *  - On Windows, it is in the installation root.
+ */
+export function getPyPyBinaryPath(installDir: string) {
+  const _binDir = path.join(installDir, 'bin');
+  return IS_WINDOWS ? installDir : _binDir;
+}
+
+export function pypyVersionToSemantic(versionSpec: string) {
+  const prereleaseVersion = /(\d+\.\d+\.\d+)((?:a|b|rc))(\d*)/g;
+  return versionSpec.replace(prereleaseVersion, '$1-$2.$3');
+}

src/install-python.ts

@@ -3,7 +3,7 @@ import * as core from '@actions/core';
 import * as tc from '@actions/tool-cache';
 import * as exec from '@actions/exec';
 import {ExecOptions} from '@actions/exec/lib/interfaces';
-import {stderr} from 'process';
+import {IS_WINDOWS, IS_LINUX} from './utils';
 
 const TOKEN = core.getInput('token');
 const AUTH = !TOKEN || isGhes() ? undefined : `token ${TOKEN}`;
@@ -12,9 +12,6 @@ const MANIFEST_REPO_NAME = 'python-versions';
 const MANIFEST_REPO_BRANCH = 'main';
 export const MANIFEST_URL = `https://raw.githubusercontent.com/${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}/${MANIFEST_REPO_BRANCH}/versions-manifest.json`;
 
-const IS_WINDOWS = process.platform === 'win32';
-const IS_LINUX = process.platform === 'linux';
-
 export async function findReleaseFromManifest(
   semanticVersionSpec: string,
   architecture: string

src/setup-python.ts

@@ -1,15 +1,29 @@
 import * as core from '@actions/core';
 import * as finder from './find-python';
+import * as finderPyPy from './find-pypy';
 import * as path from 'path';
 import * as os from 'os';
 
+function isPyPyVersion(versionSpec: string) {
+  return versionSpec.startsWith('pypy-');
+}
+
 async function run() {
   try {
     let version = core.getInput('python-version');
     if (version) {
       const arch: string = core.getInput('architecture') || os.arch();
-      const installed = await finder.findPythonVersion(version, arch);
-      core.info(`Successfully setup ${installed.impl} (${installed.version})`);
+      if (isPyPyVersion(version)) {
+        const installed = await finderPyPy.findPyPyVersion(version, arch);
+        core.info(
+          `Successfully setup PyPy ${installed.resolvedPyPyVersion} with Python (${installed.resolvedPythonVersion})`
+        );
+      } else {
+        const installed = await finder.findPythonVersion(version, arch);
+        core.info(
+          `Successfully setup ${installed.impl} (${installed.version})`
+        );
+      }
     }
     const matchersPath = path.join(__dirname, '..', '.github');
     core.info(`##[add-matcher]${path.join(matchersPath, 'python.json')}`);

src/utils.ts

@@ -0,0 +1,92 @@
+import fs from 'fs';
+import * as path from 'path';
+import * as semver from 'semver';
+
+export const IS_WINDOWS = process.platform === 'win32';
+export const IS_LINUX = process.platform === 'linux';
+const PYPY_VERSION_FILE = 'PYPY_VERSION';
+
+export interface IPyPyManifestAsset {
+  filename: string;
+  arch: string;
+  platform: string;
+  download_url: string;
+}
+
+export interface IPyPyManifestRelease {
+  pypy_version: string;
+  python_version: string;
+  stable: boolean;
+  latest_pypy: boolean;
+  files: IPyPyManifestAsset[];
+}
+
+/** create Symlinks for downloaded PyPy
+ *  It should be executed only for downloaded versions in runtime, because
+ *  toolcache versions have this setup.
+ */
+export function createSymlinkInFolder(
+  folderPath: string,
+  sourceName: string,
+  targetName: string,
+  setExecutable = false
+) {
+  const sourcePath = path.join(folderPath, sourceName);
+  const targetPath = path.join(folderPath, targetName);
+  if (fs.existsSync(targetPath)) {
+    return;
+  }
+
+  fs.symlinkSync(sourcePath, targetPath);
+  if (!IS_WINDOWS && setExecutable) {
+    fs.chmodSync(targetPath, '755');
+  }
+}
+
+export function validateVersion(version: string) {
+  return isNightlyKeyword(version) || Boolean(semver.validRange(version));
+}
+
+export function isNightlyKeyword(pypyVersion: string) {
+  return pypyVersion === 'nightly';
+}
+
+export function getPyPyVersionFromPath(installDir: string) {
+  return path.basename(path.dirname(installDir));
+}
+
+/**
+ * In tool-cache, we put PyPy to '<toolcache_root>/PyPy/<python_version>/x64'
+ * There is no easy way to determine what PyPy version is located in specific folder
+ * 'pypy --version' is not reliable enough since it is not set properly for preview versions
+ * "7.3.3rc1" is marked as '7.3.3' in 'pypy --version'
+ * so we put PYPY_VERSION file to PyPy directory when install it to VM and read it when we need to know version
+ * PYPY_VERSION contains exact version from 'versions.json'
+ */
+export function readExactPyPyVersionFile(installDir: string) {
+  let pypyVersion = '';
+  let fileVersion = path.join(installDir, PYPY_VERSION_FILE);
+  if (fs.existsSync(fileVersion)) {
+    pypyVersion = fs.readFileSync(fileVersion).toString();
+  }
+
+  return pypyVersion;
+}
+
+export function writeExactPyPyVersionFile(
+  installDir: string,
+  resolvedPyPyVersion: string
+) {
+  const pypyFilePath = path.join(installDir, PYPY_VERSION_FILE);
+  fs.writeFileSync(pypyFilePath, resolvedPyPyVersion);
+}
+
+/**
+ * Python version should be specified explicitly like "x.y" (2.7, 3.6, 3.7)
+ * "3.x" or "3" are not supported
+ * because it could cause ambiguity when both PyPy version and Python version are not precise
+ */
+export function validatePythonVersionFormatForPyPy(version: string) {
+  const re = /^\d+\.\d+$/;
+  return re.test(version);
+}