Add GraalPy support (#694)

* Add support for graalpy

* add graalpy test workflow

* format, lint and build

* symlink graalpy binaries names

* fix macos names for graalpy

* Don't attempt to update pip for graalpy

* Remove test schedule

* Extract common getBinaryDirectory function for PyPy and GraalPy

* Clean up and format

* Pass GitHub token to GraalPy queries

* Utilize pagination when querying GraalPy GitHub releases

* Build

* Fix lint errors

* Deal with possible multiple artifacts for a single releases

* Skip few GraalPy tests on windows - we don't have a windows release yet

* Fix GraalPy test on Mac OS

* Build

* Skip one more GraalPy test on windows

---------

Co-authored-by: Michael Simacek <michael.simacek@oracle.com>

src/find-graalpy.ts

@@ -0,0 +1,146 @@
+import * as path from 'path';
+import * as graalpyInstall from './install-graalpy';
+import {
+  IS_WINDOWS,
+  validateVersion,
+  IGraalPyManifestRelease,
+  getBinaryDirectory
+} from './utils';
+
+import * as semver from 'semver';
+import * as core from '@actions/core';
+import * as tc from '@actions/tool-cache';
+
+export async function findGraalPyVersion(
+  versionSpec: string,
+  architecture: string,
+  updateEnvironment: boolean,
+  checkLatest: boolean,
+  allowPreReleases: boolean
+): Promise<string> {
+  let resolvedGraalPyVersion = '';
+  let installDir: string | null;
+  let releases: IGraalPyManifestRelease[] | undefined;
+
+  let graalpyVersionSpec = parseGraalPyVersion(versionSpec);
+
+  if (checkLatest) {
+    releases = await graalpyInstall.getAvailableGraalPyVersions();
+    if (releases && releases.length > 0) {
+      const releaseData = graalpyInstall.findRelease(
+        releases,
+        graalpyVersionSpec,
+        architecture,
+        false
+      );
+
+      if (releaseData) {
+        core.info(`Resolved as GraalPy ${releaseData.resolvedGraalPyVersion}`);
+        graalpyVersionSpec = releaseData.resolvedGraalPyVersion;
+      } else {
+        core.info(
+          `Failed to resolve GraalPy ${graalpyVersionSpec} from manifest`
+        );
+      }
+    }
+  }
+
+  ({installDir, resolvedGraalPyVersion} = findGraalPyToolCache(
+    graalpyVersionSpec,
+    architecture
+  ));
+
+  if (!installDir) {
+    ({installDir, resolvedGraalPyVersion} = await graalpyInstall.installGraalPy(
+      graalpyVersionSpec,
+      architecture,
+      allowPreReleases,
+      releases
+    ));
+  }
+
+  const pipDir = IS_WINDOWS ? 'Scripts' : 'bin';
+  const _binDir = path.join(installDir, pipDir);
+  const binaryExtension = IS_WINDOWS ? '.exe' : '';
+  const pythonPath = path.join(
+    IS_WINDOWS ? installDir : _binDir,
+    `python${binaryExtension}`
+  );
+  const pythonLocation = getBinaryDirectory(installDir);
+  if (updateEnvironment) {
+    core.exportVariable('pythonLocation', installDir);
+    // https://cmake.org/cmake/help/latest/module/FindPython.html#module:FindPython
+    core.exportVariable('Python_ROOT_DIR', installDir);
+    // https://cmake.org/cmake/help/latest/module/FindPython2.html#module:FindPython2
+    core.exportVariable('Python2_ROOT_DIR', installDir);
+    // https://cmake.org/cmake/help/latest/module/FindPython3.html#module:FindPython3
+    core.exportVariable('Python3_ROOT_DIR', installDir);
+    core.exportVariable('PKG_CONFIG_PATH', pythonLocation + '/lib/pkgconfig');
+    core.addPath(pythonLocation);
+    core.addPath(_binDir);
+  }
+  core.setOutput('python-version', 'graalpy' + resolvedGraalPyVersion);
+  core.setOutput('python-path', pythonPath);
+
+  return resolvedGraalPyVersion;
+}
+
+export function findGraalPyToolCache(
+  graalpyVersion: string,
+  architecture: string
+) {
+  let resolvedGraalPyVersion = '';
+  let installDir: string | null = tc.find(
+    'GraalPy',
+    graalpyVersion,
+    architecture
+  );
+
+  if (installDir) {
+    // 'tc.find' finds tool based on Python version but we also need to check
+    // whether GraalPy version satisfies requested version.
+    resolvedGraalPyVersion = path.basename(path.dirname(installDir));
+
+    const isGraalPyVersionSatisfies = semver.satisfies(
+      resolvedGraalPyVersion,
+      graalpyVersion
+    );
+    if (!isGraalPyVersionSatisfies) {
+      installDir = null;
+      resolvedGraalPyVersion = '';
+    }
+  }
+
+  if (!installDir) {
+    core.info(
+      `GraalPy version ${graalpyVersion} was not found in the local cache`
+    );
+  }
+
+  return {installDir, resolvedGraalPyVersion};
+}
+
+export function parseGraalPyVersion(versionSpec: string): string {
+  const versions = versionSpec.split('-').filter(item => !!item);
+
+  if (/^(graalpy)(.+)/.test(versions[0])) {
+    const version = versions[0].replace('graalpy', '');
+    versions.splice(0, 1, 'graalpy', version);
+  }
+
+  if (versions.length < 2 || versions[0] != 'graalpy') {
+    throw new Error(
+      "Invalid 'version' property for GraalPy. GraalPy version should be specified as 'graalpy<python-version>' or 'graalpy-<python-version>'. See README for examples and documentation."
+    );
+  }
+
+  const pythonVersion = versions[1];
+
+  if (!validateVersion(pythonVersion)) {
+    throw new Error(
+      "Invalid 'version' property for GraalPy. GraalPy versions should satisfy SemVer notation. See README for examples and documentation."
+    );
+  }
+
+  return pythonVersion;
+}

src/find-pypy.ts

@@ -7,7 +7,8 @@ import {
   getPyPyVersionFromPath,
   readExactPyPyVersionFile,
   validatePythonVersionFormatForPyPy,
-  IPyPyManifestRelease
+  IPyPyManifestRelease,
+  getBinaryDirectory
 } from './utils';
 
 import * as semver from 'semver';
@@ -82,7 +83,7 @@ export async function findPyPyVersion(
     IS_WINDOWS ? installDir : _binDir,
     `python${binaryExtension}`
   );
-  const pythonLocation = pypyInstall.getPyPyBinaryPath(installDir);
+  const pythonLocation = getBinaryDirectory(installDir);
   if (updateEnvironment) {
     core.exportVariable('pythonLocation', installDir);
     // https://cmake.org/cmake/help/latest/module/FindPython.html#module:FindPython

src/install-graalpy.ts

@@ -0,0 +1,262 @@
+import * as os from 'os';
+import * as path from 'path';
+import * as core from '@actions/core';
+import * as tc from '@actions/tool-cache';
+import * as semver from 'semver';
+import * as httpm from '@actions/http-client';
+import * as ifm from '@actions/http-client/interfaces';
+import * as exec from '@actions/exec';
+import fs from 'fs';
+
+import {
+  IS_WINDOWS,
+  IGraalPyManifestRelease,
+  createSymlinkInFolder,
+  isNightlyKeyword,
+  getBinaryDirectory,
+  getNextPageUrl
+} from './utils';
+
+const TOKEN = core.getInput('token');
+const AUTH = !TOKEN ? undefined : `token ${TOKEN}`;
+
+export async function installGraalPy(
+  graalpyVersion: string,
+  architecture: string,
+  allowPreReleases: boolean,
+  releases: IGraalPyManifestRelease[] | undefined
+) {
+  let downloadDir;
+
+  releases = releases ?? (await getAvailableGraalPyVersions());
+
+  if (!releases || !releases.length) {
+    throw new Error('No release was found in GraalPy version.json');
+  }
+
+  let releaseData = findRelease(releases, graalpyVersion, architecture, false);
+
+  if (allowPreReleases && (!releaseData || !releaseData.foundAsset)) {
+    // check for pre-release
+    core.info(
+      [
+        `Stable GraalPy version ${graalpyVersion} with arch ${architecture} not found`,
+        `Trying pre-release versions`
+      ].join(os.EOL)
+    );
+    releaseData = findRelease(releases, graalpyVersion, architecture, true);
+  }
+
+  if (!releaseData || !releaseData.foundAsset) {
+    throw new Error(
+      `GraalPy version ${graalpyVersion} with arch ${architecture} not found`
+    );
+  }
+
+  const {foundAsset, resolvedGraalPyVersion} = releaseData;
+  const downloadUrl = `${foundAsset.browser_download_url}`;
+
+  core.info(`Downloading GraalPy from "${downloadUrl}" ...`);
+
+  try {
+    const graalpyPath = await tc.downloadTool(downloadUrl, undefined, AUTH);
+
+    core.info('Extracting downloaded archive...');
+    downloadDir = await tc.extractTar(graalpyPath);
+
+    // root folder in archive can have unpredictable name so just take the first folder
+    // downloadDir is unique folder under TEMP and can't contain any other folders
+    const archiveName = fs.readdirSync(downloadDir)[0];
+
+    const toolDir = path.join(downloadDir, archiveName);
+    let installDir = toolDir;
+    if (!isNightlyKeyword(resolvedGraalPyVersion)) {
+      installDir = await tc.cacheDir(
+        toolDir,
+        'GraalPy',
+        resolvedGraalPyVersion,
+        architecture
+      );
+    }
+
+    const binaryPath = getBinaryDirectory(installDir);
+    await createGraalPySymlink(binaryPath, resolvedGraalPyVersion);
+    await installPip(binaryPath);
+
+    return {installDir, resolvedGraalPyVersion};
+  } catch (err) {
+    if (err instanceof Error) {
+      // Rate limit?
+      if (
+        err instanceof tc.HTTPError &&
+        (err.httpStatusCode === 403 || err.httpStatusCode === 429)
+      ) {
+        core.info(
+          `Received HTTP status code ${err.httpStatusCode}.  This usually indicates the rate limit has been exceeded`
+        );
+      } else {
+        core.info(err.message);
+      }
+      if (err.stack !== undefined) {
+        core.debug(err.stack);
+      }
+    }
+    throw err;
+  }
+}
+
+export async function getAvailableGraalPyVersions() {
+  const http: httpm.HttpClient = new httpm.HttpClient('tool-cache');
+
+  const headers: ifm.IHeaders = {};
+  if (AUTH) {
+    headers.authorization = AUTH;
+  }
+
+  let url: string | null =
+    'https://api.github.com/repos/oracle/graalpython/releases';
+  const result: IGraalPyManifestRelease[] = [];
+  do {
+    const response: ifm.ITypedResponse<IGraalPyManifestRelease[]> =
+      await http.getJson(url, headers);
+    if (!response.result) {
+      throw new Error(
+        `Unable to retrieve the list of available GraalPy versions from '${url}'`
+      );
+    }
+    result.push(...response.result);
+    url = getNextPageUrl(response);
+  } while (url);
+
+  return result;
+}
+
+async function createGraalPySymlink(
+  graalpyBinaryPath: string,
+  graalpyVersion: string
+) {
+  const version = semver.coerce(graalpyVersion)!;
+  const pythonBinaryPostfix = semver.major(version);
+  const pythonMinor = semver.minor(version);
+  const graalpyMajorMinorBinaryPostfix = `${pythonBinaryPostfix}.${pythonMinor}`;
+  const binaryExtension = IS_WINDOWS ? '.exe' : '';
+
+  core.info('Creating symlinks...');
+  createSymlinkInFolder(
+    graalpyBinaryPath,
+    `graalpy${binaryExtension}`,
+    `python${pythonBinaryPostfix}${binaryExtension}`,
+    true
+  );
+
+  createSymlinkInFolder(
+    graalpyBinaryPath,
+    `graalpy${binaryExtension}`,
+    `python${binaryExtension}`,
+    true
+  );
+
+  createSymlinkInFolder(
+    graalpyBinaryPath,
+    `graalpy${binaryExtension}`,
+    `graalpy${graalpyMajorMinorBinaryPostfix}${binaryExtension}`,
+    true
+  );
+}
+
+async function installPip(pythonLocation: string) {
+  core.info(
+    "Installing pip (GraalPy doesn't update pip because it uses a patched version of pip)"
+  );
+  const pythonBinary = path.join(pythonLocation, 'python');
+  await exec.exec(`${pythonBinary} -m ensurepip --default-pip`);
+}
+
+export function graalPyTagToVersion(tag: string) {
+  const versionPattern = /.*-(\d+\.\d+\.\d+(?:\.\d+)?)((?:a|b|rc))?(\d*)?/;
+  const match = tag.match(versionPattern);
+  if (match && match[2]) {
+    return `${match[1]}-${match[2]}.${match[3]}`;
+  } else if (match) {
+    return match[1];
+  } else {
+    return tag.replace(/.*-/, '');
+  }
+}
+
+export function findRelease(
+  releases: IGraalPyManifestRelease[],
+  graalpyVersion: string,
+  architecture: string,
+  includePrerelease: boolean
+) {
+  const options = {includePrerelease: includePrerelease};
+  const filterReleases = releases.filter(item => {
+    const isVersionSatisfied = semver.satisfies(
+      graalPyTagToVersion(item.tag_name),
+      graalpyVersion,
+      options
+    );
+    return (
+      isVersionSatisfied && !!findAsset(item, architecture, process.platform)
+    );
+  });
+
+  if (!filterReleases.length) {
+    return null;
+  }
+
+  const sortedReleases = filterReleases.sort((previous, current) =>
+    semver.compare(
+      semver.coerce(graalPyTagToVersion(current.tag_name))!,
+      semver.coerce(graalPyTagToVersion(previous.tag_name))!
+    )
+  );
+
+  const foundRelease = sortedReleases[0];
+  const foundAsset = findAsset(foundRelease, architecture, process.platform);
+
+  return {
+    foundAsset,
+    resolvedGraalPyVersion: graalPyTagToVersion(foundRelease.tag_name)
+  };
+}
+
+export function toGraalPyPlatform(platform: string) {
+  switch (platform) {
+    case 'win32':
+      return 'windows';
+    case 'darwin':
+      return 'macos';
+  }
+  return platform;
+}
+
+export function toGraalPyArchitecture(architecture: string) {
+  switch (architecture) {
+    case 'x64':
+      return 'amd64';
+    case 'arm64':
+      return 'aarch64';
+  }
+  return architecture;
+}
+
+export function findAsset(
+  item: IGraalPyManifestRelease,
+  architecture: string,
+  platform: string
+) {
+  const graalpyArch = toGraalPyArchitecture(architecture);
+  const graalpyPlatform = toGraalPyPlatform(platform);
+  const found = item.assets.filter(
+    file =>
+      file.name.startsWith('graalpy') &&
+      file.name.endsWith(`-${graalpyPlatform}-${graalpyArch}.tar.gz`)
+  );
+  /*
+  In the future there could be more variants of GraalPy for a single release. Pick the shortest name, that one is the most likely to be the primary variant.
+  */
+  found.sort((f1, f2) => f1.name.length - f2.name.length);
+  return found[0];
+}

src/install-pypy.ts

@@ -13,7 +13,8 @@ import {
   IPyPyManifestRelease,
   createSymlinkInFolder,
   isNightlyKeyword,
-  writeExactPyPyVersionFile
+  writeExactPyPyVersionFile,
+  getBinaryDirectory
 } from './utils';
 
 export async function installPyPy(
@@ -94,7 +95,7 @@ export async function installPyPy(
 
     writeExactPyPyVersionFile(installDir, resolvedPyPyVersion);
 
-    const binaryPath = getPyPyBinaryPath(installDir);
+    const binaryPath = getBinaryDirectory(installDir);
     await createPyPySymlink(binaryPath, resolvedPythonVersion);
     await installPip(binaryPath);
 
@@ -237,15 +238,6 @@ export function findRelease(
   };
 }
 
-/** Get PyPy binary location from the tool of installation directory
- *  - On Linux and macOS, the Python interpreter is in 'bin'.
- *  - On Windows, it is in the installation root.
- */
-export function getPyPyBinaryPath(installDir: string) {
-  const _binDir = path.join(installDir, 'bin');
-  return IS_WINDOWS ? installDir : _binDir;
-}
-
 export function pypyVersionToSemantic(versionSpec: string) {
   const prereleaseVersion = /(\d+\.\d+\.\d+)((?:a|b|rc))(\d*)/g;
   return versionSpec.replace(prereleaseVersion, '$1-$2.$3');

src/setup-python.ts

@@ -1,6 +1,7 @@
 import * as core from '@actions/core';
 import * as finder from './find-python';
 import * as finderPyPy from './find-pypy';
+import * as finderGraalPy from './find-graalpy';
 import * as path from 'path';
 import * as os from 'os';
 import fs from 'fs';
@@ -17,6 +18,10 @@ function isPyPyVersion(versionSpec: string) {
   return versionSpec.startsWith('pypy');
 }
 
+function isGraalPyVersion(versionSpec: string) {
+  return versionSpec.startsWith('graalpy');
+}
+
 async function cacheDependencies(cache: string, pythonVersion: string) {
   const cacheDependencyPath =
     core.getInput('cache-dependency-path') || undefined;
@@ -106,6 +111,16 @@ async function run() {
           core.info(
             `Successfully set up PyPy ${installed.resolvedPyPyVersion} with Python (${installed.resolvedPythonVersion})`
           );
+        } else if (isGraalPyVersion(version)) {
+          const installed = await finderGraalPy.findGraalPyVersion(
+            version,
+            arch,
+            updateEnvironment,
+            checkLatest,
+            allowPreReleases
+          );
+          pythonVersion = `${installed}`;
+          core.info(`Successfully set up GraalPy ${installed}`);
         } else {
           if (version.startsWith('2')) {
             core.warning(

src/utils.ts

@@ -6,6 +6,7 @@ import * as path from 'path';
 import * as semver from 'semver';
 import * as toml from '@iarna/toml';
 import * as exec from '@actions/exec';
+import * as ifm from '@actions/http-client/interfaces';
 
 export const IS_WINDOWS = process.platform === 'win32';
 export const IS_LINUX = process.platform === 'linux';
@@ -29,6 +30,16 @@ export interface IPyPyManifestRelease {
   files: IPyPyManifestAsset[];
 }
 
+export interface IGraalPyManifestAsset {
+  name: string;
+  browser_download_url: string;
+}
+
+export interface IGraalPyManifestRelease {
+  tag_name: string;
+  assets: IGraalPyManifestAsset[];
+}
+
 /** create Symlinks for downloaded PyPy
  *  It should be executed only for downloaded versions in runtime, because
  *  toolcache versions have this setup.
@@ -266,3 +277,34 @@ export function getVersionInputFromFile(versionFile: string): string[] {
     return getVersionInputFromPlainFile(versionFile);
   }
 }
+
+/**
+ * Get the directory containing interpreter binary from installation directory of PyPy or GraalPy
+ *  - On Linux and macOS, the Python interpreter is in 'bin'.
+ *  - On Windows, it is in the installation root.
+ */
+export function getBinaryDirectory(installDir: string) {
+  return IS_WINDOWS ? installDir : path.join(installDir, 'bin');
+}
+
+/**
+ * Extract next page URL from a HTTP response "link" header. Such headers are used in GitHub APIs.
+ */
+export function getNextPageUrl<T>(response: ifm.ITypedResponse<T>) {
+  const responseHeaders = <ifm.IHeaders>response.headers;
+  const linkHeader = responseHeaders.link;
+  if (typeof linkHeader === 'string') {
+    for (const link of linkHeader.split(/\s*,\s*/)) {
+      const match = link.match(/<([^>]+)>(.*)/);
+      if (match) {
+        const url = match[1];
+        for (const param of match[2].split(/\s*;\s*/)) {
+          if (param.match(/rel="?next"?/)) {
+            return url;
+          }
+        }
+      }
+    }
+  }
+  return null;
+}