Implementation of python's caching (#266)

src/cache-distributions/cache-distributor.ts

@@ -0,0 +1,53 @@
+import * as cache from '@actions/cache';
+import * as core from '@actions/core';
+
+export enum State {
+  STATE_CACHE_PRIMARY_KEY = 'cache-primary-key',
+  CACHE_MATCHED_KEY = 'cache-matched-key',
+  CACHE_PATHS = 'cache-paths'
+}
+
+abstract class CacheDistributor {
+  protected CACHE_KEY_PREFIX = 'setup-python';
+  constructor(
+    protected packageManager: string,
+    protected cacheDependencyPath: string
+  ) {}
+
+  protected abstract getCacheGlobalDirectories(): Promise<string[]>;
+  protected abstract computeKeys(): Promise<{
+    primaryKey: string;
+    restoreKey: string[] | undefined;
+  }>;
+
+  public async restoreCache() {
+    const {primaryKey, restoreKey} = await this.computeKeys();
+    if (primaryKey.endsWith('-')) {
+      throw new Error(
+        `No file in ${process.cwd()} matched to [${this.cacheDependencyPath
+          .split('\n')
+          .join(',')}], make sure you have checked out the target repository`
+      );
+    }
+
+    const cachePath = await this.getCacheGlobalDirectories();
+
+    core.saveState(State.CACHE_PATHS, cachePath);
+    core.saveState(State.STATE_CACHE_PRIMARY_KEY, primaryKey);
+
+    const matchedKey = await cache.restoreCache(
+      cachePath,
+      primaryKey,
+      restoreKey
+    );
+
+    if (matchedKey) {
+      core.saveState(State.CACHE_MATCHED_KEY, matchedKey);
+      core.info(`Cache restored from key: ${matchedKey}`);
+    } else {
+      core.info(`${this.packageManager} cache is not found`);
+    }
+  }
+}
+
+export default CacheDistributor;

src/cache-distributions/cache-factory.ts

@@ -0,0 +1,22 @@
+import PipCache from './pip-cache';
+import PipenvCache from './pipenv-cache';
+
+export enum PackageManagers {
+  Pip = 'pip',
+  Pipenv = 'pipenv'
+}
+
+export function getCacheDistributor(
+  packageManager: string,
+  pythonVersion: string,
+  cacheDependencyPath: string | undefined
+) {
+  switch (packageManager) {
+    case PackageManagers.Pip:
+      return new PipCache(cacheDependencyPath);
+    case PackageManagers.Pipenv:
+      return new PipenvCache(pythonVersion, cacheDependencyPath);
+    default:
+      throw new Error(`Caching for '${packageManager}' is not supported`);
+  }
+}

src/cache-distributions/pip-cache.ts

@@ -0,0 +1,49 @@
+import * as glob from '@actions/glob';
+import * as core from '@actions/core';
+import * as exec from '@actions/exec';
+
+import * as path from 'path';
+import os from 'os';
+
+import CacheDistributor from './cache-distributor';
+
+class PipCache extends CacheDistributor {
+  constructor(cacheDependencyPath: string = '**/requirements.txt') {
+    super('pip', cacheDependencyPath);
+  }
+
+  protected async getCacheGlobalDirectories() {
+    const {stdout, stderr, exitCode} = await exec.getExecOutput(
+      'pip cache dir'
+    );
+
+    if (exitCode && stderr) {
+      throw new Error(
+        `Could not get cache folder path for pip package manager`
+      );
+    }
+
+    let resolvedPath = stdout.trim();
+
+    if (resolvedPath.includes('~')) {
+      resolvedPath = path.join(os.homedir(), resolvedPath.slice(1));
+    }
+
+    core.debug(`global cache directory path is ${resolvedPath}`);
+
+    return [resolvedPath];
+  }
+
+  protected async computeKeys() {
+    const hash = await glob.hashFiles(this.cacheDependencyPath);
+    const primaryKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-${this.packageManager}-${hash}`;
+    const restoreKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-${this.packageManager}`;
+
+    return {
+      primaryKey,
+      restoreKey: [restoreKey]
+    };
+  }
+}
+
+export default PipCache;

src/cache-distributions/pipenv-cache.ts

@@ -0,0 +1,44 @@
+import * as glob from '@actions/glob';
+import * as os from 'os';
+import * as path from 'path';
+import * as core from '@actions/core';
+
+import CacheDistributor from './cache-distributor';
+
+class PipenvCache extends CacheDistributor {
+  constructor(
+    private pythonVersion: string,
+    protected patterns: string = '**/Pipfile.lock'
+  ) {
+    super('pipenv', patterns);
+  }
+
+  protected async getCacheGlobalDirectories() {
+    let virtualEnvRelativePath;
+
+    // Default virtualenv directories are hardcoded,
+    // because pipenv is not preinstalled on hosted images and virtualenv is not created:
+    // https://github.com/pypa/pipenv/blob/1daaa0de9a0b00d386c6baeb809d8d4ee6795cfd/pipenv/utils.py#L1990-L2002
+    if (process.platform === 'win32') {
+      virtualEnvRelativePath = '.virtualenvs';
+    } else {
+      virtualEnvRelativePath = '.local/share/virtualenvs';
+    }
+    const resolvedPath = path.join(os.homedir(), virtualEnvRelativePath);
+    core.debug(`global cache directory path is ${resolvedPath}`);
+
+    return [resolvedPath];
+  }
+
+  protected async computeKeys() {
+    const hash = await glob.hashFiles(this.patterns);
+    const primaryKey = `${this.CACHE_KEY_PREFIX}-${process.env['RUNNER_OS']}-python-${this.pythonVersion}-${this.packageManager}-${hash}`;
+    const restoreKey = undefined;
+    return {
+      primaryKey,
+      restoreKey
+    };
+  }
+}
+
+export default PipenvCache;

src/cache-save.ts

@@ -0,0 +1,67 @@
+import * as core from '@actions/core';
+import * as cache from '@actions/cache';
+
+import fs from 'fs';
+import {State} from './cache-distributions/cache-distributor';
+
+export async function run() {
+  try {
+    const cache = core.getInput('cache');
+    if (cache) {
+      await saveCache(cache);
+    }
+  } catch (error) {
+    const err = error as Error;
+    core.setFailed(err.message);
+  }
+}
+
+async function saveCache(packageManager: string) {
+  const cachePaths = JSON.parse(core.getState(State.CACHE_PATHS)) as string[];
+
+  core.debug(`paths for caching are ${cachePaths.join(', ')}`);
+
+  if (!isCacheDirectoryExists(cachePaths)) {
+    throw new Error(
+      `Cache folder path is retrieved for ${packageManager} but doesn't exist on disk: ${cachePaths.join(
+        ', '
+      )}`
+    );
+  }
+
+  const primaryKey = core.getState(State.STATE_CACHE_PRIMARY_KEY);
+  const matchedKey = core.getState(State.CACHE_MATCHED_KEY);
+
+  if (!primaryKey) {
+    core.warning('Error retrieving key from state.');
+    return;
+  } else if (matchedKey === primaryKey) {
+    // no change in target directories
+    core.info(
+      `Cache hit occurred on the primary key ${primaryKey}, not saving cache.`
+    );
+    return;
+  }
+
+  try {
+    await cache.saveCache(cachePaths, primaryKey);
+    core.info(`Cache saved with the key: ${primaryKey}`);
+  } catch (error) {
+    const err = error as Error;
+    if (err.name === cache.ReserveCacheError.name) {
+      core.info(err.message);
+    } else {
+      throw error;
+    }
+  }
+}
+
+function isCacheDirectoryExists(cacheDirectory: string[]) {
+  const result = cacheDirectory.reduce((previousValue, currentValue) => {
+    return previousValue || fs.existsSync(currentValue);
+  }, false);
+
+  return result;
+}
+
+run();

src/install-python.ts

@@ -3,7 +3,7 @@ import * as core from '@actions/core';
 import * as tc from '@actions/tool-cache';
 import * as exec from '@actions/exec';
 import {ExecOptions} from '@actions/exec/lib/interfaces';
-import {IS_WINDOWS, IS_LINUX} from './utils';
+import {IS_WINDOWS, IS_LINUX, isGhes} from './utils';
 
 const TOKEN = core.getInput('token');
 const AUTH = !TOKEN || isGhes() ? undefined : `token ${TOKEN}`;
@@ -71,10 +71,3 @@ export async function installCpythonFromRelease(release: tc.IToolRelease) {
   core.info('Execute installation script');
   await installPython(pythonExtractedFolder);
 }
-
-function isGhes(): boolean {
-  const ghUrl = new URL(
-    process.env['GITHUB_SERVER_URL'] || 'https://github.com'
-  );
-  return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM';
-}

src/setup-python.ts

@@ -3,32 +3,54 @@ import * as finder from './find-python';
 import * as finderPyPy from './find-pypy';
 import * as path from 'path';
 import * as os from 'os';
+import {getCacheDistributor} from './cache-distributions/cache-factory';
+import {isGhes} from './utils';
 
 function isPyPyVersion(versionSpec: string) {
   return versionSpec.startsWith('pypy-');
 }
 
+async function cacheDependencies(cache: string, pythonVersion: string) {
+  if (isGhes()) {
+    throw new Error('Caching is not supported on GHES');
+  }
+  const cacheDependencyPath =
+    core.getInput('cache-dependency-path') || undefined;
+  const cacheDistributor = getCacheDistributor(
+    cache,
+    pythonVersion,
+    cacheDependencyPath
+  );
+  await cacheDistributor.restoreCache();
+}
+
 async function run() {
   try {
-    let version = core.getInput('python-version');
+    const version = core.getInput('python-version');
     if (version) {
+      let pythonVersion: string;
       const arch: string = core.getInput('architecture') || os.arch();
       if (isPyPyVersion(version)) {
         const installed = await finderPyPy.findPyPyVersion(version, arch);
+        pythonVersion = `${installed.resolvedPyPyVersion}-${installed.resolvedPythonVersion}`;
         core.info(
           `Successfully setup PyPy ${installed.resolvedPyPyVersion} with Python (${installed.resolvedPythonVersion})`
         );
       } else {
         const installed = await finder.findPythonVersion(version, arch);
-        core.info(
-          `Successfully setup ${installed.impl} (${installed.version})`
-        );
+        pythonVersion = installed.version;
+        core.info(`Successfully setup ${installed.impl} (${pythonVersion})`);
+      }
+
+      const cache = core.getInput('cache');
+      if (cache) {
+        await cacheDependencies(cache, pythonVersion);
       }
     }
-    const matchersPath = path.join(__dirname, '..', '.github');
+    const matchersPath = path.join(__dirname, '../..', '.github');
     core.info(`##[add-matcher]${path.join(matchersPath, 'python.json')}`);
   } catch (err) {
-    core.setFailed(err.message);
+    core.setFailed((err as Error).message);
   }
 }
 

src/utils.ts

@@ -92,3 +92,10 @@ export function validatePythonVersionFormatForPyPy(version: string) {
   const re = /^\d+\.\d+$/;
   return re.test(version);
 }
+
+export function isGhes(): boolean {
+  const ghUrl = new URL(
+    process.env['GITHUB_SERVER_URL'] || 'https://github.com'
+  );
+  return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM';
+}