2022-04-10 08:40:39 +00:00
|
|
|
import dataclasses
|
|
|
|
import gzip
|
|
|
|
import io
|
|
|
|
import json
|
|
|
|
from pathlib import Path
|
|
|
|
import sys
|
|
|
|
from typing import Iterator
|
|
|
|
import urllib.parse
|
|
|
|
import urllib.request
|
|
|
|
import zipfile
|
|
|
|
|
|
|
|
|
|
|
|
@dataclasses.dataclass
|
|
|
|
class Artifact:
|
|
|
|
repo: str
|
|
|
|
run_id: int
|
|
|
|
name: str
|
|
|
|
download_url: str
|
|
|
|
|
|
|
|
@property
|
Bump linter versions (#188)
The isort we had has some weird poetry issue, I figured I might as well
bump the other linters at the same time
```
[INFO] Installing environment for https://github.com/PyCQA/isort.
[INFO] Once installed this environment will be reused.
[INFO] This may take a few minutes...
An unexpected error has occurred: CalledProcessError: command: ('/home/runner/.cache/pre-commit/repo0m3eczdf/py_env-python3.7/bin/python', '-mpip', 'install', '.')
return code: 1
stdout:
Processing /home/runner/.cache/pre-commit/repo0m3eczdf
Installing build dependencies: started
Installing build dependencies: finished with status 'done'
Getting requirements to build wheel: started
Getting requirements to build wheel: finished with status 'done'
Preparing metadata (pyproject.toml): started
Preparing metadata (pyproject.toml): finished with status 'error'
stderr:
error: subprocess-exited-with-error
× Preparing metadata (pyproject.toml) did not run successfully.
│ exit code: 1
╰─> [14 lines of output]
Traceback (most recent call last):
File "/home/runner/.cache/pre-commit/repo0m3eczdf/py_env-python3.7/lib/python3.7/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 353, in <module>
main()
File "/home/runner/.cache/pre-commit/repo0m3eczdf/py_env-python3.7/lib/python3.7/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 335, in main
json_out['return_val'] = hook(**hook_input['kwargs'])
File "/home/runner/.cache/pre-commit/repo0m3eczdf/py_env-python3.7/lib/python3.7/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 149, in prepare_metadata_for_build_wheel
return hook(metadata_directory, config_settings)
File "/tmp/pip-build-env-beaf5dxh/overlay/lib/python3.7/site-packages/poetry/core/masonry/api.py", line 40, in prepare_metadata_for_build_wheel
poetry = Factory().create_poetry(Path(".").resolve(), with_groups=False)
File "/tmp/pip-build-env-beaf5dxh/overlay/lib/python3.7/site-packages/poetry/core/factory.py", line 57, in create_poetry
raise RuntimeError("The Poetry configuration is invalid:\n" + message)
RuntimeError: The Poetry configuration is invalid:
- [extras.pipfile_deprecated_finder.2] 'pip-shims<=0.3.4' does not match '^[a-zA-Z-_.0-9]+$'
[end of output]
note: This error originates from a subprocess, and is likely not a problem with pip.
error: metadata-generation-failed
× Encountered error while generating package metadata.
╰─> See above for output.
note: This is an issue with the package mentioned above, not pip.
hint: See above for details.
```
2023-03-04 09:51:40 +00:00
|
|
|
def public_download_url(self) -> str:
|
2022-04-10 08:40:39 +00:00
|
|
|
# GitHub API is not available publicly for artifacts, we need to use
|
|
|
|
# a third-party proxy to access it...
|
|
|
|
name = urllib.parse.quote(self.name)
|
|
|
|
return f"https://nightly.link/{repo}/actions/runs/{self.run_id}/{name}.zip"
|
|
|
|
|
|
|
|
|
|
|
|
def iter_run_artifacts(repo: str, run_id: int) -> Iterator[Artifact]:
|
|
|
|
request = urllib.request.Request(
|
2022-04-12 16:33:02 +00:00
|
|
|
f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/artifacts"
|
|
|
|
"?per_page=100",
|
2022-04-10 08:40:39 +00:00
|
|
|
headers={"Accept": "application/vnd.github.v3+json"},
|
|
|
|
)
|
|
|
|
|
|
|
|
response = urllib.request.urlopen(request)
|
|
|
|
|
|
|
|
for artifact in json.load(response)["artifacts"]:
|
2022-04-12 16:33:02 +00:00
|
|
|
if not artifact["name"].startswith(("pytest-results_", "pytest results ")):
|
2022-04-10 08:40:39 +00:00
|
|
|
continue
|
|
|
|
if artifact["expired"]:
|
|
|
|
continue
|
|
|
|
yield Artifact(
|
|
|
|
repo=repo,
|
|
|
|
run_id=run_id,
|
|
|
|
name=artifact["name"],
|
|
|
|
download_url=artifact["archive_download_url"],
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def download_artifact(output_name: Path, url: str) -> None:
|
|
|
|
if output_name.exists():
|
|
|
|
return
|
|
|
|
response = urllib.request.urlopen(url)
|
|
|
|
archive_bytes = response.read() # Can't stream it, it's a ZIP
|
|
|
|
with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive:
|
|
|
|
with archive.open("pytest.xml") as input_fd:
|
|
|
|
pytest_xml = input_fd.read()
|
|
|
|
|
|
|
|
tmp_output_path = output_name.with_suffix(".tmp")
|
|
|
|
with gzip.open(tmp_output_path, "wb") as output_fd:
|
|
|
|
output_fd.write(pytest_xml)
|
|
|
|
|
|
|
|
# Atomically write to the output path, so that we don't write partial files in case
|
|
|
|
# the download process is interrupted
|
|
|
|
tmp_output_path.rename(output_name)
|
|
|
|
|
|
|
|
|
|
|
|
def main(output_dir: Path, repo: str, run_id: int) -> int:
|
|
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
run_path = output_dir / str(run_id)
|
|
|
|
run_path.mkdir(exist_ok=True)
|
|
|
|
|
|
|
|
for artifact in iter_run_artifacts(repo, run_id):
|
|
|
|
artifact_path = run_path / artifact.name / "pytest.xml.gz"
|
|
|
|
artifact_path.parent.mkdir(exist_ok=True)
|
|
|
|
try:
|
|
|
|
download_artifact(artifact_path, artifact.download_url)
|
|
|
|
except Exception:
|
|
|
|
download_artifact(artifact_path, artifact.public_download_url)
|
|
|
|
print("downloaded", artifact.name)
|
|
|
|
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
(_, output_path, repo, run_id) = sys.argv
|
|
|
|
exit(main(Path(output_path), repo, int(run_id)))
|