From 4ff23b4eab946d5fb31f07aeacddc5d84f88369f Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 2 Dec 2019 15:18:54 +0100 Subject: [PATCH] Support for conda as a language --- azure-pipelines.yml | 3 + pre_commit/languages/all.py | 2 + pre_commit/languages/conda.py | 66 +++++++++++++++++++ .../resources/empty_template_environment.yml | 9 +++ pre_commit/store.py | 2 +- .../conda_hooks_repo/.pre-commit-hooks.yaml | 10 +++ .../conda_hooks_repo/environment.yml | 6 ++ tests/repository_test.py | 40 +++++++++++ 8 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 pre_commit/languages/conda.py create mode 100644 pre_commit/resources/empty_template_environment.yml create mode 100644 testing/resources/conda_hooks_repo/.pre-commit-hooks.yaml create mode 100644 testing/resources/conda_hooks_repo/environment.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5b57e89..9d61eb6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,6 +22,9 @@ jobs: COVERAGE_IGNORE_WINDOWS: '# pragma: windows no cover' TOX_TESTENV_PASSENV: COVERAGE_IGNORE_WINDOWS TEMP: C:\Temp # remove when dropping python2 + pre_test: + - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" + displayName: Add conda to PATH - template: job--python-tox.yml@asottile parameters: toxenvs: [py37] diff --git a/pre_commit/languages/all.py b/pre_commit/languages/all.py index 051656b..3d139d9 100644 --- a/pre_commit/languages/all.py +++ b/pre_commit/languages/all.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +from pre_commit.languages import conda from pre_commit.languages import docker from pre_commit.languages import docker_image from pre_commit.languages import fail @@ -52,6 +53,7 @@ from pre_commit.languages import system # """ languages = { + 'conda': conda, 'docker': docker, 'docker_image': docker_image, 'fail': fail, diff --git a/pre_commit/languages/conda.py b/pre_commit/languages/conda.py new file mode 100644 index 0000000..a89d6c9 --- /dev/null +++ b/pre_commit/languages/conda.py @@ -0,0 +1,66 @@ +import contextlib +import os + +from pre_commit.envcontext import envcontext +from pre_commit.envcontext import UNSET +from pre_commit.envcontext import Var +from pre_commit.languages import helpers +from pre_commit.util import clean_path_on_failure +from pre_commit.util import cmd_output_b + +ENVIRONMENT_DIR = 'conda' +get_default_version = helpers.basic_get_default_version +healthy = helpers.basic_healthy + + +def get_env_patch(env): + # On non-windows systems executable live in $CONDA_PREFIX/bin, on Windows + # they can be in $CONDA_PREFIX/bin, $CONDA_PREFIX/Library/bin, + # $CONDA_PREFIX/Scripts and $CONDA_PREFIX. Whereas the latter only + # seems to be used for python.exe. + path = (os.path.join(env, 'bin'), os.pathsep, Var('PATH')) + if os.name == 'nt': # pragma: no cover (platform specific) + path = (env, os.pathsep) + path + path = (os.path.join(env, 'Scripts'), os.pathsep) + path + path = (os.path.join(env, 'Library', 'bin'), os.pathsep) + path + + return ( + ('PYTHONHOME', UNSET), + ('VIRTUAL_ENV', UNSET), + ('CONDA_PREFIX', env), + ('PATH', path), + ) + + +@contextlib.contextmanager +def in_env(prefix, language_version): + directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) + envdir = prefix.path(directory) + with envcontext(get_env_patch(envdir)): + yield + + +def install_environment(prefix, version, additional_dependencies): + helpers.assert_version_default('conda', version) + directory = helpers.environment_dir(ENVIRONMENT_DIR, version) + + env_dir = prefix.path(directory) + with clean_path_on_failure(env_dir): + cmd_output_b( + 'conda', 'env', 'create', '-p', env_dir, '--file', + 'environment.yml', cwd=prefix.prefix_dir, + ) + if additional_dependencies: + cmd_output_b( + 'conda', 'install', '-p', env_dir, *additional_dependencies, + cwd=prefix.prefix_dir + ) + + +def run_hook(hook, file_args, color): + # TODO: Some rare commands need to be run using `conda run` but mostly we + # can run them withot which is much quicker and produces a better + # output. + # cmd = ('conda', 'run', '-p', env_dir) + hook.cmd + with in_env(hook.prefix, hook.language_version): + return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/pre_commit/resources/empty_template_environment.yml b/pre_commit/resources/empty_template_environment.yml new file mode 100644 index 0000000..0f29f0c --- /dev/null +++ b/pre_commit/resources/empty_template_environment.yml @@ -0,0 +1,9 @@ +channels: + - conda-forge + - defaults +dependencies: + # This cannot be empty as otherwise no environment will be created. + # We're using openssl here as it is available on all system and will + # most likely be always installed anyways. + # See https://github.com/conda/conda/issues/9487 + - openssl diff --git a/pre_commit/store.py b/pre_commit/store.py index 2f15924..d9b674b 100644 --- a/pre_commit/store.py +++ b/pre_commit/store.py @@ -173,7 +173,7 @@ class Store(object): LOCAL_RESOURCES = ( 'Cargo.toml', 'main.go', 'main.rs', '.npmignore', 'package.json', - 'pre_commit_dummy_package.gemspec', 'setup.py', + 'pre_commit_dummy_package.gemspec', 'setup.py', 'environment.yml', ) def make_local(self, deps): diff --git a/testing/resources/conda_hooks_repo/.pre-commit-hooks.yaml b/testing/resources/conda_hooks_repo/.pre-commit-hooks.yaml new file mode 100644 index 0000000..a0d274c --- /dev/null +++ b/testing/resources/conda_hooks_repo/.pre-commit-hooks.yaml @@ -0,0 +1,10 @@ +- id: sys-exec + name: sys-exec + entry: python -c 'import os; import sys; print(sys.executable.split(os.path.sep)[-2]) if os.name == "nt" else print(sys.executable.split(os.path.sep)[-3])' + language: conda + files: \.py$ +- id: additional-deps + name: additional-deps + entry: python + language: conda + files: \.py$ diff --git a/testing/resources/conda_hooks_repo/environment.yml b/testing/resources/conda_hooks_repo/environment.yml new file mode 100644 index 0000000..e23c079 --- /dev/null +++ b/testing/resources/conda_hooks_repo/environment.yml @@ -0,0 +1,6 @@ +channels: + - conda-forge + - defaults +dependencies: + - python + - pip diff --git a/tests/repository_test.py b/tests/repository_test.py index 85afa90..5f2ed1c 100644 --- a/tests/repository_test.py +++ b/tests/repository_test.py @@ -79,6 +79,46 @@ def _test_hook_repo( assert _norm_out(out) == expected +def test_conda_hook(tempdir_factory, store): + _test_hook_repo( + tempdir_factory, store, 'conda_hooks_repo', + 'sys-exec', [os.devnull], + b'conda-default\n', + ) + + +def test_conda_with_additional_dependencies_hook(tempdir_factory, store): + _test_hook_repo( + tempdir_factory, store, 'conda_hooks_repo', + 'additional-deps', [os.devnull], + b'OK\n', + config_kwargs={ + 'hooks': [{ + 'id': 'additional-deps', + 'args': ['-c', 'import mccabe; print("OK")'], + 'additional_dependencies': ['mccabe'], + }], + }, + ) + + +def test_local_conda_additional_dependencies(store): + config = { + 'repo': 'local', + 'hooks': [{ + 'id': 'local-conda', + 'name': 'local-conda', + 'entry': 'python', + 'language': 'conda', + 'args': ['-c', 'import mccabe; print("OK")'], + 'additional_dependencies': ['mccabe'], + }], + } + ret, out = _get_hook(config, store, 'local-conda').run((), color=False) + assert ret == 0 + assert _norm_out(out) == b'OK\n' + + def test_python_hook(tempdir_factory, store): _test_hook_repo( tempdir_factory, store, 'python_hooks_repo',