Align Windows 3.7 methodology and later with venv (#1976)

Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net>
This commit is contained in:
Bernát Gábor 2020-10-12 11:25:36 +01:00 committed by GitHub
parent 2b0bbbabb4
commit ced984abfd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 207 additions and 157 deletions

View file

@ -16,7 +16,7 @@ repos:
hooks:
- id: pyupgrade
- repo: https://github.com/PyCQA/isort
rev: 5.5.4
rev: 5.6.3
hooks:
- id: isort
- repo: https://github.com/ambv/black
@ -36,11 +36,12 @@ repos:
hooks:
- id: rst-backticks
- repo: https://github.com/tox-dev/tox-ini-fmt
rev: "0.2.0"
rev: "0.5.0"
hooks:
- id: tox-ini-fmt
args: ["-p", "fix_lint"]
- repo: https://github.com/asottile/setup-cfg-fmt
rev: v1.11.0
rev: v1.15.0
hooks:
- id: setup-cfg-fmt
args: [--min-py3-version, "3.4"]

View file

@ -0,0 +1,2 @@
Align with venv module when creating virtual environments with builtin creator on Windows 3.7 and later
- by :user:`gaborbernat`.

View file

@ -5,7 +5,9 @@ long_description = file: README.md
long_description_content_type = text/markdown
url = https://virtualenv.pypa.io/
author = Bernat Gabor
author_email = gaborjbernat@gmail.com
maintainer = Bernat Gabor
maintainer_email = gaborjbernat@gmail.com
license = MIT
license_file = LICENSE
platforms = any
@ -24,14 +26,13 @@ classifiers =
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: Implementation :: CPython
Programming Language :: Python :: Implementation :: PyPy
Topic :: Software Development :: Libraries
Topic :: Software Development :: Testing
Topic :: Utilities
author-email = gaborjbernat@gmail.com
keywords = virtual, environments, isolated
maintainer-email = gaborjbernat@gmail.com
project_urls =
Source=https://github.com/pypa/virtualenv
Tracker=https://github.com/pypa/virtualenv/issues

View file

@ -6,6 +6,7 @@ from collections import OrderedDict
from six import add_metaclass
from virtualenv.create.describe import PosixSupports, WindowsSupports
from virtualenv.create.via_global_ref.builtin.ref import RefMust, RefWhen
from virtualenv.util.path import Path
from ..via_global_self_do import ViaGlobalRefVirtualenvBuiltin
@ -33,19 +34,26 @@ class CPythonPosix(CPython, PosixSupports):
targets = OrderedDict(
(i, None) for i in ["python", "python{}".format(major), "python{}.{}".format(major, minor), host_exe.name]
)
yield host_exe, list(targets.keys())
must = RefMust.COPY if interpreter.version_info.major == 2 else RefMust.NA
yield host_exe, list(targets.keys()), must, RefWhen.ANY
@add_metaclass(ABCMeta)
class CPythonWindows(CPython, WindowsSupports):
@classmethod
def _executables(cls, interpreter):
host = Path(interpreter.system_executable)
executables = cls._win_executables(Path(interpreter.system_executable), interpreter, RefWhen.ANY)
for src, targets, must, when in executables:
yield src, targets, must, when
@classmethod
def _win_executables(cls, host, interpreter, when):
must = RefMust.COPY if interpreter.version_info.major == 2 else RefMust.NA
for path in (host.parent / n for n in {"python.exe", host.name}):
yield host, [path.name]
yield host, [path.name], must, when
# for more info on pythonw.exe see https://stackoverflow.com/a/30313091
python_w = host.parent / "pythonw.exe"
yield python_w, [python_w.name]
yield python_w, [python_w.name], must, when
def is_mac_os_framework(interpreter):

View file

@ -1,12 +1,13 @@
from __future__ import absolute_import, unicode_literals
import abc
from itertools import chain
from textwrap import dedent
from six import add_metaclass
from virtualenv.create.describe import Python3Supports
from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest, RefMust, RefWhen
from virtualenv.create.via_global_ref.store import is_store_python
from virtualenv.util.path import Path
@ -55,8 +56,29 @@ class CPython3Windows(CPythonWindows, CPython3):
def sources(cls, interpreter):
for src in super(CPython3Windows, cls).sources(interpreter):
yield src
for src in cls.include_dll_and_pyd(interpreter):
yield src
if cls.venv_37p(interpreter):
for dll in (i for i in Path(interpreter.system_executable).parent.iterdir() if i.suffix == ".dll"):
yield PathRefToDest(dll, cls.to_bin, RefMust.SYMLINK, RefWhen.SYMLINK)
else:
for src in cls.include_dll_and_pyd(interpreter):
yield src
@classmethod
def _executables(cls, interpreter):
system_exe = Path(interpreter.system_executable)
if cls.venv_37p(interpreter):
# starting with CPython 3.7 Windows ships with a venvlauncher.exe that avoids the need for dll/pyd copies
launcher = Path(interpreter.system_stdlib) / "venv" / "scripts" / "nt" / "python.exe"
executables = cls._win_executables(launcher, interpreter, RefWhen.COPY)
executables = chain(executables, cls._win_executables(system_exe, interpreter, RefWhen.SYMLINK))
else:
executables = cls._win_executables(system_exe, interpreter, RefWhen.ANY)
for src, targets, must, when in executables:
yield src, targets, must, when
@staticmethod
def venv_37p(interpreter):
return interpreter.version_info.minor > 6
@classmethod
def include_dll_and_pyd(cls, interpreter):

View file

@ -9,7 +9,7 @@ from textwrap import dedent
from six import add_metaclass
from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest
from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest, RefMust
from virtualenv.util.path import Path
from virtualenv.util.six import ensure_text
@ -29,7 +29,8 @@ class CPythonmacOsFramework(CPython):
for src in super(CPythonmacOsFramework, cls).sources(interpreter):
yield src
# add a symlink to the host python image
ref = PathRefToDest(cls.image_ref(interpreter), dest=lambda self, _: self.dest / ".Python", must_symlink=True)
exe = cls.image_ref(interpreter)
ref = PathRefToDest(exe, dest=lambda self, _: self.dest / ".Python", must=RefMust.SYMLINK)
yield ref
def create(self):
@ -40,7 +41,7 @@ class CPythonmacOsFramework(CPython):
current = self.current_mach_o_image_path()
for src in self._sources:
if isinstance(src, ExePathRefToDest):
if src.must_copy or not self.symlinks:
if src.must == RefMust.COPY or not self.symlinks:
exes = [self.bin_dir / src.base]
if not self.symlinks:
exes.extend(self.bin_dir / a for a in src.aliases)
@ -49,12 +50,12 @@ class CPythonmacOsFramework(CPython):
@classmethod
def _executables(cls, interpreter):
for _, targets in super(CPythonmacOsFramework, cls)._executables(interpreter):
for _, targets, must, when in super(CPythonmacOsFramework, cls)._executables(interpreter):
# Make sure we use the embedded interpreter inside the framework, even if sys.executable points to the
# stub executable in ${sys.prefix}/bin.
# See http://groups.google.com/group/python-virtualenv/browse_thread/thread/17cab2f85da75951
fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python"
yield fixed_host_exe, targets
yield fixed_host_exe, targets, must, when
@abstractmethod
def current_mach_o_image_path(self):

View file

@ -4,7 +4,7 @@ import abc
from six import add_metaclass
from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest, RefMust, RefWhen
from virtualenv.util.path import Path
from ..via_global_self_do import ViaGlobalRefVirtualenvBuiltin
@ -20,7 +20,8 @@ class PyPy(ViaGlobalRefVirtualenvBuiltin):
def _executables(cls, interpreter):
host = Path(interpreter.system_executable)
targets = sorted("{}{}".format(name, PyPy.suffix) for name in cls.exe_names(interpreter))
yield host, targets
must = RefMust.COPY if interpreter.version_info.major == 2 else RefMust.NA
yield host, targets, must, RefWhen.ANY
@classmethod
def exe_names(cls, interpreter):

View file

@ -17,6 +17,18 @@ from virtualenv.util.path import copy, make_exe, symlink
from virtualenv.util.six import ensure_text
class RefMust(object):
NA = "NA"
COPY = "copy"
SYMLINK = "symlink"
class RefWhen(object):
ANY = "ANY"
COPY = "copy"
SYMLINK = "symlink"
@add_metaclass(ABCMeta)
class PathRef(object):
"""Base class that checks if a file reference can be symlink/copied"""
@ -24,9 +36,9 @@ class PathRef(object):
FS_SUPPORTS_SYMLINK = fs_supports_symlink()
FS_CASE_SENSITIVE = fs_is_case_sensitive()
def __init__(self, src, must_symlink, must_copy):
self.must_symlink = must_symlink
self.must_copy = must_copy
def __init__(self, src, must=RefMust.NA, when=RefWhen.ANY):
self.must = must
self.when = when
self.src = src
try:
self.exists = src.exists()
@ -35,8 +47,6 @@ class PathRef(object):
self._can_read = None if self.exists else False
self._can_copy = None if self.exists else False
self._can_symlink = None if self.exists else False
if self.must_copy is True and self.must_symlink is True:
raise ValueError("can copy and symlink at the same time")
def __repr__(self):
return "{}(src={})".format(self.__class__.__name__, self.src)
@ -57,7 +67,7 @@ class PathRef(object):
@property
def can_copy(self):
if self._can_copy is None:
if self.must_symlink:
if self.must == RefMust.SYMLINK:
self._can_copy = self.can_symlink
else:
self._can_copy = self.can_read
@ -66,7 +76,7 @@ class PathRef(object):
@property
def can_symlink(self):
if self._can_symlink is None:
if self.must_copy:
if self.must == RefMust.COPY:
self._can_symlink = self.can_copy
else:
self._can_symlink = self.FS_SUPPORTS_SYMLINK and self.can_read
@ -77,9 +87,9 @@ class PathRef(object):
raise NotImplementedError
def method(self, symlinks):
if self.must_symlink:
if self.must == RefMust.SYMLINK:
return symlink
if self.must_copy:
if self.must == RefMust.COPY:
return copy
return symlink if symlinks else copy
@ -88,8 +98,8 @@ class PathRef(object):
class ExePathRef(PathRef):
"""Base class that checks if a executable can be references via symlink/copy"""
def __init__(self, src, must_symlink, must_copy):
super(ExePathRef, self).__init__(src, must_symlink, must_copy)
def __init__(self, src, must=RefMust.NA, when=RefWhen.ANY):
super(ExePathRef, self).__init__(src, must, when)
self._can_run = None
@property
@ -114,8 +124,8 @@ class ExePathRef(PathRef):
class PathRefToDest(PathRef):
"""Link a path on the file system"""
def __init__(self, src, dest, must_symlink=False, must_copy=False):
super(PathRefToDest, self).__init__(src, must_symlink, must_copy)
def __init__(self, src, dest, must=RefMust.NA, when=RefWhen.ANY):
super(PathRefToDest, self).__init__(src, must, when)
self.dest = dest
def run(self, creator, symlinks):
@ -131,15 +141,14 @@ class PathRefToDest(PathRef):
class ExePathRefToDest(PathRefToDest, ExePathRef):
"""Link a exe path on the file system"""
def __init__(self, src, targets, dest, must_symlink=False, must_copy=False):
ExePathRef.__init__(self, src, must_symlink, must_copy)
PathRefToDest.__init__(self, src, dest, must_symlink, must_copy)
def __init__(self, src, targets, dest, must=RefMust.NA, when=RefWhen.ANY):
ExePathRef.__init__(self, src, must, when)
PathRefToDest.__init__(self, src, dest, must, when)
if not self.FS_CASE_SENSITIVE:
targets = list(OrderedDict((i.lower(), None) for i in targets).keys())
self.base = targets[0]
self.aliases = targets[1:]
self.dest = dest
self.must_copy = must_copy
def run(self, creator, symlinks):
bin_dir = self.dest(creator, self.src).parent

View file

@ -4,7 +4,7 @@ from abc import ABCMeta
from six import add_metaclass
from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest
from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, RefMust
from virtualenv.util.path import ensure_dir
from ..api import ViaGlobalRefApi, ViaGlobalRefMeta
@ -27,27 +27,37 @@ class ViaGlobalRefVirtualenvBuiltin(ViaGlobalRefApi, VirtualenvBuiltin):
def can_create(cls, interpreter):
"""By default all built-in methods assume that if we can describe it we can create it"""
# first we must be able to describe it
if cls.can_describe(interpreter):
meta = cls.setup_meta(interpreter)
if meta is not None and meta:
for src in cls.sources(interpreter):
if src.exists:
if meta.can_copy and not src.can_copy:
meta.copy_error = "cannot copy {}".format(src)
if meta.can_symlink and not src.can_symlink:
meta.symlink_error = "cannot symlink {}".format(src)
if not meta.can_copy and not meta.can_symlink:
meta.error = "neither copy or symlink supported, copy: {} symlink: {}".format(
meta.copy_error,
meta.symlink_error,
)
else:
meta.error = "missing required file {}".format(src)
if meta.error:
break
meta.sources.append(src)
return meta
return None
if not cls.can_describe(interpreter):
return None
meta = cls.setup_meta(interpreter)
if meta is not None and meta:
cls._sources_can_be_applied(interpreter, meta)
return meta
@classmethod
def _sources_can_be_applied(cls, interpreter, meta):
for src in cls.sources(interpreter):
if src.exists:
if meta.can_copy and not src.can_copy:
meta.copy_error = "cannot copy {}".format(src)
if meta.can_symlink and not src.can_symlink:
meta.symlink_error = "cannot symlink {}".format(src)
else:
msg = "missing required file {}".format(src)
if src.when == RefMust.NA:
meta.error = msg
elif src.when == RefMust.COPY:
meta.copy_error = msg
elif src.when == RefMust.SYMLINK:
meta.symlink_error = msg
if not meta.can_copy and not meta.can_symlink:
meta.error = "neither copy or symlink supported, copy: {} symlink: {}".format(
meta.copy_error,
meta.symlink_error,
)
if meta.error:
break
meta.sources.append(src)
@classmethod
def setup_meta(cls, interpreter):
@ -55,9 +65,8 @@ class ViaGlobalRefVirtualenvBuiltin(ViaGlobalRefApi, VirtualenvBuiltin):
@classmethod
def sources(cls, interpreter):
is_py2 = interpreter.version_info.major == 2
for host_exe, targets in cls._executables(interpreter):
yield ExePathRefToDest(host_exe, dest=cls.to_bin, targets=targets, must_copy=is_py2)
for host_exe, targets, must, when in cls._executables(interpreter):
yield ExePathRefToDest(host_exe, dest=cls.to_bin, targets=targets, must=must, when=when)
def to_bin(self, src):
return self.bin_dir / src.name

View file

@ -18,14 +18,14 @@ class CreatorSelector(ComponentBuilder):
@classmethod
def for_interpreter(cls, interpreter):
key_to_class, key_to_meta, builtin_key, describe = OrderedDict(), {}, None, None
errored = defaultdict(list)
errors = defaultdict(list)
for key, creator_class in cls.options("virtualenv.create").items():
if key == "builtin":
raise RuntimeError("builtin creator is a reserved name")
meta = creator_class.can_create(interpreter)
if meta:
if meta.error:
errored[meta.error].append(creator_class)
errors[meta.error].append(creator_class)
else:
if "builtin" not in key_to_class and issubclass(creator_class, VirtualenvBuiltin):
builtin_key = key
@ -36,12 +36,9 @@ class CreatorSelector(ComponentBuilder):
if describe is None and issubclass(creator_class, Describe) and creator_class.can_describe(interpreter):
describe = creator_class
if not key_to_meta:
if errored:
raise RuntimeError(
"\n".join(
"{} for creators {}".format(k, ", ".join(i.__name__ for i in v)) for k, v in errored.items()
),
)
if errors:
rows = ["{} for creators {}".format(k, ", ".join(i.__name__ for i in v)) for k, v in errors.items()]
raise RuntimeError("\n".join(rows))
else:
raise RuntimeError("No virtualenv implementation for {}".format(interpreter))
return CreatorInfo(

175
tox.ini
View file

@ -1,17 +1,18 @@
[tox]
envlist =
py38
py37
py36
py35
py34
py27
fix_lint
pypy3
pypy
coverage
readme
docs
fix_lint
py39
py38
py37
py36
py35
py34
py27
pypy3
pypy2
coverage
readme
docs
isolated_build = true
skip_missing_interpreters = true
minversion = 3.14
@ -19,142 +20,140 @@ minversion = 3.14
[testenv]
description = run tests with {basepython}
passenv =
CI_RUN
HOME
PIP_*
PYTEST_*
TERM
http_proxy
https_proxy
no_proxy
CI_RUN
HOME
PIP_*
PYTEST_*
TERM
setenv =
COVERAGE_FILE = {toxworkdir}/.coverage.{envname}
COVERAGE_PROCESS_START = {toxinidir}/.coveragerc
PYTHONIOENCODING = utf-8
_COVERAGE_SRC = {envsitepackagesdir}/virtualenv
{py34,py27,pypy, upgrade}: PYTHONWARNINGS = ignore:DEPRECATION::pip._internal.cli.base_command
{py34,pypy,py27}: PYTEST_XDIST = 0
COVERAGE_FILE = {toxworkdir}/.coverage.{envname}
COVERAGE_PROCESS_START = {toxinidir}/.coveragerc
PYTHONIOENCODING = utf-8
_COVERAGE_SRC = {envsitepackagesdir}/virtualenv
{py34,py27,pypy2, upgrade}: PYTHONWARNINGS = ignore:DEPRECATION::pip._internal.cli.base_command
{py34,pypy2,py27}: PYTEST_XDIST = 0
extras =
testing
testing
commands =
python -m coverage erase
python -m coverage run -m pytest \
--junitxml {toxworkdir}/junit.{envname}.xml \
{posargs:tests --int --timeout 600 -n {env:PYTEST_XDIST:auto}}
python -m coverage combine
python -m coverage report --skip-covered --show-missing
python -m coverage xml -o {toxworkdir}/coverage.{envname}.xml
python -m coverage html -d {envtmpdir}/htmlcov \
!py34: --show-contexts \
--title virtualenv-{envname}-coverage
python -m coverage erase
python -m coverage run -m pytest \
--junitxml {toxworkdir}/junit.{envname}.xml \
{posargs:tests --int --timeout 600 -n {env:PYTEST_XDIST:auto}}
python -m coverage combine
python -m coverage report --skip-covered --show-missing
python -m coverage xml -o {toxworkdir}/coverage.{envname}.xml
python -m coverage html -d {envtmpdir}/htmlcov \
!py34: --show-contexts \
--title virtualenv-{envname}-coverage
install_command = python -m pip install {opts} {packages} --disable-pip-version-check
[testenv:fix_lint]
description = format the code base to adhere to our styles, and complain about what we cannot do automatically
passenv =
*
basepython = python3.8
*
basepython = python3.9
skip_install = true
deps =
pre-commit>=2
pre-commit>=2
commands =
pre-commit run --all-files --show-diff-on-failure
python -c 'import pathlib; print("hint: run \{\} install to add checks as pre-commit hook".format(pathlib.Path(r"{envdir}") / "bin" / "pre-commit"))'
pre-commit run --all-files --show-diff-on-failure
python -c 'import pathlib; print("hint: run \{\} install to add checks as pre-commit hook".format(pathlib.Path(r"{envdir}") / "bin" / "pre-commit"))'
[testenv:coverage]
description = [run locally after tests]: combine coverage data and create report;
generates a diff coverage against origin/main (can be changed by setting DIFF_AGAINST env var)
generates a diff coverage against origin/main (can be changed by setting DIFF_AGAINST env var)
passenv =
DIFF_AGAINST
DIFF_AGAINST
setenv =
COVERAGE_FILE = {toxworkdir}/.coverage
COVERAGE_FILE = {toxworkdir}/.coverage
skip_install = true
deps =
coverage>=5.0.1
diff_cover>=3
coverage>=5.0.1
diff_cover>=3
extras =
parallel_show_output = true
commands =
python -m coverage combine
python -m coverage report --skip-covered --show-missing
python -m coverage xml -o {toxworkdir}/coverage.xml
python -m coverage html -d {toxworkdir}/htmlcov
python -m diff_cover.diff_cover_tool --compare-branch {env:DIFF_AGAINST:origin/main} {toxworkdir}/coverage.xml
python -m coverage combine
python -m coverage report --skip-covered --show-missing
python -m coverage xml -o {toxworkdir}/coverage.xml
python -m coverage html -d {toxworkdir}/htmlcov
python -m diff_cover.diff_cover_tool --compare-branch {env:DIFF_AGAINST:origin/main} {toxworkdir}/coverage.xml
depends =
py38
py37
py36
py35
py34
py27
pypy
pypy3
py39
py38
py37
py36
py35
py34
py27
pypy
pypy3
[testenv:readme]
description = check that the long description is valid (need for PyPI)
skip_install = true
deps =
pep517>=0.8.2
twine>=1.12.1
build>=0.0.4
twine>=3
extras =
commands =
python -m pep517.build -o {envtmpdir} -b -s .
twine check {envtmpdir}/*
python -m build -o {envtmpdir} --wheel --sdist .
twine check {envtmpdir}/*
[testenv:docs]
description = build documentation
basepython = python3.8
basepython = python3.9
extras =
docs
docs
commands =
python -c 'import glob; import subprocess; subprocess.call(["proselint"] + glob.glob("docs/*.rst") + glob.glob("docs/**/*.rst"))'
sphinx-build -d "{envtmpdir}/doctree" docs "{toxworkdir}/docs_out" --color -b html {posargs}
python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))'
python -c 'import glob; import subprocess; subprocess.call(["proselint"] + glob.glob("docs/*.rst") + glob.glob("docs/**/*.rst"))'
sphinx-build -d "{envtmpdir}/doctree" docs "{toxworkdir}/docs_out" --color -b html {posargs}
python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))'
[testenv:upgrade]
description = upgrade pip/wheels/setuptools to latest
passenv =
UPGRADE_ADVISORY
UPGRADE_ADVISORY
skip_install = true
deps =
black
commands =
python upgrade_wheels.py
black
changedir = {toxinidir}/tasks
commands =
python upgrade_wheels.py
[testenv:release]
description = do a release, required posarg of the version number
passenv =
*
basepython = python3.8
*
basepython = python3.9
deps =
gitpython>=3
packaging>=17.1
towncrier>=19.9.0rc1
commands =
python release.py --version {posargs}
gitpython>=3
packaging>=17.1
towncrier>=19.9.0rc1
changedir = {toxinidir}/tasks
commands =
python release.py --version {posargs}
[testenv:dev]
description = generate a DEV environment
usedevelop = true
deps =
setuptools_scm[toml]>=3.4
{[testenv:release]deps}
{[testenv:release]deps}
setuptools_scm[toml]>=3.4
extras =
docs
testing
docs
testing
commands =
python -m pip list --format=columns
python -c 'import sys; print(sys.executable)'
python -m pip list --format=columns
python -c 'import sys; print(sys.executable)'
[testenv:zipapp]
description = generate a zipapp
skip_install = true
deps =
packaging>=20
packaging>=20
commands =
python tasks/make_zipapp.py
python tasks/make_zipapp.py
[isort]
profile = black