Blame SOURCES/pyproject_buildrequires.py

557ab5
import glob
557ab5
import io
838e4d
import os
838e4d
import sys
838e4d
import importlib.metadata
838e4d
import argparse
b6f61c
import tempfile
838e4d
import traceback
838e4d
import contextlib
838e4d
import json
838e4d
import subprocess
838e4d
import re
838e4d
import tempfile
838e4d
import email.parser
838e4d
import pathlib
557ab5
import zipfile
838e4d
838e4d
from pyproject_requirements_txt import convert_requirements_txt
838e4d
838e4d
838e4d
# Some valid Python version specifiers are not supported.
838e4d
# Allow only the forms we know we can handle.
838e4d
VERSION_RE = re.compile(r'[a-zA-Z0-9.-]+(\.\*)?')
838e4d
838e4d
838e4d
class EndPass(Exception):
838e4d
    """End current pass of generating requirements"""
838e4d
838e4d
838e4d
# nb: we don't use functools.partial to be able to use pytest's capsys
838e4d
# see https://github.com/pytest-dev/pytest/issues/8900
838e4d
def print_err(*args, **kwargs):
838e4d
    kwargs.setdefault('file', sys.stderr)
838e4d
    print(*args, **kwargs)
838e4d
838e4d
838e4d
try:
838e4d
    from packaging.requirements import Requirement, InvalidRequirement
838e4d
    from packaging.utils import canonicalize_name
838e4d
except ImportError as e:
838e4d
    print_err('Import error:', e)
838e4d
    # already echoed by the %pyproject_buildrequires macro
838e4d
    sys.exit(0)
838e4d
838e4d
# uses packaging, needs to be imported after packaging is verified to be present
838e4d
from pyproject_convert import convert
838e4d
838e4d
838e4d
@contextlib.contextmanager
838e4d
def hook_call():
b6f61c
    """Context manager that records all stdout content (on FD level)
b6f61c
    and prints it to stderr at the end, with a 'HOOK STDOUT: ' prefix."""
b6f61c
    tmpfile = io.TextIOWrapper(
b6f61c
        tempfile.TemporaryFile(buffering=0),
b6f61c
        encoding='utf-8',
b6f61c
        errors='replace',
b6f61c
        write_through=True,
b6f61c
    )
b6f61c
b6f61c
    stdout_fd = 1
b6f61c
    stdout_fd_dup = os.dup(stdout_fd)
b6f61c
    stdout_orig = sys.stdout
b6f61c
b6f61c
    # begin capture
b6f61c
    sys.stdout = tmpfile
b6f61c
    os.dup2(tmpfile.fileno(), stdout_fd)
b6f61c
b6f61c
    try:
838e4d
        yield
b6f61c
    finally:
b6f61c
        # end capture
b6f61c
        sys.stdout = stdout_orig
b6f61c
        os.dup2(stdout_fd_dup, stdout_fd)
b6f61c
b6f61c
        tmpfile.seek(0)  # rewind
b6f61c
        for line in tmpfile:
b6f61c
            print_err('HOOK STDOUT:', line, end='')
b6f61c
b6f61c
        tmpfile.close()
838e4d
838e4d
838e4d
def guess_reason_for_invalid_requirement(requirement_str):
838e4d
    if ':' in requirement_str:
838e4d
        message = (
838e4d
            'It might be an URL. '
838e4d
            '%pyproject_buildrequires cannot handle all URL-based requirements. '
838e4d
            'Add PackageName@ (see PEP 508) to the URL to at least require any version of PackageName.'
838e4d
        )
838e4d
        if '@' in requirement_str:
838e4d
            message += ' (but note that URLs might not work well with other features)'
838e4d
        return message
838e4d
    if '/' in requirement_str:
838e4d
        return (
838e4d
            'It might be a local path. '
838e4d
            '%pyproject_buildrequires cannot handle local paths as requirements. '
838e4d
            'Use an URL with PackageName@ (see PEP 508) to at least require any version of PackageName.'
838e4d
        )
838e4d
    # No more ideas
838e4d
    return None
838e4d
838e4d
838e4d
class Requirements:
838e4d
    """Requirement printer"""
838e4d
    def __init__(self, get_installed_version, extras=None,
838e4d
                 generate_extras=False, python3_pkgversion='3'):
838e4d
        self.get_installed_version = get_installed_version
838e4d
        self.extras = set()
838e4d
838e4d
        if extras:
838e4d
            for extra in extras:
838e4d
                self.add_extras(*extra.split(','))
838e4d
838e4d
        self.missing_requirements = False
838e4d
838e4d
        self.generate_extras = generate_extras
838e4d
        self.python3_pkgversion = python3_pkgversion
838e4d
838e4d
    def add_extras(self, *extras):
838e4d
        self.extras |= set(e.strip() for e in extras)
838e4d
838e4d
    @property
838e4d
    def marker_envs(self):
838e4d
        if self.extras:
838e4d
            return [{'extra': e} for e in sorted(self.extras)]
838e4d
        return [{'extra': ''}]
838e4d
b6f61c
    def evaluate_all_environments(self, requirement):
838e4d
        for marker_env in self.marker_envs:
838e4d
            if requirement.marker.evaluate(environment=marker_env):
838e4d
                return True
838e4d
        return False
838e4d
838e4d
    def add(self, requirement_str, *, source=None):
838e4d
        """Output a Python-style requirement string as RPM dep"""
838e4d
        print_err(f'Handling {requirement_str} from {source}')
838e4d
838e4d
        try:
838e4d
            requirement = Requirement(requirement_str)
838e4d
        except InvalidRequirement:
838e4d
            hint = guess_reason_for_invalid_requirement(requirement_str)
838e4d
            message = f'Requirement {requirement_str!r} from {source} is invalid.'
838e4d
            if hint:
838e4d
                message += f' Hint: {hint}'
838e4d
            raise ValueError(message)
838e4d
838e4d
        if requirement.url:
838e4d
            print_err(
838e4d
                f'WARNING: Simplifying {requirement_str!r} to {requirement.name!r}.'
838e4d
            )
838e4d
838e4d
        name = canonicalize_name(requirement.name)
838e4d
        if (requirement.marker is not None and
b6f61c
                not self.evaluate_all_environments(requirement)):
838e4d
            print_err(f'Ignoring alien requirement:', requirement_str)
838e4d
            return
838e4d
98862f
        # We need to always accept pre-releases as satisfying the requirement
98862f
        # Otherwise e.g. installed cffi version 1.15.0rc2 won't even satisfy the requirement for "cffi"
98862f
        # https://bugzilla.redhat.com/show_bug.cgi?id=2014639#c3
98862f
        requirement.specifier.prereleases = True
98862f
838e4d
        try:
838e4d
            # TODO: check if requirements with extras are satisfied
838e4d
            installed = self.get_installed_version(requirement.name)
838e4d
        except importlib.metadata.PackageNotFoundError:
838e4d
            print_err(f'Requirement not satisfied: {requirement_str}')
838e4d
            installed = None
838e4d
        if installed and installed in requirement.specifier:
838e4d
            print_err(f'Requirement satisfied: {requirement_str}')
838e4d
            print_err(f'   (installed: {requirement.name} {installed})')
838e4d
            if requirement.extras:
838e4d
                print_err(f'   (extras are currently not checked)')
838e4d
        else:
838e4d
            self.missing_requirements = True
838e4d
838e4d
        if self.generate_extras:
838e4d
            extra_names = [f'{name}[{extra.lower()}]' for extra in sorted(requirement.extras)]
838e4d
        else:
838e4d
            extra_names = []
838e4d
838e4d
        for name in [name] + extra_names:
838e4d
            together = []
838e4d
            for specifier in sorted(
838e4d
                requirement.specifier,
838e4d
                key=lambda s: (s.operator, s.version),
838e4d
            ):
838e4d
                if not VERSION_RE.fullmatch(str(specifier.version)):
838e4d
                    raise ValueError(
838e4d
                        f'Unknown character in version: {specifier.version}. '
838e4d
                        + '(This might be a bug in pyproject-rpm-macros.)',
838e4d
                    )
838e4d
                together.append(convert(python3dist(name, python3_pkgversion=self.python3_pkgversion),
838e4d
                                        specifier.operator, specifier.version))
838e4d
            if len(together) == 0:
838e4d
                print(python3dist(name,
838e4d
                                  python3_pkgversion=self.python3_pkgversion))
838e4d
            elif len(together) == 1:
838e4d
                print(together[0])
838e4d
            else:
838e4d
                print(f"({' with '.join(together)})")
838e4d
838e4d
    def check(self, *, source=None):
838e4d
        """End current pass if any unsatisfied dependencies were output"""
838e4d
        if self.missing_requirements:
838e4d
            print_err(f'Exiting dependency generation pass: {source}')
838e4d
            raise EndPass(source)
838e4d
838e4d
    def extend(self, requirement_strs, **kwargs):
838e4d
        """add() several requirements"""
838e4d
        for req_str in requirement_strs:
838e4d
            self.add(req_str, **kwargs)
838e4d
838e4d
557ab5
def toml_load(opened_binary_file):
838e4d
    try:
557ab5
        # tomllib is in the standard library since 3.11.0b1
557ab5
        import tomllib as toml_module
557ab5
        load_from = opened_binary_file
557ab5
    except ImportError:
838e4d
        try:
557ab5
            # note: we could use tomli here,
557ab5
            # but for backwards compatibility with RHEL 9, we use toml instead
557ab5
            import toml as toml_module
557ab5
            load_from = io.TextIOWrapper(opened_binary_file, encoding='utf-8')
838e4d
        except ImportError as e:
838e4d
            print_err('Import error:', e)
838e4d
            # already echoed by the %pyproject_buildrequires macro
838e4d
            sys.exit(0)
557ab5
    return toml_module.load(load_from)
557ab5
557ab5
557ab5
def get_backend(requirements):
557ab5
    try:
557ab5
        f = open('pyproject.toml', 'rb')
557ab5
    except FileNotFoundError:
557ab5
        pyproject_data = {}
557ab5
    else:
838e4d
        with f:
557ab5
            pyproject_data = toml_load(f)
838e4d
838e4d
    buildsystem_data = pyproject_data.get('build-system', {})
838e4d
    requirements.extend(
838e4d
        buildsystem_data.get('requires', ()),
838e4d
        source='build-system.requires',
838e4d
    )
838e4d
838e4d
    backend_name = buildsystem_data.get('build-backend')
838e4d
    if not backend_name:
838e4d
        # https://www.python.org/dev/peps/pep-0517/:
838e4d
        # If the pyproject.toml file is absent, or the build-backend key is
838e4d
        # missing, the source tree is not using this specification, and tools
838e4d
        # should revert to the legacy behaviour of running setup.py
838e4d
        # (either directly, or by implicitly invoking the [following] backend).
838e4d
        # If setup.py is also not present program will mimick pip's behavior
838e4d
        # and end with an error.
838e4d
        if not os.path.exists('setup.py'):
838e4d
            raise FileNotFoundError('File "setup.py" not found for legacy project.')
838e4d
        backend_name = 'setuptools.build_meta:__legacy__'
838e4d
838e4d
        # Note: For projects without pyproject.toml, this was already echoed
838e4d
        # by the %pyproject_buildrequires macro, but this also handles cases
838e4d
        # with pyproject.toml without a specified build backend.
838e4d
        # If the default requirements change, also change them in the macro!
838e4d
        requirements.add('setuptools >= 40.8', source='default build backend')
838e4d
        requirements.add('wheel', source='default build backend')
838e4d
838e4d
    requirements.check(source='build backend')
838e4d
838e4d
    backend_path = buildsystem_data.get('backend-path')
838e4d
    if backend_path:
838e4d
        # PEP 517 example shows the path as a list, but some projects don't follow that
838e4d
        if isinstance(backend_path, str):
838e4d
            backend_path = [backend_path]
838e4d
        sys.path = backend_path + sys.path
838e4d
838e4d
    module_name, _, object_name = backend_name.partition(":")
838e4d
    backend_module = importlib.import_module(module_name)
838e4d
838e4d
    if object_name:
838e4d
        return getattr(backend_module, object_name)
838e4d
838e4d
    return backend_module
838e4d
838e4d
838e4d
def generate_build_requirements(backend, requirements):
838e4d
    get_requires = getattr(backend, 'get_requires_for_build_wheel', None)
838e4d
    if get_requires:
838e4d
        with hook_call():
838e4d
            new_reqs = get_requires()
838e4d
        requirements.extend(new_reqs, source='get_requires_for_build_wheel')
838e4d
        requirements.check(source='get_requires_for_build_wheel')
838e4d
838e4d
557ab5
def requires_from_metadata_file(metadata_file):
557ab5
    message = email.parser.Parser().parse(metadata_file, headersonly=True)
557ab5
    return {k: message.get_all(k, ()) for k in ('Requires', 'Requires-Dist')}
557ab5
557ab5
557ab5
def generate_run_requirements_hook(backend, requirements):
838e4d
    hook_name = 'prepare_metadata_for_build_wheel'
838e4d
    prepare_metadata = getattr(backend, hook_name, None)
838e4d
    if not prepare_metadata:
838e4d
        raise ValueError(
557ab5
            'The build backend cannot provide build metadata '
557ab5
            '(incl. runtime requirements) before build. '
557ab5
            'Use the provisional -w flag to build the wheel and parse the metadata from it, '
557ab5
            'or use the -R flag not to generate runtime dependencies.'
838e4d
        )
838e4d
    with hook_call():
838e4d
        dir_basename = prepare_metadata('.')
557ab5
    with open(dir_basename + '/METADATA') as metadata_file:
557ab5
        for key, requires in requires_from_metadata_file(metadata_file).items():
557ab5
            requirements.extend(requires, source=f'hook generated metadata: {key}')
557ab5
557ab5
557ab5
def find_built_wheel(wheeldir):
557ab5
    wheels = glob.glob(os.path.join(wheeldir, '*.whl'))
557ab5
    if not wheels:
557ab5
        return None
557ab5
    if len(wheels) > 1:
557ab5
        raise RuntimeError('Found multiple wheels in %{_pyproject_wheeldir}, '
557ab5
                           'this is not supported with %pyproject_buildrequires -w.')
557ab5
    return wheels[0]
557ab5
557ab5
557ab5
def generate_run_requirements_wheel(backend, requirements, wheeldir):
557ab5
    # Reuse the wheel from the previous round of %pyproject_buildrequires (if it exists)
557ab5
    wheel = find_built_wheel(wheeldir)
557ab5
    if not wheel:
557ab5
        import pyproject_wheel
557ab5
        returncode = pyproject_wheel.build_wheel(wheeldir=wheeldir, stdout=sys.stderr)
557ab5
        if returncode != 0:
557ab5
            raise RuntimeError('Failed to build the wheel for %pyproject_buildrequires -w.')
557ab5
        wheel = find_built_wheel(wheeldir)
557ab5
    if not wheel:
557ab5
        raise RuntimeError('Cannot locate the built wheel for %pyproject_buildrequires -w.')
557ab5
557ab5
    print_err(f'Reading metadata from {wheel}')
557ab5
    with zipfile.ZipFile(wheel) as wheelfile:
557ab5
        for name in wheelfile.namelist():
557ab5
            if name.count('/') == 1 and name.endswith('.dist-info/METADATA'):
557ab5
                with io.TextIOWrapper(wheelfile.open(name), encoding='utf-8') as metadata_file:
557ab5
                    for key, requires in requires_from_metadata_file(metadata_file).items():
557ab5
                        requirements.extend(requires, source=f'built wheel metadata: {key}')
557ab5
                break
557ab5
        else:
557ab5
            raise RuntimeError('Could not find *.dist-info/METADATA in built wheel.')
557ab5
557ab5
557ab5
def generate_run_requirements(backend, requirements, *, build_wheel, wheeldir):
557ab5
    if build_wheel:
557ab5
        generate_run_requirements_wheel(backend, requirements, wheeldir)
557ab5
    else:
557ab5
        generate_run_requirements_hook(backend, requirements)
838e4d
838e4d
838e4d
def generate_tox_requirements(toxenv, requirements):
838e4d
    toxenv = ','.join(toxenv)
838e4d
    requirements.add('tox-current-env >= 0.0.6', source='tox itself')
838e4d
    requirements.check(source='tox itself')
838e4d
    with tempfile.NamedTemporaryFile('r') as deps, \
838e4d
        tempfile.NamedTemporaryFile('r') as extras, \
838e4d
            tempfile.NamedTemporaryFile('r') as provision:
838e4d
        r = subprocess.run(
838e4d
            [sys.executable, '-m', 'tox',
838e4d
             '--print-deps-to', deps.name,
838e4d
             '--print-extras-to', extras.name,
838e4d
             '--no-provision', provision.name,
557ab5
             '-q', '-r', '-e', toxenv],
838e4d
            check=False,
838e4d
            encoding='utf-8',
838e4d
            stdout=subprocess.PIPE,
838e4d
            stderr=subprocess.STDOUT,
838e4d
        )
838e4d
        if r.stdout:
838e4d
            print_err(r.stdout, end='')
838e4d
838e4d
        provision_content = provision.read()
838e4d
        if provision_content and r.returncode != 0:
838e4d
            provision_requires = json.loads(provision_content)
838e4d
            if 'minversion' in provision_requires:
838e4d
                requirements.add(f'tox >= {provision_requires["minversion"]}',
838e4d
                                 source='tox provision (minversion)')
838e4d
            if 'requires' in provision_requires:
838e4d
                requirements.extend(provision_requires["requires"],
838e4d
                                    source='tox provision (requires)')
838e4d
            requirements.check(source='tox provision')  # this terminates the script
838e4d
            raise RuntimeError(
838e4d
                'Dependencies requested by tox provisioning appear installed, '
838e4d
                'but tox disagreed.')
838e4d
        else:
838e4d
            r.check_returncode()
838e4d
838e4d
        deplines = deps.read().splitlines()
838e4d
        packages = convert_requirements_txt(deplines)
838e4d
        requirements.add_extras(*extras.read().splitlines())
838e4d
        requirements.extend(packages,
838e4d
                            source=f'tox --print-deps-only: {toxenv}')
838e4d
838e4d
838e4d
def python3dist(name, op=None, version=None, python3_pkgversion="3"):
838e4d
    prefix = f"python{python3_pkgversion}dist"
838e4d
838e4d
    if op is None:
838e4d
        if version is not None:
838e4d
            raise AssertionError('op and version go together')
838e4d
        return f'{prefix}({name})'
838e4d
    else:
838e4d
        return f'{prefix}({name}) {op} {version}'
838e4d
838e4d
838e4d
def generate_requires(
557ab5
    *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None,
838e4d
    get_installed_version=importlib.metadata.version,  # for dep injection
838e4d
    generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True
838e4d
):
838e4d
    """Generate the BuildRequires for the project in the current directory
838e4d
838e4d
    This is the main Python entry point.
838e4d
    """
838e4d
    requirements = Requirements(
838e4d
        get_installed_version, extras=extras or [],
838e4d
        generate_extras=generate_extras,
838e4d
        python3_pkgversion=python3_pkgversion
838e4d
    )
838e4d
838e4d
    try:
838e4d
        if (include_runtime or toxenv) and not use_build_system:
838e4d
            raise ValueError('-N option cannot be used in combination with -r, -e, -t, -x options')
838e4d
        if requirement_files:
838e4d
            for req_file in requirement_files:
838e4d
                requirements.extend(
838e4d
                    convert_requirements_txt(req_file, pathlib.Path(req_file.name)),
838e4d
                    source=f'requirements file {req_file.name}'
838e4d
                )
838e4d
            requirements.check(source='all requirements files')
838e4d
        if use_build_system:
838e4d
            backend = get_backend(requirements)
838e4d
            generate_build_requirements(backend, requirements)
838e4d
        if toxenv:
838e4d
            include_runtime = True
838e4d
            generate_tox_requirements(toxenv, requirements)
838e4d
        if include_runtime:
557ab5
            generate_run_requirements(backend, requirements, build_wheel=build_wheel, wheeldir=wheeldir)
838e4d
    except EndPass:
838e4d
        return
838e4d
838e4d
838e4d
def main(argv):
838e4d
    parser = argparse.ArgumentParser(
b6f61c
        description='Generate BuildRequires for a Python project.',
b6f61c
        prog='%pyproject_buildrequires',
b6f61c
        add_help=False,
b6f61c
    )
b6f61c
    parser.add_argument(
b6f61c
        '--help', action='help',
b6f61c
        default=argparse.SUPPRESS,
b6f61c
        help=argparse.SUPPRESS,
838e4d
    )
838e4d
    parser.add_argument(
98862f
        '-r', '--runtime', action='store_true', default=True,
b6f61c
        help=argparse.SUPPRESS,  # Generate run-time requirements (backwards-compatibility only)
98862f
    )
98862f
    parser.add_argument(
b6f61c
        '--generate-extras', action='store_true',
b6f61c
        help=argparse.SUPPRESS,
557ab5
    )
557ab5
    parser.add_argument(
b6f61c
        '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION',
b6f61c
        default="3", help=argparse.SUPPRESS,
557ab5
    )
557ab5
    parser.add_argument(
b6f61c
        '--wheeldir', metavar='PATH', default=None,
b6f61c
        help=argparse.SUPPRESS,
838e4d
    )
838e4d
    parser.add_argument(
b6f61c
        '-x', '--extras', metavar='EXTRAS', action='append',
b6f61c
        help='comma separated list of "extras" for runtime requirements '
b6f61c
             '(e.g. -x testing,feature-x) (implies --runtime, can be repeated)',
838e4d
    )
838e4d
    parser.add_argument(
838e4d
        '-t', '--tox', action='store_true',
838e4d
        help=('generate test tequirements from tox environment '
838e4d
              '(implies --runtime)'),
838e4d
    )
838e4d
    parser.add_argument(
b6f61c
        '-e', '--toxenv', metavar='TOXENVS', action='append',
b6f61c
        help=('specify tox environments (comma separated and/or repeated)'
b6f61c
              '(implies --tox)'),
838e4d
    )
838e4d
    parser.add_argument(
b6f61c
        '-w', '--wheel', action='store_true', default=False,
b6f61c
        help=('Generate run-time requirements by building the wheel '
b6f61c
              '(useful for build backends without the prepare_metadata_for_build_wheel hook)'),
838e4d
    )
838e4d
    parser.add_argument(
b6f61c
        '-R', '--no-runtime', action='store_false', dest='runtime',
b6f61c
        help="Don't generate run-time requirements (implied by -N)",
838e4d
    )
838e4d
    parser.add_argument(
838e4d
        '-N', '--no-use-build-system', dest='use_build_system',
838e4d
        action='store_false', help='Use -N to indicate that project does not use any build system',
838e4d
    )
838e4d
    parser.add_argument(
b6f61c
        'requirement_files', nargs='*', type=argparse.FileType('r'),
b6f61c
        metavar='REQUIREMENTS.TXT',
838e4d
        help=('Add buildrequires from file'),
838e4d
    )
838e4d
838e4d
    args = parser.parse_args(argv)
838e4d
98862f
    if not args.use_build_system:
98862f
        args.runtime = False
98862f
557ab5
    if args.wheel:
557ab5
        if not args.wheeldir:
557ab5
            raise ValueError('--wheeldir must be set when -w.')
557ab5
838e4d
    if args.toxenv:
838e4d
        args.tox = True
838e4d
838e4d
    if args.tox:
838e4d
        args.runtime = True
838e4d
        if not args.toxenv:
838e4d
            _default = f'py{sys.version_info.major}{sys.version_info.minor}'
838e4d
            args.toxenv = [os.getenv('RPM_TOXENV', _default)]
838e4d
838e4d
    if args.extras:
838e4d
        args.runtime = True
838e4d
838e4d
    try:
838e4d
        generate_requires(
838e4d
            include_runtime=args.runtime,
557ab5
            build_wheel=args.wheel,
557ab5
            wheeldir=args.wheeldir,
838e4d
            toxenv=args.toxenv,
838e4d
            extras=args.extras,
838e4d
            generate_extras=args.generate_extras,
838e4d
            python3_pkgversion=args.python3_pkgversion,
838e4d
            requirement_files=args.requirement_files,
838e4d
            use_build_system=args.use_build_system,
838e4d
        )
838e4d
    except Exception:
838e4d
        # Log the traceback explicitly (it's useful debug info)
838e4d
        traceback.print_exc()
838e4d
        exit(1)
838e4d
838e4d
838e4d
if __name__ == '__main__':
838e4d
    main(sys.argv[1:])