Blame SOURCES/pyproject_buildrequires.py

838e4d
import os
838e4d
import sys
838e4d
import importlib.metadata
838e4d
import argparse
838e4d
import traceback
838e4d
import contextlib
838e4d
from io import StringIO
838e4d
import json
838e4d
import subprocess
838e4d
import re
838e4d
import tempfile
838e4d
import email.parser
838e4d
import pathlib
838e4d
838e4d
from pyproject_requirements_txt import convert_requirements_txt
838e4d
838e4d
838e4d
# Some valid Python version specifiers are not supported.
838e4d
# Allow only the forms we know we can handle.
838e4d
VERSION_RE = re.compile(r'[a-zA-Z0-9.-]+(\.\*)?')
838e4d
838e4d
838e4d
class EndPass(Exception):
838e4d
    """End current pass of generating requirements"""
838e4d
838e4d
838e4d
# nb: we don't use functools.partial to be able to use pytest's capsys
838e4d
# see https://github.com/pytest-dev/pytest/issues/8900
838e4d
def print_err(*args, **kwargs):
838e4d
    kwargs.setdefault('file', sys.stderr)
838e4d
    print(*args, **kwargs)
838e4d
838e4d
838e4d
try:
838e4d
    from packaging.requirements import Requirement, InvalidRequirement
838e4d
    from packaging.utils import canonicalize_name
838e4d
except ImportError as e:
838e4d
    print_err('Import error:', e)
838e4d
    # already echoed by the %pyproject_buildrequires macro
838e4d
    sys.exit(0)
838e4d
838e4d
# uses packaging, needs to be imported after packaging is verified to be present
838e4d
from pyproject_convert import convert
838e4d
838e4d
838e4d
@contextlib.contextmanager
838e4d
def hook_call():
838e4d
    captured_out = StringIO()
838e4d
    with contextlib.redirect_stdout(captured_out):
838e4d
        yield
838e4d
    for line in captured_out.getvalue().splitlines():
838e4d
        print_err('HOOK STDOUT:', line)
838e4d
838e4d
838e4d
def guess_reason_for_invalid_requirement(requirement_str):
838e4d
    if ':' in requirement_str:
838e4d
        message = (
838e4d
            'It might be an URL. '
838e4d
            '%pyproject_buildrequires cannot handle all URL-based requirements. '
838e4d
            'Add PackageName@ (see PEP 508) to the URL to at least require any version of PackageName.'
838e4d
        )
838e4d
        if '@' in requirement_str:
838e4d
            message += ' (but note that URLs might not work well with other features)'
838e4d
        return message
838e4d
    if '/' in requirement_str:
838e4d
        return (
838e4d
            'It might be a local path. '
838e4d
            '%pyproject_buildrequires cannot handle local paths as requirements. '
838e4d
            'Use an URL with PackageName@ (see PEP 508) to at least require any version of PackageName.'
838e4d
        )
838e4d
    # No more ideas
838e4d
    return None
838e4d
838e4d
838e4d
class Requirements:
838e4d
    """Requirement printer"""
838e4d
    def __init__(self, get_installed_version, extras=None,
838e4d
                 generate_extras=False, python3_pkgversion='3'):
838e4d
        self.get_installed_version = get_installed_version
838e4d
        self.extras = set()
838e4d
838e4d
        if extras:
838e4d
            for extra in extras:
838e4d
                self.add_extras(*extra.split(','))
838e4d
838e4d
        self.missing_requirements = False
838e4d
838e4d
        self.generate_extras = generate_extras
838e4d
        self.python3_pkgversion = python3_pkgversion
838e4d
838e4d
    def add_extras(self, *extras):
838e4d
        self.extras |= set(e.strip() for e in extras)
838e4d
838e4d
    @property
838e4d
    def marker_envs(self):
838e4d
        if self.extras:
838e4d
            return [{'extra': e} for e in sorted(self.extras)]
838e4d
        return [{'extra': ''}]
838e4d
838e4d
    def evaluate_all_environamnets(self, requirement):
838e4d
        for marker_env in self.marker_envs:
838e4d
            if requirement.marker.evaluate(environment=marker_env):
838e4d
                return True
838e4d
        return False
838e4d
838e4d
    def add(self, requirement_str, *, source=None):
838e4d
        """Output a Python-style requirement string as RPM dep"""
838e4d
        print_err(f'Handling {requirement_str} from {source}')
838e4d
838e4d
        try:
838e4d
            requirement = Requirement(requirement_str)
838e4d
        except InvalidRequirement:
838e4d
            hint = guess_reason_for_invalid_requirement(requirement_str)
838e4d
            message = f'Requirement {requirement_str!r} from {source} is invalid.'
838e4d
            if hint:
838e4d
                message += f' Hint: {hint}'
838e4d
            raise ValueError(message)
838e4d
838e4d
        if requirement.url:
838e4d
            print_err(
838e4d
                f'WARNING: Simplifying {requirement_str!r} to {requirement.name!r}.'
838e4d
            )
838e4d
838e4d
        name = canonicalize_name(requirement.name)
838e4d
        if (requirement.marker is not None and
838e4d
                not self.evaluate_all_environamnets(requirement)):
838e4d
            print_err(f'Ignoring alien requirement:', requirement_str)
838e4d
            return
838e4d
838e4d
        try:
838e4d
            # TODO: check if requirements with extras are satisfied
838e4d
            installed = self.get_installed_version(requirement.name)
838e4d
        except importlib.metadata.PackageNotFoundError:
838e4d
            print_err(f'Requirement not satisfied: {requirement_str}')
838e4d
            installed = None
838e4d
        if installed and installed in requirement.specifier:
838e4d
            print_err(f'Requirement satisfied: {requirement_str}')
838e4d
            print_err(f'   (installed: {requirement.name} {installed})')
838e4d
            if requirement.extras:
838e4d
                print_err(f'   (extras are currently not checked)')
838e4d
        else:
838e4d
            self.missing_requirements = True
838e4d
838e4d
        if self.generate_extras:
838e4d
            extra_names = [f'{name}[{extra.lower()}]' for extra in sorted(requirement.extras)]
838e4d
        else:
838e4d
            extra_names = []
838e4d
838e4d
        for name in [name] + extra_names:
838e4d
            together = []
838e4d
            for specifier in sorted(
838e4d
                requirement.specifier,
838e4d
                key=lambda s: (s.operator, s.version),
838e4d
            ):
838e4d
                if not VERSION_RE.fullmatch(str(specifier.version)):
838e4d
                    raise ValueError(
838e4d
                        f'Unknown character in version: {specifier.version}. '
838e4d
                        + '(This might be a bug in pyproject-rpm-macros.)',
838e4d
                    )
838e4d
                together.append(convert(python3dist(name, python3_pkgversion=self.python3_pkgversion),
838e4d
                                        specifier.operator, specifier.version))
838e4d
            if len(together) == 0:
838e4d
                print(python3dist(name,
838e4d
                                  python3_pkgversion=self.python3_pkgversion))
838e4d
            elif len(together) == 1:
838e4d
                print(together[0])
838e4d
            else:
838e4d
                print(f"({' with '.join(together)})")
838e4d
838e4d
    def check(self, *, source=None):
838e4d
        """End current pass if any unsatisfied dependencies were output"""
838e4d
        if self.missing_requirements:
838e4d
            print_err(f'Exiting dependency generation pass: {source}')
838e4d
            raise EndPass(source)
838e4d
838e4d
    def extend(self, requirement_strs, **kwargs):
838e4d
        """add() several requirements"""
838e4d
        for req_str in requirement_strs:
838e4d
            self.add(req_str, **kwargs)
838e4d
838e4d
838e4d
def get_backend(requirements):
838e4d
    try:
838e4d
        f = open('pyproject.toml')
838e4d
    except FileNotFoundError:
838e4d
        pyproject_data = {}
838e4d
    else:
838e4d
        try:
838e4d
            # lazy import toml here, not needed without pyproject.toml
838e4d
            import toml
838e4d
        except ImportError as e:
838e4d
            print_err('Import error:', e)
838e4d
            # already echoed by the %pyproject_buildrequires macro
838e4d
            sys.exit(0)
838e4d
        with f:
838e4d
            pyproject_data = toml.load(f)
838e4d
838e4d
    buildsystem_data = pyproject_data.get('build-system', {})
838e4d
    requirements.extend(
838e4d
        buildsystem_data.get('requires', ()),
838e4d
        source='build-system.requires',
838e4d
    )
838e4d
838e4d
    backend_name = buildsystem_data.get('build-backend')
838e4d
    if not backend_name:
838e4d
        # https://www.python.org/dev/peps/pep-0517/:
838e4d
        # If the pyproject.toml file is absent, or the build-backend key is
838e4d
        # missing, the source tree is not using this specification, and tools
838e4d
        # should revert to the legacy behaviour of running setup.py
838e4d
        # (either directly, or by implicitly invoking the [following] backend).
838e4d
        # If setup.py is also not present program will mimick pip's behavior
838e4d
        # and end with an error.
838e4d
        if not os.path.exists('setup.py'):
838e4d
            raise FileNotFoundError('File "setup.py" not found for legacy project.')
838e4d
        backend_name = 'setuptools.build_meta:__legacy__'
838e4d
838e4d
        # Note: For projects without pyproject.toml, this was already echoed
838e4d
        # by the %pyproject_buildrequires macro, but this also handles cases
838e4d
        # with pyproject.toml without a specified build backend.
838e4d
        # If the default requirements change, also change them in the macro!
838e4d
        requirements.add('setuptools >= 40.8', source='default build backend')
838e4d
        requirements.add('wheel', source='default build backend')
838e4d
838e4d
    requirements.check(source='build backend')
838e4d
838e4d
    backend_path = buildsystem_data.get('backend-path')
838e4d
    if backend_path:
838e4d
        # PEP 517 example shows the path as a list, but some projects don't follow that
838e4d
        if isinstance(backend_path, str):
838e4d
            backend_path = [backend_path]
838e4d
        sys.path = backend_path + sys.path
838e4d
838e4d
    module_name, _, object_name = backend_name.partition(":")
838e4d
    backend_module = importlib.import_module(module_name)
838e4d
838e4d
    if object_name:
838e4d
        return getattr(backend_module, object_name)
838e4d
838e4d
    return backend_module
838e4d
838e4d
838e4d
def generate_build_requirements(backend, requirements):
838e4d
    get_requires = getattr(backend, 'get_requires_for_build_wheel', None)
838e4d
    if get_requires:
838e4d
        with hook_call():
838e4d
            new_reqs = get_requires()
838e4d
        requirements.extend(new_reqs, source='get_requires_for_build_wheel')
838e4d
        requirements.check(source='get_requires_for_build_wheel')
838e4d
838e4d
838e4d
def generate_run_requirements(backend, requirements):
838e4d
    hook_name = 'prepare_metadata_for_build_wheel'
838e4d
    prepare_metadata = getattr(backend, hook_name, None)
838e4d
    if not prepare_metadata:
838e4d
        raise ValueError(
838e4d
            'build backend cannot provide build metadata '
838e4d
            + '(incl. runtime requirements) before build'
838e4d
        )
838e4d
    with hook_call():
838e4d
        dir_basename = prepare_metadata('.')
838e4d
    with open(dir_basename + '/METADATA') as f:
838e4d
        message = email.parser.Parser().parse(f, headersonly=True)
838e4d
    for key in 'Requires', 'Requires-Dist':
838e4d
        requires = message.get_all(key, ())
838e4d
        requirements.extend(requires, source=f'wheel metadata: {key}')
838e4d
838e4d
838e4d
def generate_tox_requirements(toxenv, requirements):
838e4d
    toxenv = ','.join(toxenv)
838e4d
    requirements.add('tox-current-env >= 0.0.6', source='tox itself')
838e4d
    requirements.check(source='tox itself')
838e4d
    with tempfile.NamedTemporaryFile('r') as deps, \
838e4d
        tempfile.NamedTemporaryFile('r') as extras, \
838e4d
            tempfile.NamedTemporaryFile('r') as provision:
838e4d
        r = subprocess.run(
838e4d
            [sys.executable, '-m', 'tox',
838e4d
             '--print-deps-to', deps.name,
838e4d
             '--print-extras-to', extras.name,
838e4d
             '--no-provision', provision.name,
838e4d
             '-qre', toxenv],
838e4d
            check=False,
838e4d
            encoding='utf-8',
838e4d
            stdout=subprocess.PIPE,
838e4d
            stderr=subprocess.STDOUT,
838e4d
        )
838e4d
        if r.stdout:
838e4d
            print_err(r.stdout, end='')
838e4d
838e4d
        provision_content = provision.read()
838e4d
        if provision_content and r.returncode != 0:
838e4d
            provision_requires = json.loads(provision_content)
838e4d
            if 'minversion' in provision_requires:
838e4d
                requirements.add(f'tox >= {provision_requires["minversion"]}',
838e4d
                                 source='tox provision (minversion)')
838e4d
            if 'requires' in provision_requires:
838e4d
                requirements.extend(provision_requires["requires"],
838e4d
                                    source='tox provision (requires)')
838e4d
            requirements.check(source='tox provision')  # this terminates the script
838e4d
            raise RuntimeError(
838e4d
                'Dependencies requested by tox provisioning appear installed, '
838e4d
                'but tox disagreed.')
838e4d
        else:
838e4d
            r.check_returncode()
838e4d
838e4d
        deplines = deps.read().splitlines()
838e4d
        packages = convert_requirements_txt(deplines)
838e4d
        requirements.add_extras(*extras.read().splitlines())
838e4d
        requirements.extend(packages,
838e4d
                            source=f'tox --print-deps-only: {toxenv}')
838e4d
838e4d
838e4d
def python3dist(name, op=None, version=None, python3_pkgversion="3"):
838e4d
    prefix = f"python{python3_pkgversion}dist"
838e4d
838e4d
    if op is None:
838e4d
        if version is not None:
838e4d
            raise AssertionError('op and version go together')
838e4d
        return f'{prefix}({name})'
838e4d
    else:
838e4d
        return f'{prefix}({name}) {op} {version}'
838e4d
838e4d
838e4d
def generate_requires(
838e4d
    *, include_runtime=False, toxenv=None, extras=None,
838e4d
    get_installed_version=importlib.metadata.version,  # for dep injection
838e4d
    generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True
838e4d
):
838e4d
    """Generate the BuildRequires for the project in the current directory
838e4d
838e4d
    This is the main Python entry point.
838e4d
    """
838e4d
    requirements = Requirements(
838e4d
        get_installed_version, extras=extras or [],
838e4d
        generate_extras=generate_extras,
838e4d
        python3_pkgversion=python3_pkgversion
838e4d
    )
838e4d
838e4d
    try:
838e4d
        if (include_runtime or toxenv) and not use_build_system:
838e4d
            raise ValueError('-N option cannot be used in combination with -r, -e, -t, -x options')
838e4d
        if requirement_files:
838e4d
            for req_file in requirement_files:
838e4d
                requirements.extend(
838e4d
                    convert_requirements_txt(req_file, pathlib.Path(req_file.name)),
838e4d
                    source=f'requirements file {req_file.name}'
838e4d
                )
838e4d
            requirements.check(source='all requirements files')
838e4d
        if use_build_system:
838e4d
            backend = get_backend(requirements)
838e4d
            generate_build_requirements(backend, requirements)
838e4d
        if toxenv:
838e4d
            include_runtime = True
838e4d
            generate_tox_requirements(toxenv, requirements)
838e4d
        if include_runtime:
838e4d
            generate_run_requirements(backend, requirements)
838e4d
    except EndPass:
838e4d
        return
838e4d
838e4d
838e4d
def main(argv):
838e4d
    parser = argparse.ArgumentParser(
838e4d
        description='Generate BuildRequires for a Python project.'
838e4d
    )
838e4d
    parser.add_argument(
838e4d
        '-r', '--runtime', action='store_true',
838e4d
        help='Generate run-time requirements',
838e4d
    )
838e4d
    parser.add_argument(
838e4d
        '-e', '--toxenv', metavar='TOXENVS', action='append',
838e4d
        help=('specify tox environments (comma separated and/or repeated)'
838e4d
              '(implies --tox)'),
838e4d
    )
838e4d
    parser.add_argument(
838e4d
        '-t', '--tox', action='store_true',
838e4d
        help=('generate test tequirements from tox environment '
838e4d
              '(implies --runtime)'),
838e4d
    )
838e4d
    parser.add_argument(
838e4d
        '-x', '--extras', metavar='EXTRAS', action='append',
838e4d
        help='comma separated list of "extras" for runtime requirements '
838e4d
             '(e.g. -x testing,feature-x) (implies --runtime, can be repeated)',
838e4d
    )
838e4d
    parser.add_argument(
838e4d
        '--generate-extras', action='store_true',
838e4d
        help='Generate build requirements on Python Extras',
838e4d
    )
838e4d
    parser.add_argument(
838e4d
        '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION',
838e4d
        default="3", help=('Python version for pythonXdist()'
838e4d
                           'or pythonX.Ydist() requirements'),
838e4d
    )
838e4d
    parser.add_argument(
838e4d
        '-N', '--no-use-build-system', dest='use_build_system',
838e4d
        action='store_false', help='Use -N to indicate that project does not use any build system',
838e4d
    )
838e4d
    parser.add_argument(
838e4d
       'requirement_files', nargs='*', type=argparse.FileType('r'),
838e4d
        help=('Add buildrequires from file'),
838e4d
    )
838e4d
838e4d
    args = parser.parse_args(argv)
838e4d
838e4d
    if args.toxenv:
838e4d
        args.tox = True
838e4d
838e4d
    if args.tox:
838e4d
        args.runtime = True
838e4d
        if not args.toxenv:
838e4d
            _default = f'py{sys.version_info.major}{sys.version_info.minor}'
838e4d
            args.toxenv = [os.getenv('RPM_TOXENV', _default)]
838e4d
838e4d
    if args.extras:
838e4d
        args.runtime = True
838e4d
838e4d
    try:
838e4d
        generate_requires(
838e4d
            include_runtime=args.runtime,
838e4d
            toxenv=args.toxenv,
838e4d
            extras=args.extras,
838e4d
            generate_extras=args.generate_extras,
838e4d
            python3_pkgversion=args.python3_pkgversion,
838e4d
            requirement_files=args.requirement_files,
838e4d
            use_build_system=args.use_build_system,
838e4d
        )
838e4d
    except Exception:
838e4d
        # Log the traceback explicitly (it's useful debug info)
838e4d
        traceback.print_exc()
838e4d
        exit(1)
838e4d
838e4d
838e4d
if __name__ == '__main__':
838e4d
    main(sys.argv[1:])