diff --git a/README.md b/README.md index 7342728d557c602f51c6d278bba9f3dd9faaf356..970580b12c6b7e9fee2afa8660f37e707abdc9bc 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,508 @@ -Anolis OS -======================================= -# 代码仓库说明 -## 分支说明 ->进行代码开发工作时,请注意选择当前版本对应的分支 -* aX分支为对应大版本的主分支,如a8分支对应当前最新版本 -* aX.Y分支为对应小版本的维护分支,如a8.2分支对应8.2版本 -## 开发流程 -1. 首先fork目标分支到自己的namespace -2. 在自己的fork分支上做出修改 -3. 向对应的仓库中提交merge request,源分支为fork分支 +pyproject RPM macros +==================== + +These macros allow projects that follow the Python [packaging specifications] +to be packaged as RPMs. + +They work for: + +* traditional Setuptools-based projects that use the `setup.py` file, +* newer Setuptools-based projects that have a `setup.cfg` file, +* general Python projects that use the [PEP 517] `pyproject.toml` file (which allows using any build system, such as setuptools, flit or poetry). + +These macros replace `%py3_build` and `%py3_install`, which only work with `setup.py`. + +[packaging specifications]: https://packaging.python.org/specifications/ + + +Usage +----- + +To use these macros, first BuildRequire the devel package for the Python you +are building against. In Fedora, that's `python3-devel`. + + BuildRequires: python3-devel + +The macros will be pulled in as a dependency on Fedora and EPEL 9+. +In other distributions you need to BuildRequire the macros as well: + + BuildRequires: python3-devel + BuildRequires: pyproject-rpm-macros + +Next, you need to generate more build dependencies (of your projects and +the macros themselves) by running `%pyproject_buildrequires` in the +`%generate_buildrequires` section: + + %generate_buildrequires + %pyproject_buildrequires + +This will add build dependencies according to [PEP 517] and [PEP 518]. +This also adds run-time dependencies by default and +can add test-time dependencies, see the section below. +If you need more dependencies, such as non-Python libraries, BuildRequire +them manually. + +Note that `%generate_buildrequires` may produce error messages `(exit 11)` in +the build log. This is expected behavior of BuildRequires generators; see +[the Fedora change] for details. + +[the Fedora change]: https://fedoraproject.org/wiki/Changes/DynamicBuildRequires + +Then, build a wheel in `%build` with `%pyproject_wheel`: + + %build + %pyproject_wheel + +And install the wheel in `%install` with `%pyproject_install`: + + %install + %pyproject_install + +`%pyproject_install` installs all wheels in `pyproject-wheeldir/` located in the root of the source tree. + + +Adding run-time and test-time dependencies +------------------------------------------ + +To run tests or import checks in the `%check` section, +the package's runtime dependencies need to also be included as build requirements. + +Hence, `%pyproject_buildrequires` also generates runtime dependencies by default. + +For this to work, the project's build system must support the [prepare-metadata-for-build-wheel hook]. +The popular buildsystems (setuptools, flit, poetry) do support it. + +This behavior can be disabled +(e.g. when the project's build system does not support it) +using the `-R` flag: + + %generate_buildrequires + %pyproject_buildrequires -R + +Alternatively, the runtime dependencies can be obtained by building the wheel and reading the metadata from the built wheel. +This can be enabled by using the `-w` flag. +Support for building wheels with `%pyproject_buildrequires -w` is **provisional** and the behavior might change. +Please subscribe to Fedora's [python-devel list] if you use the option. + + %generate_buildrequires + %pyproject_buildrequires -w + +When this is used, the wheel is going to be built at least twice, +becasue the `%generate_buildrequires` section runs repeatedly. +To avoid accidentally reusing a wheel leaking from a previous (different) build, +it cannot be reused between `%generate_buildrequires` rounds. +Contrarily to that, rebuilding the wheel again in the `%build` section is redundant +and the packager can omit the `%build` section entirely +to reuse the wheel built from the last round of `%generate_buildrequires`. +Be extra careful when attempting to modify the sources after `%pyproject_buildrequires`, +e.g. when running extra commands in the `%build` section: + + %build + cython src/wrong.pyx # this is too late with %%pyproject_buildrequires -w + %pyproject_wheel + +For projects that specify test requirements using an [`extra` +provide](https://packaging.python.org/specifications/core-metadata/#provides-extra-multiple-use), +these can be added using the `-x` flag. +Multiple extras can be supplied by repeating the flag or as a comma separated list. +For example, if upstream suggests installing test dependencies with +`pip install mypackage[testing]`, the test deps would be generated by: + + %generate_buildrequires + %pyproject_buildrequires -x testing + +For projects that specify test requirements in their [tox] configuration, +these can be added using the `-t` flag (default tox environment) +or the `-e` flag followed by the tox environment. +The default tox environment (such as `py37` assuming the Fedora's Python version is 3.7) +is available in the `%{toxenv}` macro. +For example, if upstream suggests running the tests on Python 3.7 with `tox -e py37`, +the test deps would be generated by: + + %generate_buildrequires + %pyproject_buildrequires -t + +If upstream uses a custom derived environment, such as `py37-unit`, use: + + %pyproject_buildrequires -e %{toxenv}-unit + +Or specify more environments if needed: + + %pyproject_buildrequires -e %{toxenv}-unit,%{toxenv}-integration + +The `-e` option redefines `%{toxenv}` for further reuse. +Use `%{default_toxenv}` to get the default value. + +The `-t`/`-e` option uses [tox-current-env]'s `--print-deps-to-file` behind the scenes. + +If your package specifies some tox plugins in `tox.requires`, +such plugins will be BuildRequired as well. +Not all plugins are guaranteed to play well with [tox-current-env], +in worst case, patch/sed the requirement out from the tox configuration. + +Note that neither `-x` or `-t` can be used with `-R`, +because runtime dependencies are always required for testing. +You can only use those options if the build backend supports the [prepare-metadata-for-build-wheel hook], +or together with `-w`. + +[tox]: https://tox.readthedocs.io/ +[tox-current-env]: https://github.com/fedora-python/tox-current-env/ +[prepare-metadata-for-build-wheel hook]: https://www.python.org/dev/peps/pep-0517/#prepare-metadata-for-build-wheel + +Additionally to generated requirements you can supply multiple file names to `%pyproject_buildrequires` macro. +Dependencies will be loaded from them: + + %pyproject_buildrequires requirements/tests.in requirements/docs.in requirements/dev.in + +For packages not using build system you can use `-N` to entirely skip automatical +generation of requirements and install requirements only from manually specified files. +`-N` option implies `-R` and cannot be used in combination with other options mentioned above +(`-w`, `-e`, `-t`, `-x`). + +The `%pyproject_buildrequires` macro also accepts the `-r` flag for backward compatibility; +it means "include runtime dependencies" which has been the default since version 0-53. + + +Running tox based tests +----------------------- + +In case you want to run the tests as specified in [tox] configuration, +you must use `%pyproject_buildrequires` with `-t` or `-e` as explained above. +Then, use the `%tox` macro in `%check`: + + %check + %tox + +The macro: + + - Sets environment variables via `%{py3_test_envvars}`, namely: + - Always prepends `$PATH` with `%{buildroot}%{_bindir}` + - If not defined, sets `$PYTHONPATH` to `%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}` + - If not defined, sets `$TOX_TESTENV_PASSENV` to `*` + - Runs `tox` with `-q` (quiet), `--recreate` and `--current-env` (from [tox-current-env]) flags + - Implicitly uses the tox environment name stored in `%{toxenv}` - as overridden by `%pyproject_buildrequires -e` + +By using the `-e` flag, you can use a different tox environment(s): + + %check + %tox + %if %{with integration_test} + %tox -e %{default_toxenv}-integration + %endif + +If you wish to provide custom `tox` flags or arguments, add them after `--`: + + %tox -- --flag-for-tox + +If you wish to pass custom `posargs` to tox, use another `--`: + + %tox -- --flag-for-tox -- --flag-for-posargs + +Or (note the two sequential `--`s): + + %tox -- -- --flag-for-posargs + + + +Generating the %files section +----------------------------- + +To generate the list of files in the `%files` section, you can use `%pyproject_save_files` after the `%pyproject_install` macro. +It takes toplevel module names (i.e. the names used with `import` in Python) and stores paths for those modules and metadata for the package (dist-info directory) to a file stored at `%{pyproject_files}`. +For example, if a package provides the modules `requests` and `_requests`, write: + + %install + %pyproject_install + %pyproject_save_files requests _requests + +To add listed files to the `%files` section, use `%files -f %{pyproject_files}`. +Note that you still need to add any documentation manually (for now). + + %files -n python3-requests -f %{pyproject_files} + %doc README.rst + +You can use globs in the module names if listing them explicitly would be too tedious: + + %install + %pyproject_install + %pyproject_save_files '*requests' + +In fully automated environments, you can use the `*` glob to include all modules (put it in single quotes to prevent Shell from expanding it). In Fedora however, you should always use a more specific glob to avoid accidentally packaging unwanted files (for example, a top level module named `test`). + +Speaking about automated environments, some files cannot be classified with `%pyproject_save_files`, but it is possible to list all unclassified files by adding a special `+auto` argument. + + %install + %pyproject_install + %pyproject_save_files '*' +auto + + %files -n python3-requests -f %{pyproject_files} + +However, in Fedora packages, always list executables explicitly to avoid unintended collisions with other packages or accidental missing executables: + + %install + %pyproject_install + %pyproject_save_files requests _requests + + %files -n python3-requests -f %{pyproject_files} + %doc README.rst + %{_bindir}/downloader + +`%pyproject_save_files` can automatically mark license files with `%license` macro +and language (`*.mo`) files with `%lang` macro and appropriate language code. +Only license files declared via [PEP 639] `License-File` field are detected. +[PEP 639] is still a draft and can be changed in the future. + +Note that `%pyproject_save_files` uses data from the [RECORD file](https://www.python.org/dev/peps/pep-0627/). +If you wish to rename, remove or otherwise change the installed files of a package +*after* `%pyproject_install`, `%pyproject_save_files` might break. +If possible, remove/rename such files in `%prep`. +If not possible, avoid using `%pyproject_save_files` or edit/replace `%{pyproject_files}`. + + +Performing an import check on all importable modules +---------------------------------------------------- + +If the upstream test suite cannot be used during the package build +and you use `%pyproject_save_files`, +you can benefit from the `%pyproject_check_import` macro. +If `%pyproject_save_files` is not used, calling `%pyproject_check_import` will fail. + +When `%pyproject_save_files` is invoked, +it creates a list of all valid and public (i.e. not starting with `_`) +importable module names found in the package. +Each top-level module name matches at least one of the globs provided as an argument to `%pyproject_save_files`. +This list is then usable by `%pyproject_check_import` which performs an import check for each listed module. +When a module fails to import, the build fails. + +The modules are imported from both installed and buildroot's `%{python3_sitearch}` +and `%{python3_sitelib}`, not from the current directory. + +Use the macro in `%check`: + + %check + %pyproject_check_import + +By using the `-e` flag, you can exclude module names matching the given glob(s) from the import check +(put it in single quotes to prevent Shell from expanding it). +The flag can be used repeatedly. +For example, to exclude all submodules ending with `config` and all submodules starting with `test`, you can use: + + %pyproject_check_import -e '*.config' -e '*.test*' + +There must be at least one module left for the import check; +if, as a result of greedy excluding, no modules are left to check, the check fails. + +When the `-t` flag is used, only top-level modules are checked, +qualified module names with a dot (`.`) are excluded. +If the modules detected by `%pyproject_save_files` are `requests`, `requests.models`, and `requests.packages`, this will only perform an import of `requests`: + + %pyproject_check_import -t + +The modifying flags should only be used when there is a valid reason for not checking all available modules. +The reason should be documented in a comment. + +The `%pyproject_check_import` macro also accepts positional arguments with +additional qualified module names to check, useful for example if some modules are installed manually. +Note that filtering by `-t`/`-e` also applies to the positional arguments. + + +Generating Extras subpackages +----------------------------- + +The `%pyproject_extras_subpkg` macro generates simple subpackage(s) +for Python extras. + +The macro should be placed after the base package's `%description` to avoid +issues in building the SRPM. + +For example, if the `requests` project's metadata defines the extras +`security` and `socks`, the following invocation will generate the subpackage +`python3-requests+security` that provides `python3dist(requests[security])`, +and a similar one for `socks`. + + %pyproject_extras_subpkg -n python3-requests security socks + +The macro works like `%python_extras_subpkg`, +except the `-i`/`-f`/`-F` arguments are optional and discouraged. +A filelist written by `%pyproject_install` is used by default. +For more information on `%python_extras_subpkg`, see the [Fedora change]. + +[Fedora change]: https://fedoraproject.org/wiki/Changes/PythonExtras + +These arguments are still required: + +* -n: name of the “base” package (e.g. python3-requests) +* Positional arguments: the extra name(s). + Multiple subpackages are generated when multiple names are provided. + + +PROVISIONAL: Importing just-built (extension) modules in %build +--------------------------------------------------------------- + +Sometimes, it is desired to be able to import the *just-built* extension modules +in the `%build` section, e.g. to build the documentation with Sphinx. + + %build + %pyproject_wheel + ... build the docs here ... + +With pure Python packages, it might be possible to set `PYTHONPATH=${PWD}` or `PYTHONPATH=${PWD}/src`. +However, it is a bit more complicated with extension modules. + +The location of just-built modules might differ depending on Python version, architecture, pip version, etc. +Hence, the macro `%{pyproject_build_lib}` exists to be used like this: + + %build + %pyproject_wheel + PYTHONPATH=%{pyproject_build_lib} ... build the docs here ... + +This macro is currently **provisional** and the behavior might change. +Please subscribe to Fedora's [python-devel list] if you use the macro. + +The `%{pyproject_build_lib}` macro expands to an Shell `$(...)` expression and does not work when put into single quotes (`'`). + +Depending on the pip version, the expanded value will differ: + +[python-devel list]: https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/ + +### New pip 21.3+ with in-tree-build and setuptools 62.1+ (Fedora 37+) + +Always use the macro from the same directory where you called `%pyproject_wheel` from. +The value will expand to something like: + +* `/builddir/build/BUILD/%{name}-%{version}/build/lib.linux-x86_64-cpython-311` for wheels with extension modules +* `/builddir/build/BUILD/%{name}-%{version}/build/lib` for pure Python wheels + +If multiple wheels were built from the same directory, +some pure Python and some with extension modules, +the expanded value will be combined with `:`: + +* `/builddir/build/BUILD/%{name}-%{version}/build/lib.linux-x86_64-cypthon-311:/builddir/build/BUILD/%{name}-%{version}/build/lib` + +If multiple wheels were built from different directories, +the value will differ depending on the current directory. + + +### New pip 21.3+ with in-tree-build and older setuptools (Fedora 36) + +Always use the macro from the same directory where you called `%pyproject_wheel` from. +The value will expand to something like: + +* `/builddir/build/BUILD/%{name}-%{version}/build/lib.linux-x86_64-3.10` for wheels with extension modules +* `/builddir/build/BUILD/%{name}-%{version}/build/lib` for pure Python wheels + +If multiple wheels were built from the same directory, +some pure Python and some with extension modules, +the expanded value will be combined with `:`: + +* `/builddir/build/BUILD/%{name}-%{version}/build/lib.linux-x86_64-3.10:/builddir/build/BUILD/%{name}-%{version}/build/lib` + +If multiple wheels were built from different directories, +the value will differ depending on the current directory. + + +### Older pip with out-of-tree-build (Fedora 35 and EL 9) + +The value will expand to something like: + +* `/builddir/build/BUILD/%{name}-%{version}/.pyproject-builddir/pip-req-build-xxxxxxxx/build/lib.linux-x86_64-3.10` for wheels with extension modules +* `/builddir/build/BUILD/%{name}-%{version}/.pyproject-builddir/pip-req-build-xxxxxxxx/build/lib` for pure Python wheels + +Note that the exact value is **not stable** between builds +(the `xxxxxxxx` part is randomly generated, +neither you should consider the `.pyproject-builddir` directory to remain stable). + +If multiple wheels are built, +the expanded value will always be combined with `:` regardless of the current directory, e.g.: + +* `/builddir/build/BUILD/%{name}-%{version}/.pyproject-builddir/pip-req-build-xxxxxxxx/build/lib.linux-x86_64-3.10:/builddir/build/BUILD/%{name}-%{version}/.pyproject-builddir/pip-req-build-yyyyyyyy/build/lib.linux-x86_64-3.10:/builddir/build/BUILD/%{name}-%{version}/.pyproject-builddir/pip-req-build-zzzzzzzz/build/lib` + +**Note:** If you manage to build some wheels with in-tree-build and some with out-of-tree-build option, +the expanded value will contain all relevant directories. + + +Limitations +----------- + +`%pyproject_install` changes shebang lines of every Python script in `%{buildroot}%{_bindir}` to `#!%{__python3} %{py3_shbang_opt}` (`#!/usr/bin/python3 -s`). +Existing Python flags in shebangs are preserved. +For example `#!/usr/bin/python3 -Ru` will be updated to `#!/usr/bin/python3 -sRu`. +Sometimes, this can interfere with tests that run such scripts directly by name, +because in tests we usually rely on `PYTHONPATH` (and `-s` ignores that). +Would this behavior be undesired for any reason, +undefine `%{py3_shbang_opt}` to turn it off. + +Some valid Python version specifiers are not supported. + +When a dependency is specified via an URL or local path, for example as: + + https://github.com/ActiveState/appdirs/archive/8eacfa312d77aba28d483fbfb6f6fc54099622be.zip + /some/path/foo-1.2.3.tar.gz + git+https://github.com/sphinx-doc/sphinx.git@96dbe5e3 + +The `%pyproject_buildrequires` macro is unable to convert it to an appropriate RPM requirement and will fail. +If the URL contains the `packageName @` prefix as specified in [PEP 508], +the requirement will be generated without a version constraint: + + appdirs@https://github.com/ActiveState/appdirs/archive/8eacfa312d77aba28d483fbfb6f6fc54099622be.zip + foo@file:///some/path/foo-1.2.3.tar.gz + +Will be converted to: + + python3dist(appdirs) + python3dist(foo) + +Alternatively, when an URL requirement parsed from a text file +given as positional argument to `%pyproject_buildrequires` +contains the `#egg=packageName` fragment, +as documented in [pip's documentation]: + + git+https://github.com/sphinx-doc/sphinx.git@96dbe5e3#egg=sphinx + +The requirements will be converted to package names without versions, e.g.: + + python3dist(sphinx) + +However upstreams usually only use direct URLs for their requirements as workarounds, +so be prepared for problems. + +[PEP 508]: https://www.python.org/dev/peps/pep-0508/ +[PEP 517]: https://www.python.org/dev/peps/pep-0517/ +[PEP 518]: https://www.python.org/dev/peps/pep-0518/ +[PEP 639]: https://www.python.org/dev/peps/pep-0639/ +[pip's documentation]: https://pip.pypa.io/en/stable/cli/pip_install/#vcs-support + + +Testing the macros +------------------ + +This repository has two kinds of tests. +First, there is RPM `%check` section, run when building the `python-rpm-macros` +package. + +Then there are CI tests. +There is currently [no way to run Fedora CI tests locally][ci-rfe], +but you can do what the tests do manually using mock. +For each `$PKG.spec` in `tests/`: + + - clean your mock environment: + + mock -r fedora-rawhide-x86_64 clean + + - install the version of `python-rpm-macros` you're testing, e.g.: + + mock -r fedora-rawhide-x86_64 install .../python-rpm-macros-*.noarch.rpm + + - download the sources: + + spectool -g -R $PKG.spec + + - build a SRPM: + + rpmbuild -bs $PKG.spec + + - build in mock, using the path from the command above as `$SRPM`: + + mock -r fedora-rawhide-x86_64 -n -N $SRPM + +[ci-rfe]: https://pagure.io/fedora-ci/general/issue/4 diff --git a/macros.pyproject b/macros.pyproject index d8cfb4114875d37fb6446ec37884f09b8bde6cfe..b3b2ec07bf2a19148711c441d83dabcf16cc6f6e 100644 --- a/macros.pyproject +++ b/macros.pyproject @@ -1,4 +1,4 @@ -# This is a directory where wheels are stored and installed from, relative to PWD +# This is a directory where wheels are stored and installed from, absolute %_pyproject_wheeldir %{_builddir}%{?buildsubdir:/%{buildsubdir}}/pyproject-wheeldir # This is a directory used as TMPDIR, where pip copies sources to and builds from, relative to PWD @@ -10,10 +10,15 @@ # https://docs.pytest.org/en/latest/reference.html#confval-norecursedirs %_pyproject_builddir %{_builddir}%{?buildsubdir:/%{buildsubdir}}/.pyproject-builddir -%pyproject_files %{_builddir}/pyproject-files -%_pyproject_modules %{_builddir}/pyproject-modules -%_pyproject_ghost_distinfo %{_builddir}/pyproject-ghost-distinfo -%_pyproject_record %{_builddir}/pyproject-record +# We prefix all created files with this value to make them unique +# Ideally, we would put them into %%{buildsubdir}, but that value changes during the spec +# The used value is similar to the one used to define the default %%buildroot +%_pyproject_files_prefix %{name}-%{version}-%{release}.%{_arch} + +%pyproject_files %{_builddir}/%{_pyproject_files_prefix}-pyproject-files +%_pyproject_modules %{_builddir}/%{_pyproject_files_prefix}-pyproject-modules +%_pyproject_ghost_distinfo %{_builddir}/%{_pyproject_files_prefix}-pyproject-ghost-distinfo +%_pyproject_record %{_builddir}/%{_pyproject_files_prefix}-pyproject-record # Avoid leaking %%{_pyproject_builddir} to pytest collection # https://bugzilla.redhat.com/show_bug.cgi?id=1935212 @@ -24,13 +29,16 @@ %_set_pytest_addopts mkdir -p "%{_pyproject_builddir}" CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}" TMPDIR="%{_pyproject_builddir}" \\\ -%{__python3} -m pip wheel --wheel-dir %{_pyproject_wheeldir} --no-deps --use-pep517 --no-build-isolation --disable-pip-version-check --no-clean --progress-bar off --verbose . +%{__python3} -Bs %{_rpmconfigdir}/anolis/pyproject_wheel.py %{_pyproject_wheeldir} } %pyproject_build_lib %{expand:\\\ $( pyproject_build_lib=() +if [ -d build/lib.%{python3_platform}-cpython-%{python3_version_nodots} ]; then + pyproject_build_lib+=( "${PWD}/build/lib.%{python3_platform}-cpython-%{python3_version_nodots}" ) +fi if [ -d build/lib.%{python3_platform}-%{python3_version} ]; then pyproject_build_lib+=( "${PWD}/build/lib.%{python3_platform}-%{python3_version}" ) fi @@ -49,7 +57,7 @@ echo $(IFS=:; echo "${pyproject_build_lib[*]}") %pyproject_install() %{expand:\\\ specifier=$(ls %{_pyproject_wheeldir}/*.whl | xargs basename --multiple | sed -E 's/([^-]+)-([^-]+)-.+\\\.whl/\\\1==\\\2/') -TMPDIR="%{_pyproject_builddir}" %{__python3} -m pip install --root %{buildroot} --no-deps --disable-pip-version-check --progress-bar off --verbose --ignore-installed --no-warn-script-location --no-index --no-cache-dir --find-links %{_pyproject_wheeldir} $specifier +TMPDIR="%{_pyproject_builddir}" %{__python3} -m pip install --root %{buildroot} --prefix %{_prefix} --no-deps --disable-pip-version-check --progress-bar off --verbose --ignore-installed --no-warn-script-location --no-index --no-cache-dir --find-links %{_pyproject_wheeldir} $specifier if [ -d %{buildroot}%{_bindir} ]; then %py3_shebang_fix %{buildroot}%{_bindir}/* rm -rfv %{buildroot}%{_bindir}/__pycache__ @@ -116,13 +124,24 @@ fi %toxenv %{default_toxenv} -%pyproject_buildrequires(rRxtNe:) %{expand:\\\ -%{-R:%{-r:%{error:The -R and -r options are mutually exclusive}}} +%pyproject_buildrequires(rRxtNwe:) %{expand:\\\ +%_set_pytest_addopts +# The _auto_set_build_flags feature does not do this in %%generate_buildrequires section, +# but we want to get an environment consistent with %%build: +%{?_auto_set_build_flags:%set_build_flags} +# The default flags expect the package note file to exist +# see https://bugzilla.redhat.com/show_bug.cgi?id=2097535 +%{?_package_note_flags:%_generate_package_note_file} +%{-R: +%{-r:%{error:The -R and -r options are mutually exclusive}} +%{-w:%{error:The -R and -w options are mutually exclusive}} +} %{-N: %{-r:%{error:The -N and -r options are mutually exclusive}} %{-x:%{error:The -N and -x options are mutually exclusive}} %{-e:%{error:The -N and -e options are mutually exclusive}} %{-t:%{error:The -N and -t options are mutually exclusive}} +%{-w:%{error:The -N and -w options are mutually exclusive}} } %{-e:%{expand:%global toxenv %(%{__python3} -s %{_rpmconfigdir}/anolis/pyproject_construct_toxenv.py %{?**})}} echo 'pyproject-rpm-macros' # we already have this installed, but this way, it's repoqueryable @@ -130,7 +149,13 @@ echo 'python%{python3_pkgversion}-devel' echo 'python%{python3_pkgversion}dist(pip) >= 19' echo 'python%{python3_pkgversion}dist(packaging)' %{!-N:if [ -f pyproject.toml ]; then - echo 'python%{python3_pkgversion}dist(toml)' + %["%{python3_pkgversion}" == "3" + ? "echo '(python%{python3_pkgversion}dist(toml) if python%{python3_pkgversion}-devel < 3.11)'" + : "%[v"%{python3_pkgversion}" < v"3.11" + ? "echo 'python%{python3_pkgversion}dist(toml)'" + : "true # will use tomllib, echo nothing" + ]" + ] elif [ -f setup.py ]; then # Note: If the default requirements change, also change them in the script! echo 'python%{python3_pkgversion}dist(setuptools) >= 40.8' @@ -142,17 +167,19 @@ fi} # setuptools assumes no pre-existing dist-info rm -rfv *.dist-info/ >&2 if [ -f %{__python3} ]; then - RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -s %{_rpmconfigdir}/anolis/pyproject_buildrequires.py %{?!_python_no_extras_requires:--generate-extras} --python3_pkgversion %{python3_pkgversion} %{?**} + mkdir -p "%{_pyproject_builddir}" + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}" TMPDIR="%{_pyproject_builddir}" \\\ + RPM_TOXENV="%{toxenv}" HOSTNAME="rpmbuild" %{__python3} -Bs %{_rpmconfigdir}/anolis/pyproject_buildrequires.py %{?!_python_no_extras_requires:--generate-extras} --python3_pkgversion %{python3_pkgversion} --wheeldir %{_pyproject_wheeldir} %{?**} fi } %tox(e:) %{expand:\\\ TOX_TESTENV_PASSENV="${TOX_TESTENV_PASSENV:-*}" \\ -PYTHONDONTWRITEBYTECODE=1 \\ +%{?py3_test_envvars}%{?!py3_test_envvars:PYTHONDONTWRITEBYTECODE=1 \\ PATH="%{buildroot}%{_bindir}:$PATH" \\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}" \\ -%{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"} \\ +%{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}} \\ HOSTNAME="rpmbuild" \\ %{__python3} -m tox --current-env -q --recreate -e "%{-e:%{-e*}}%{!-e:%{toxenv}}" %{?*} } diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index 22f893c9902efe0504530c84c989ecd6e337398d..d0f74210c7b0ebc8ff7a1000139c65277e3c8660 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -11,7 +11,7 @@ License: MIT # Increment Y and reset Z when new macros or features are added # Increment Z when this is a bugfix or a cosmetic change # Dropping support for EOL Fedoras is *not* considered a breaking change -Version: 1.0.0 +Version: 1.5.1 Release: %{anolis_release}%{?dist} # Macro files @@ -24,6 +24,7 @@ Source103: pyproject_convert.py Source104: pyproject_preprocess_record.py Source105: pyproject_construct_toxenv.py Source106: pyproject_requirements_txt.py +Source107: pyproject_wheel.py # Tests Source201: test_pyproject_buildrequires.py @@ -45,24 +46,27 @@ URL: https://src.fedoraproject.org/rpms/pyproject-rpm-macros BuildArch: noarch %if %{with tests} -BuildRequires: python3dist(pytest) -BuildRequires: python3dist(pyyaml) -BuildRequires: python3dist(packaging) -BuildRequires: python3dist(pip) -BuildRequires: python3dist(setuptools) -BuildRequires: python3dist(toml) -BuildRequires: python3dist(tox-current-env) >= 0.0.6 -BuildRequires: python3dist(wheel) +BuildRequires: python3dist(pytest) +BuildRequires: python3dist(pyyaml) +BuildRequires: python3dist(packaging) +BuildRequires: python3dist(pip) +BuildRequires: python3dist(setuptools) +BuildRequires: python3dist(tox-current-env) >= 0.0.6 +BuildRequires: python3dist(wheel) +BuildRequires: (python3dist(toml) if python3-devel < 3.11) %endif # We build on top of those: -Requires: python-rpm-macros -Requires: python-srpm-macros -Requires: python3-rpm-macros +BuildRequires: python-rpm-macros +BuildRequires: python-srpm-macros +BuildRequires: python3-rpm-macros +Requires: python-rpm-macros +Requires: python-srpm-macros +Requires: python3-rpm-macros # We use the following tools outside of coreutils -Requires: /usr/bin/find -Requires: /usr/bin/sed +Requires: /usr/bin/find +Requires: /usr/bin/sed %description These macros allow projects that follow the Python packaging specifications @@ -91,18 +95,19 @@ cp -p %{sources} . %install mkdir -p %{buildroot}%{_rpmmacrodir} mkdir -p %{buildroot}%{_rpmconfigdir}/anolis -install -m 644 macros.pyproject %{buildroot}%{_rpmmacrodir}/ -install -m 644 pyproject_buildrequires.py %{buildroot}%{_rpmconfigdir}/anolis/ -install -m 644 pyproject_convert.py %{buildroot}%{_rpmconfigdir}/anolis/ -install -m 644 pyproject_save_files.py %{buildroot}%{_rpmconfigdir}/anolis/ -install -m 644 pyproject_preprocess_record.py %{buildroot}%{_rpmconfigdir}/anolis/ -install -m 644 pyproject_construct_toxenv.py %{buildroot}%{_rpmconfigdir}/anolis/ -install -m 644 pyproject_requirements_txt.py %{buildroot}%{_rpmconfigdir}/anolis/ +install -pm 644 macros.pyproject %{buildroot}%{_rpmmacrodir}/ +install -pm 644 pyproject_buildrequires.py %{buildroot}%{_rpmconfigdir}/anolis/ +install -pm 644 pyproject_convert.py %{buildroot}%{_rpmconfigdir}/anolis/ +install -pm 644 pyproject_save_files.py %{buildroot}%{_rpmconfigdir}/anolis/ +install -pm 644 pyproject_preprocess_record.py %{buildroot}%{_rpmconfigdir}/anolis/ +install -pm 644 pyproject_construct_toxenv.py %{buildroot}%{_rpmconfigdir}/anolis/ +install -pm 644 pyproject_requirements_txt.py %{buildroot}%{_rpmconfigdir}/anolis/ +install -pm 644 pyproject_wheel.py %{buildroot}%{_rpmconfigdir}/anolis/ %if %{with tests} %check export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856356 -%{python3} -m pytest -vv --doctest-modules +%pytest -vv --doctest-modules # brp-compress is provided as an argument to get the right directory macro expansion %{python3} compare_mandata.py -f %{_rpmconfigdir}/brp-compress @@ -117,10 +122,14 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 %{_rpmconfigdir}/anolis/pyproject_preprocess_record.py %{_rpmconfigdir}/anolis/pyproject_construct_toxenv.py %{_rpmconfigdir}/anolis/pyproject_requirements_txt.py +%{_rpmconfigdir}/anolis/pyproject_wheel.py %doc README.md %license LICENSE %changelog -* Thu Mar 02 2022 Chunmei Xu - 1.0.0-1 +* Fri Jan 20 2023 Funda Wang - 1.5.1-1 +- Sync with upstream 1.5.1 + +* Wed Mar 02 2022 Chunmei Xu - 1.0.0-1 - upstream version 1.0.0 diff --git a/pyproject_buildrequires.py b/pyproject_buildrequires.py index 21e74da32cc00322834fb773687767973ee8cb9f..01c57f124785970e4c97f64fc428297cee45b971 100644 --- a/pyproject_buildrequires.py +++ b/pyproject_buildrequires.py @@ -1,3 +1,5 @@ +import glob +import io import os import sys import importlib.metadata @@ -11,6 +13,7 @@ import re import tempfile import email.parser import pathlib +import zipfile from pyproject_requirements_txt import convert_requirements_txt @@ -97,7 +100,7 @@ class Requirements: return [{'extra': e} for e in sorted(self.extras)] return [{'extra': ''}] - def evaluate_all_environamnets(self, requirement): + def evaluate_all_environments(self, requirement): for marker_env in self.marker_envs: if requirement.marker.evaluate(environment=marker_env): return True @@ -123,7 +126,7 @@ class Requirements: name = canonicalize_name(requirement.name) if (requirement.marker is not None and - not self.evaluate_all_environamnets(requirement)): + not self.evaluate_all_environments(requirement)): print_err(f'Ignoring alien requirement:', requirement_str) return @@ -184,21 +187,32 @@ class Requirements: self.add(req_str, **kwargs) -def get_backend(requirements): +def toml_load(opened_binary_file): try: - f = open('pyproject.toml') - except FileNotFoundError: - pyproject_data = {} - else: + # tomllib is in the standard library since 3.11.0b1 + import tomllib as toml_module + load_from = opened_binary_file + except ImportError: try: - # lazy import toml here, not needed without pyproject.toml - import toml + # note: we could use tomli here, + # but for backwards compatibility with RHEL 9, we use toml instead + import toml as toml_module + load_from = io.TextIOWrapper(opened_binary_file, encoding='utf-8') except ImportError as e: print_err('Import error:', e) # already echoed by the %pyproject_buildrequires macro sys.exit(0) + return toml_module.load(load_from) + + +def get_backend(requirements): + try: + f = open('pyproject.toml', 'rb') + except FileNotFoundError: + pyproject_data = {} + else: with f: - pyproject_data = toml.load(f) + pyproject_data = toml_load(f) buildsystem_data = pyproject_data.get('build-system', {}) requirements.extend( @@ -253,21 +267,67 @@ def generate_build_requirements(backend, requirements): requirements.check(source='get_requires_for_build_wheel') -def generate_run_requirements(backend, requirements): +def requires_from_metadata_file(metadata_file): + message = email.parser.Parser().parse(metadata_file, headersonly=True) + return {k: message.get_all(k, ()) for k in ('Requires', 'Requires-Dist')} + + +def generate_run_requirements_hook(backend, requirements): hook_name = 'prepare_metadata_for_build_wheel' prepare_metadata = getattr(backend, hook_name, None) if not prepare_metadata: raise ValueError( - 'build backend cannot provide build metadata ' - + '(incl. runtime requirements) before build' + 'The build backend cannot provide build metadata ' + '(incl. runtime requirements) before build. ' + 'Use the provisional -w flag to build the wheel and parse the metadata from it, ' + 'or use the -R flag not to generate runtime dependencies.' ) with hook_call(): dir_basename = prepare_metadata('.') - with open(dir_basename + '/METADATA') as f: - message = email.parser.Parser().parse(f, headersonly=True) - for key in 'Requires', 'Requires-Dist': - requires = message.get_all(key, ()) - requirements.extend(requires, source=f'wheel metadata: {key}') + with open(dir_basename + '/METADATA') as metadata_file: + for key, requires in requires_from_metadata_file(metadata_file).items(): + requirements.extend(requires, source=f'hook generated metadata: {key}') + + +def find_built_wheel(wheeldir): + wheels = glob.glob(os.path.join(wheeldir, '*.whl')) + if not wheels: + return None + if len(wheels) > 1: + raise RuntimeError('Found multiple wheels in %{_pyproject_wheeldir}, ' + 'this is not supported with %pyproject_buildrequires -w.') + return wheels[0] + + +def generate_run_requirements_wheel(backend, requirements, wheeldir): + # Reuse the wheel from the previous round of %pyproject_buildrequires (if it exists) + wheel = find_built_wheel(wheeldir) + if not wheel: + import pyproject_wheel + returncode = pyproject_wheel.build_wheel(wheeldir=wheeldir, stdout=sys.stderr) + if returncode != 0: + raise RuntimeError('Failed to build the wheel for %pyproject_buildrequires -w.') + wheel = find_built_wheel(wheeldir) + if not wheel: + raise RuntimeError('Cannot locate the built wheel for %pyproject_buildrequires -w.') + + print_err(f'Reading metadata from {wheel}') + with zipfile.ZipFile(wheel) as wheelfile: + for name in wheelfile.namelist(): + if name.count('/') == 1 and name.endswith('.dist-info/METADATA'): + with io.TextIOWrapper(wheelfile.open(name), encoding='utf-8') as metadata_file: + for key, requires in requires_from_metadata_file(metadata_file).items(): + requirements.extend(requires, source=f'built wheel metadata: {key}') + break + else: + raise RuntimeError('Could not find *.dist-info/METADATA in built wheel.') + + +def generate_run_requirements(backend, requirements, *, build_wheel, wheeldir): + if build_wheel: + generate_run_requirements_wheel(backend, requirements, wheeldir) + else: + generate_run_requirements_hook(backend, requirements) def generate_tox_requirements(toxenv, requirements): @@ -326,7 +386,7 @@ def python3dist(name, op=None, version=None, python3_pkgversion="3"): def generate_requires( - *, include_runtime=False, toxenv=None, extras=None, + *, include_runtime=False, build_wheel=False, wheeldir=None, toxenv=None, extras=None, get_installed_version=importlib.metadata.version, # for dep injection generate_extras=False, python3_pkgversion="3", requirement_files=None, use_build_system=True ): @@ -357,32 +417,37 @@ def generate_requires( include_runtime = True generate_tox_requirements(toxenv, requirements) if include_runtime: - generate_run_requirements(backend, requirements) + generate_run_requirements(backend, requirements, build_wheel=build_wheel, wheeldir=wheeldir) except EndPass: return def main(argv): parser = argparse.ArgumentParser( - description='Generate BuildRequires for a Python project.' + description='Generate BuildRequires for a Python project.', + prog='%pyproject_buildrequires', + add_help=False, + ) + parser.add_argument( + '--help', action='help', + default=argparse.SUPPRESS, + help=argparse.SUPPRESS, ) parser.add_argument( '-r', '--runtime', action='store_true', default=True, - help='Generate run-time requirements (default, disable with -R)', + help=argparse.SUPPRESS, # Generate run-time requirements (backwards-compatibility only) ) parser.add_argument( - '-R', '--no-runtime', action='store_false', dest='runtime', - help="Don't generate run-time requirements (implied by -N)", + '--generate-extras', action='store_true', + help=argparse.SUPPRESS, ) parser.add_argument( - '-e', '--toxenv', metavar='TOXENVS', action='append', - help=('specify tox environments (comma separated and/or repeated)' - '(implies --tox)'), + '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION', + default="3", help=argparse.SUPPRESS, ) parser.add_argument( - '-t', '--tox', action='store_true', - help=('generate test tequirements from tox environment ' - '(implies --runtime)'), + '--wheeldir', metavar='PATH', default=None, + help=argparse.SUPPRESS, ) parser.add_argument( '-x', '--extras', metavar='EXTRAS', action='append', @@ -390,20 +455,31 @@ def main(argv): '(e.g. -x testing,feature-x) (implies --runtime, can be repeated)', ) parser.add_argument( - '--generate-extras', action='store_true', - help='Generate build requirements on Python Extras', + '-t', '--tox', action='store_true', + help=('generate test tequirements from tox environment ' + '(implies --runtime)'), ) parser.add_argument( - '-p', '--python3_pkgversion', metavar='PYTHON3_PKGVERSION', - default="3", help=('Python version for pythonXdist()' - 'or pythonX.Ydist() requirements'), + '-e', '--toxenv', metavar='TOXENVS', action='append', + help=('specify tox environments (comma separated and/or repeated)' + '(implies --tox)'), + ) + parser.add_argument( + '-w', '--wheel', action='store_true', default=False, + help=('Generate run-time requirements by building the wheel ' + '(useful for build backends without the prepare_metadata_for_build_wheel hook)'), + ) + parser.add_argument( + '-R', '--no-runtime', action='store_false', dest='runtime', + help="Don't generate run-time requirements (implied by -N)", ) parser.add_argument( '-N', '--no-use-build-system', dest='use_build_system', action='store_false', help='Use -N to indicate that project does not use any build system', ) parser.add_argument( - 'requirement_files', nargs='*', type=argparse.FileType('r'), + 'requirement_files', nargs='*', type=argparse.FileType('r'), + metavar='REQUIREMENTS.TXT', help=('Add buildrequires from file'), ) @@ -412,6 +488,10 @@ def main(argv): if not args.use_build_system: args.runtime = False + if args.wheel: + if not args.wheeldir: + raise ValueError('--wheeldir must be set when -w.') + if args.toxenv: args.tox = True @@ -427,6 +507,8 @@ def main(argv): try: generate_requires( include_runtime=args.runtime, + build_wheel=args.wheel, + wheeldir=args.wheeldir, toxenv=args.toxenv, extras=args.extras, generate_extras=args.generate_extras, diff --git a/pyproject_buildrequires_testcases.yaml b/pyproject_buildrequires_testcases.yaml index 5e4b5e5f4699719d03b6cdeb82325a8da108fc72..75934dfc07def8422e23ff1314324bd7af8527bd 100644 --- a/pyproject_buildrequires_testcases.yaml +++ b/pyproject_buildrequires_testcases.yaml @@ -268,6 +268,8 @@ Run dependencies with extras (not selected): def main(): setup( + name = "pytest", + version = "6.6.6", setup_requires=["setuptools>=40.0"], # fmt: off extras_require={ @@ -358,6 +360,35 @@ Run dependencies with multiple extras: python3dist(dep1) result: 0 +Run dependencies with extras and build wheel option: + installed: + setuptools: 50 + wheel: 1 + pyyaml: 1 + include_runtime: true + build_wheel: true + extras: + - testing + setup.py: *pytest_setup_py + expected: | + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(setuptools) >= 40 + python3dist(py) >= 1.5 + python3dist(six) >= 1.10 + python3dist(setuptools) + python3dist(attrs) >= 17.4 + python3dist(atomicwrites) >= 1 + python3dist(pluggy) >= 0.11 + python3dist(more-itertools) >= 4 + python3dist(argcomplete) + python3dist(hypothesis) >= 3.56 + python3dist(nose) + python3dist(requests) + result: 0 + stderr_contains: "Reading metadata from {wheeldir}/pytest-6.6.6-py3-none-any.whl" + Tox dependencies: installed: setuptools: 50 @@ -382,14 +413,24 @@ Tox dependencies: toxdep2 commands = true - expected: | - python3dist(setuptools) >= 40.8 - python3dist(wheel) - python3dist(wheel) - python3dist(tox-current-env) >= 0.0.6 - python3dist(toxdep1) - python3dist(toxdep2) - python3dist(inst) + expected: + - | # tox 3 + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(tox-current-env) >= 0.0.6 + python3dist(toxdep1) + python3dist(toxdep2) + python3dist(inst) + - | # tox 4 + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(tox-current-env) >= 0.0.6 + python3dist(tox) + python3dist(toxdep1) + python3dist(toxdep2) + python3dist(inst) result: 0 Tox extras: @@ -424,20 +465,36 @@ Tox extras: extra1 commands = true - expected: | - python3dist(setuptools) >= 40.8 - python3dist(wheel) - python3dist(wheel) - python3dist(tox-current-env) >= 0.0.6 - python3dist(toxdep) - python3dist(inst) - python3dist(dep11) > 11.0 - python3dist(dep12) - python3dist(dep21) - python3dist(dep22) - python3dist(dep23) - python3dist(extra-dep) - python3dist(extra-dep[extra_dep]) + expected: + - | # tox 3 + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(tox-current-env) >= 0.0.6 + python3dist(toxdep) + python3dist(inst) + python3dist(dep11) > 11.0 + python3dist(dep12) + python3dist(dep21) + python3dist(dep22) + python3dist(dep23) + python3dist(extra-dep) + python3dist(extra-dep[extra_dep]) + - | # tox 4 + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(tox-current-env) >= 0.0.6 + python3dist(tox) + python3dist(toxdep) + python3dist(inst) + python3dist(dep11) > 11.0 + python3dist(dep12) + python3dist(dep21) + python3dist(dep22) + python3dist(dep23) + python3dist(extra-dep) + python3dist(extra-dep[extra_dep]) result: 0 Tox provision unsatisfied: @@ -465,14 +522,24 @@ Tox provision unsatisfied: deps = toxdep1 toxdep2 - expected: | - python3dist(setuptools) >= 40.8 - python3dist(wheel) - python3dist(wheel) - python3dist(tox-current-env) >= 0.0.6 - python3dist(tox) >= 3.999 - python3dist(setuptools) > 40.0 - python3dist(wheel) > 2.0 + expected: + - | # tox 3 + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(tox-current-env) >= 0.0.6 + python3dist(tox) >= 3.999 + python3dist(setuptools) > 40.0 + python3dist(wheel) > 2.0 + - | # tox 4 + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(tox-current-env) >= 0.0.6 + python3dist(tox) >= 3.999 + python3dist(setuptools) > 40.0 + python3dist(wheel) > 2.0 + python3dist(tox) >= 3.999 result: 0 Tox provision satisfied: @@ -499,16 +566,27 @@ Tox provision satisfied: deps = toxdep1 toxdep2 - expected: | - python3dist(setuptools) >= 40.8 - python3dist(wheel) - python3dist(wheel) - python3dist(tox-current-env) >= 0.0.6 - python3dist(tox) >= 3.5 - python3dist(setuptools) > 40.0 - python3dist(toxdep1) - python3dist(toxdep2) - python3dist(inst) + expected: + - | # tox 3 + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(tox-current-env) >= 0.0.6 + python3dist(tox) >= 3.5 + python3dist(setuptools) > 40.0 + python3dist(toxdep1) + python3dist(toxdep2) + python3dist(inst) + - | # tox 4 + python3dist(setuptools) >= 40.8 + python3dist(wheel) + python3dist(wheel) + python3dist(tox-current-env) >= 0.0.6 + python3dist(setuptools) > 40.0 + python3dist(tox) >= 3.5 + python3dist(toxdep1) + python3dist(toxdep2) + python3dist(inst) result: 0 Default build system, unmet deps in requirements file: diff --git a/pyproject_requirements_txt.py b/pyproject_requirements_txt.py index d2389e8100aed0d539089d8028cd13b599e9901b..5ff1f26f5c0f3cdfb1c90338be1ecb0830203cc3 100644 --- a/pyproject_requirements_txt.py +++ b/pyproject_requirements_txt.py @@ -98,6 +98,4 @@ def expand_env_vars(lines): return match['var'] return value for line in lines: - if match := ENV_VAR_RE.search(line): - var = match['var'] yield ENV_VAR_RE.sub(repl, line) diff --git a/pyproject_save_files.py b/pyproject_save_files.py index 6cd53e778bbd86a0ea452fa1a3955bd758b39c70..9728b227bc88ca9cf3a1a609a20f4140797d3b44 100644 --- a/pyproject_save_files.py +++ b/pyproject_save_files.py @@ -286,6 +286,36 @@ def module_names_from_path(path): return {'.'.join(parts[:x+1]) for x in range(len(parts))} +def is_license_file(path, license_files, license_directories): + """ + Check if the given BuildrootPath path matches any of the "License-File" entries. + The path is considered matched when resolved from any of the license_directories + matches string-wise what is stored in any "License-File" entry (license_files). + + Examples: + >>> site_packages = BuildrootPath('/usr/lib/python3.12/site-packages') + >>> distinfo = site_packages / 'foo-1.0.dist-info' + >>> license_directories = [distinfo / 'licenses', distinfo] + >>> license_files = ['LICENSE.txt', 'AUTHORS.md'] + >>> is_license_file(distinfo / 'AUTHORS.md', license_files, license_directories) + True + >>> is_license_file(distinfo / 'licenses/LICENSE.txt', license_files, license_directories) + True + >>> # we don't match based on directory only + >>> is_license_file(distinfo / 'licenses/COPYING', license_files, license_directories) + False + >>> is_license_file(site_packages / 'foo/LICENSE.txt', license_files, license_directories) + False + """ + if not license_files or not license_directories: + return False + for license_dir in license_directories: + if (path.is_relative_to(license_dir) and + str(path.relative_to(license_dir)) in license_files): + return True + return False + + def classify_paths( record_path, parsed_record_content, metadata, sitedirs, python_version, prefix ): @@ -311,24 +341,36 @@ def classify_paths( "other": {"files": []}, # regular %file entries we could not parse :( } + license_files = metadata.get_all('License-File') + license_directory = distinfo / 'licenses' # See PEP 369 "Root License Directory" + # setuptools was the first known build backend to implement License-File. + # Unfortunately they don't put licenses to the license directory (yet): + # https://github.com/pypa/setuptools/issues/3596 + # Hence, we check licenses in both licenses and dist-info + license_directories = (license_directory, distinfo) + # In RECORDs generated by pip, there are no directories, only files. # The example RECORD from PEP 376 does not contain directories either. # Hence, we'll only assume files, but TODO get it officially documented. - license_files = metadata.get_all('License-File') for path in parsed_record_content: if path.suffix == ".pyc": # we handle bytecode separately continue - if path.parent == distinfo: - if path.name in ("RECORD", "REQUESTED"): + if distinfo in path.parents: + if path.parent == distinfo and path.name in ("RECORD", "REQUESTED"): # RECORD and REQUESTED files are removed in %pyproject_install # See PEP 627 continue - if license_files and path.name in license_files: + if is_license_file(path, license_files, license_directories): paths["metadata"]["licenses"].append(path) else: paths["metadata"]["files"].append(path) + # nested directories within distinfo + index = path.parents.index(distinfo) + for parent in list(path.parents)[:index]: # no direct slice until Python 3.10 + if parent not in paths["metadata"]["dirs"]: + paths["metadata"]["dirs"].append(parent) continue for sitedir in sitedirs: @@ -480,6 +522,12 @@ def generate_file_list(paths_dict, module_globs, include_others=False): done_modules.add(name) done_globs.add(glob) + # Users using '*' don't care about the files in the package, so it's ok + # not to fail the build when no modules are detected + # There can be legitimate reasons to create a package without Python modules + if not modules and fnmatch.fnmatchcase("", glob): + done_globs.add(glob) + missed = module_globs - done_globs if missed: missed_text = ", ".join(sorted(missed)) @@ -488,6 +536,50 @@ def generate_file_list(paths_dict, module_globs, include_others=False): return sorted(files) +def generate_module_list(paths_dict, module_globs): + """ + This function takes the paths_dict created by the classify_paths() function and + reads the modules names from it. + It filters those whose top-level module names match any of the provided module_globs. + + Returns list with matching qualified module names. + + Examples: + + >>> generate_module_list({'module_names': {'foo', 'foo.bar', 'baz'}}, {'foo'}) + ['foo', 'foo.bar'] + + >>> generate_module_list({'module_names': {'foo', 'foo.bar', 'baz'}}, {'*foo'}) + ['foo', 'foo.bar'] + + >>> generate_module_list({'module_names': {'foo', 'foo.bar', 'baz'}}, {'foo', 'baz'}) + ['baz', 'foo', 'foo.bar'] + + >>> generate_module_list({'module_names': {'foo', 'foo.bar', 'baz'}}, {'*'}) + ['baz', 'foo', 'foo.bar'] + + >>> generate_module_list({'module_names': {'foo', 'foo.bar', 'baz'}}, {'bar'}) + [] + + Submodules aren't discovered: + + >>> generate_module_list({'module_names': {'foo', 'foo.bar', 'baz'}}, {'*bar'}) + [] + """ + + module_names = paths_dict['module_names'] + filtered_module_names = set() + + for glob in module_globs: + for name in module_names: + # Match the top-level part of the qualified name, eg. 'foo.bar.baz' -> 'foo' + top_level_name = name.split('.')[0] + if fnmatch.fnmatchcase(top_level_name, glob): + filtered_module_names.add(name) + + return sorted(filtered_module_names) + + def parse_varargs(varargs): """ Parse varargs from the %pyproject_save_files macro @@ -616,7 +708,7 @@ def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_versio parsed_records = load_parsed_record(pyproject_record) final_file_list = [] - all_module_names = set() + final_module_list = [] for record_path, files in parsed_records.items(): metadata = dist_metadata(buildroot, record_path) @@ -627,12 +719,11 @@ def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_versio final_file_list.extend( generate_file_list(paths_dict, globs, include_auto) ) - all_module_names.update(paths_dict["module_names"]) - - # Sort values, so they are always checked in the same order - all_module_names = sorted(all_module_names) + final_module_list.extend( + generate_module_list(paths_dict, globs) + ) - return final_file_list, all_module_names + return final_file_list, final_module_list def main(cli_args): @@ -651,17 +742,31 @@ def main(cli_args): def argparser(): - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + description="Create %{pyproject_files} for a Python project.", + prog="%pyproject_save_files", + add_help=False, + # custom usage to add +auto + usage="%(prog)s MODULE_GLOB [MODULE_GLOB ...] [+auto]", + ) + parser.add_argument( + '--help', action='help', + default=argparse.SUPPRESS, + help=argparse.SUPPRESS, + ) r = parser.add_argument_group("required arguments") - r.add_argument("--output-files", type=PosixPath, required=True) - r.add_argument("--output-modules", type=PosixPath, required=True) - r.add_argument("--buildroot", type=PosixPath, required=True) - r.add_argument("--sitelib", type=BuildrootPath, required=True) - r.add_argument("--sitearch", type=BuildrootPath, required=True) - r.add_argument("--python-version", type=str, required=True) - r.add_argument("--pyproject-record", type=PosixPath, required=True) - r.add_argument("--prefix", type=PosixPath, required=True) - parser.add_argument("varargs", nargs="+") + r.add_argument("--output-files", type=PosixPath, required=True, help=argparse.SUPPRESS) + r.add_argument("--output-modules", type=PosixPath, required=True, help=argparse.SUPPRESS) + r.add_argument("--buildroot", type=PosixPath, required=True, help=argparse.SUPPRESS) + r.add_argument("--sitelib", type=BuildrootPath, required=True, help=argparse.SUPPRESS) + r.add_argument("--sitearch", type=BuildrootPath, required=True, help=argparse.SUPPRESS) + r.add_argument("--python-version", type=str, required=True, help=argparse.SUPPRESS) + r.add_argument("--pyproject-record", type=PosixPath, required=True, help=argparse.SUPPRESS) + r.add_argument("--prefix", type=PosixPath, required=True, help=argparse.SUPPRESS) + parser.add_argument( + "varargs", nargs="+", metavar="MODULE_GLOB", + help="Shell-like glob matching top-level module names to save into %%{pyproject_files}", + ) return parser diff --git a/pyproject_save_files_test_data.yaml b/pyproject_save_files_test_data.yaml index eeed5d358baadd46b03c4296cb8ba4b1aa2d3e9d..e9eb427b961a658fd62298acbc6310253869b4c3 100644 --- a/pyproject_save_files_test_data.yaml +++ b/pyproject_save_files_test_data.yaml @@ -409,10 +409,27 @@ classified: other: files: - /usr/lib/python3.7/site-packages/zope.event-4.4-py3.7-nspkg.pth + comic2pdf: + metadata: + dirs: + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info + docs: [] + files: + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/METADATA + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/WHEEL + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/entry_points.txt + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/top_level.txt + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/zip-safe + licenses: [] + modules: [] + other: + files: + - /usr/bin/comic2pdf.py django: metadata: dirs: - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info + - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/licenses docs: [] files: - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/AUTHORS @@ -422,8 +439,8 @@ classified: - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/entry_points.txt - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/top_level.txt licenses: - - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE - - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE.python + - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/licenses/LICENSE + - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/licenses/LICENSE.python lang: django: af: @@ -7738,7 +7755,6 @@ dumped: - ipykernel.tests.utils - ipykernel.trio_runner - ipykernel.zmqshell - - ipykernel_launcher - - zope - zope - - '%dir /usr/lib/python3.7/site-packages/zope' @@ -7763,9 +7779,20 @@ dumped: - zope.event - zope.event.classhandler - zope.event.tests +- - comic2pdf + - '*' + - - '%dir /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info' + - /usr/bin/comic2pdf.py + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/METADATA + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/WHEEL + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/entry_points.txt + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/top_level.txt + - /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/zip-safe + - [] - - django - django - - '%dir /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info' + - '%dir /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/licenses' - '%dir /usr/lib/python3.7/site-packages/django' - '%dir /usr/lib/python3.7/site-packages/django/__pycache__' - '%dir /usr/lib/python3.7/site-packages/django/apps' @@ -11349,8 +11376,8 @@ dumped: - '%lang(zh) /usr/lib/python3.7/site-packages/django/contrib/sessions/locale/zh_Hant/LC_MESSAGES/django.mo' - '%lang(zh) /usr/lib/python3.7/site-packages/django/contrib/sites/locale/zh_Hans/LC_MESSAGES/django.mo' - '%lang(zh) /usr/lib/python3.7/site-packages/django/contrib/sites/locale/zh_Hant/LC_MESSAGES/django.mo' - - '%license /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE' - - '%license /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE.python' + - '%license /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/licenses/LICENSE' + - '%license /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/licenses/LICENSE.python' - /usr/bin/django-admin - /usr/bin/django-admin.py - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/AUTHORS @@ -15743,6 +15770,17 @@ records: zope/event/classhandler.py,sha256=CEx6issKWSia0Wruob_jIQI2EfYX45krokoTHyVsJFQ,1816 zope/event/tests.py,sha256=bvEzvOmPoQETMqYiqsR9EeVsC8Dzy-HOclfpQFVjDhE,1871 + comic2pdf: + path: /usr/lib/python3.7/site-packages/comic2pdf-3.1.0.dist-info/RECORD + content: | + ../../../bin/comic2pdf.py,sha256=ad0XbWxj2fzn_oYi1h-usY8jsxAvfpYA1aaify1Ym88,3266 + comic2pdf-3.1.0.dist-info/METADATA,sha256=qMVNbSPY02NdWfGex5yWNxoK1d96ereES-XoKxshVEA,3195 + comic2pdf-3.1.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92 + comic2pdf-3.1.0.dist-info/entry_points.txt,sha256=uORK0FJD-i46W74x2mNHfloSPS4QElN3-Y0vKQZ7svw,46 + comic2pdf-3.1.0.dist-info/top_level.txt,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1 + comic2pdf-3.1.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1 + comic2pdf-3.1.0.dist-info/RECORD,, + django: path: /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/RECORD content: | @@ -15751,8 +15789,8 @@ records: ../../../bin/django-admin.py,sha256=OOv0QKYqhDD2O4X3HQx3gFFQ-CC7hSLnWuzZnQXeiiA,115 Django-3.0.7.dist-info/AUTHORS,sha256=cV29hNQ1SpKhTmZuPff3LWHyQ7mHNBWP7_0JufEUHbs,36806 Django-3.0.7.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 - Django-3.0.7.dist-info/LICENSE,sha256=uEZBXRtRTpwd_xSiLeuQbXlLxUbKYSn5UKGM0JHipmk,1552 - Django-3.0.7.dist-info/LICENSE.python,sha256=Z-Pr3SuMPxOcaosqZSgr_NAjh2cFRcFyPZjP7nMeQrQ,13231 + Django-3.0.7.dist-info/licenses/LICENSE,sha256=uEZBXRtRTpwd_xSiLeuQbXlLxUbKYSn5UKGM0JHipmk,1552 + Django-3.0.7.dist-info/licenses/LICENSE.python,sha256=Z-Pr3SuMPxOcaosqZSgr_NAjh2cFRcFyPZjP7nMeQrQ,13231 Django-3.0.7.dist-info/METADATA,sha256=0ZU0N0E-CHKarXMLp4oOYf7EMUHR8eJ79f2yqw2NwoM,3574 Django-3.0.7.dist-info/RECORD,, Django-3.0.7.dist-info/WHEEL,sha256=g4nMs7d-Xl9-xC9XovUrsDHGXt-FT0E17Yqo92DEfvY,92 diff --git a/pyproject_wheel.py b/pyproject_wheel.py new file mode 100644 index 0000000000000000000000000000000000000000..1936d9c56e669ab571a82602cdf368e9c75f893e --- /dev/null +++ b/pyproject_wheel.py @@ -0,0 +1,25 @@ +import sys +import subprocess + + +def build_wheel(*, wheeldir, stdout=None): + command = ( + sys.executable, + '-m', 'pip', + 'wheel', + '--wheel-dir', wheeldir, + '--no-deps', + '--use-pep517', + '--no-build-isolation', + '--disable-pip-version-check', + '--no-clean', + '--progress-bar', 'off', + '--verbose', + '.', + ) + cp = subprocess.run(command, stdout=stdout) + return cp.returncode + + +if __name__ == '__main__': + sys.exit(build_wheel(wheeldir=sys.argv[1])) diff --git a/test_RECORD b/test_RECORD index 686d3e34bbe88c869af0d32368ac331bd1cd64ab..e917ce9e8d9110435fd1e40fade2521b9c1c01db 100644 --- a/test_RECORD +++ b/test_RECORD @@ -1,11 +1,11 @@ -../../../bin/__pycache__/tldr.cpython-37.pyc,, -../../../bin/tldr,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766 -../../../bin/tldr.py,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766 -__pycache__/tldr.cpython-37.pyc,, -tldr-0.5.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -tldr-0.5.dist-info/LICENSE,sha256=q7quAfjDWCYKC_WRk_uaP6d2wwVpOpVjUSkv8l6H7xI,1075 -tldr-0.5.dist-info/METADATA,sha256=AN5nYUVxo_zkVaMGKu34YDWWif84oA6uxKmTab213vM,3850 -tldr-0.5.dist-info/RECORD,, -tldr-0.5.dist-info/WHEEL,sha256=S8S5VL-stOTSZDYxHyf0KP7eds0J72qrK0Evu3TfyAY,92 -tldr-0.5.dist-info/top_level.txt,sha256=xHSI9WD6Y-_hONbi2b_9RIn9oiO7RBGHU3A8geJq3mI,5 -tldr.py,sha256=aJlA3tIz4QYYy8e7DZUhPyLCqTwnfFjA7Nubwm9bPe0,12779 +../../../bin/__pycache__/tldr.cpython-37.pyc,, +../../../bin/tldr,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766 +../../../bin/tldr.py,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766 +__pycache__/tldr.cpython-37.pyc,, +tldr-0.5.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +tldr-0.5.dist-info/LICENSE,sha256=q7quAfjDWCYKC_WRk_uaP6d2wwVpOpVjUSkv8l6H7xI,1075 +tldr-0.5.dist-info/METADATA,sha256=AN5nYUVxo_zkVaMGKu34YDWWif84oA6uxKmTab213vM,3850 +tldr-0.5.dist-info/RECORD,, +tldr-0.5.dist-info/WHEEL,sha256=S8S5VL-stOTSZDYxHyf0KP7eds0J72qrK0Evu3TfyAY,92 +tldr-0.5.dist-info/top_level.txt,sha256=xHSI9WD6Y-_hONbi2b_9RIn9oiO7RBGHU3A8geJq3mI,5 +tldr.py,sha256=aJlA3tIz4QYYy8e7DZUhPyLCqTwnfFjA7Nubwm9bPe0,12779 diff --git a/test_pyproject_buildrequires.py b/test_pyproject_buildrequires.py index 15075c5dbf9493b2d128e8958ed5a940b98cc412..74f3ae8837b079bf69920076d15414c1191011ff 100644 --- a/test_pyproject_buildrequires.py +++ b/test_pyproject_buildrequires.py @@ -13,12 +13,14 @@ with Path(__file__).parent.joinpath('pyproject_buildrequires_testcases.yaml').op @pytest.mark.parametrize('case_name', testcases) -def test_data(case_name, capsys, tmp_path, monkeypatch): +def test_data(case_name, capfd, tmp_path, monkeypatch): case = testcases[case_name] cwd = tmp_path.joinpath('cwd') cwd.mkdir() monkeypatch.chdir(cwd) + wheeldir = cwd.joinpath('wheeldir') + wheeldir.mkdir() if case.get('xfail'): pytest.xfail(case.get('xfail')) @@ -45,6 +47,8 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): generate_requires( get_installed_version=get_installed_version, include_runtime=case.get('include_runtime', use_build_system), + build_wheel=case.get('build_wheel', False), + wheeldir=str(wheeldir), extras=case.get('extras', []), toxenv=case.get('toxenv', None), generate_extras=case.get('generate_extras', False), @@ -64,10 +68,15 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): # if we ever need to do that, we can remove the check or change it: assert 'expected' in case or 'stderr_contains' in case - out, err = capsys.readouterr() + out, err = capfd.readouterr() if 'expected' in case: - assert out == case['expected'] + expected = case['expected'] + if isinstance(expected, list): + # at least one of them needs to match + assert any(out == e for e in expected) + else: + assert out == expected # stderr_contains may be a string or list of strings stderr_contains = case.get('stderr_contains') @@ -75,7 +84,7 @@ def test_data(case_name, capsys, tmp_path, monkeypatch): if isinstance(stderr_contains, str): stderr_contains = [stderr_contains] for expected_substring in stderr_contains: - assert expected_substring in err + assert expected_substring.format(**locals()) in err finally: for req in requirement_files: req.close()