From ce39680601fb1386507eb48ab05b6b15ec10201a Mon Sep 17 00:00:00 2001 From: Tim 'mithro' Ansell Date: Sun, 21 Mar 2021 10:11:43 -0700 Subject: [PATCH] Reworking Python packaging. * Move `setup.py` and `pyproject.toml` to root directory otherwise everything breaks. * Use `setuptool_scm` for version. * Build wheels from sdist files using pip. --- .github/workflows/ci.yml | 5 +- MANIFEST.in | 3 + api/CMakeLists.txt | 6 +- api/python/.gitignore | 3 + api/python/Makefile | 98 +++++++++++++++++++++++++++ api/python/pyproject.toml | 2 - api/python/requirements.txt | 7 ++ api/python/requirements_dev.txt | 6 -- api/python/setup.py | 32 --------- pyproject.toml | 2 + setup.py | 113 ++++++++++++++++++++++++++++++++ 11 files changed, 232 insertions(+), 45 deletions(-) create mode 100644 MANIFEST.in create mode 100644 api/python/Makefile delete mode 100644 api/python/pyproject.toml create mode 100644 api/python/requirements.txt delete mode 100644 api/python/requirements_dev.txt delete mode 100644 api/python/setup.py create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46755593..b0098705 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -551,9 +551,8 @@ jobs: run: | source .github/setenv.sh cd api/python - pip install -r requirements_dev.txt - python setup.py bdist_wheel - pip install dist/*.whl + pip install -v -r requirements.txt + find - name: api-shared64-python-test run: | source .github/setenv.sh diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..cdeb2746 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +# MANIFEST.in must be in root directory. +# See https://github.com/pypa/setuptools/issues/2615 +graft ext diff --git a/api/CMakeLists.txt b/api/CMakeLists.txt index fe74a52c..6d9e019b 100644 --- a/api/CMakeLists.txt +++ b/api/CMakeLists.txt @@ -7,6 +7,7 @@ c4_log("enabling API") cmake_policy(PUSH) cmake_policy(SET CMP0078 NEW) # https://cmake.org/cmake/help/v3.14/policy/CMP0078.html cmake_policy(SET CMP0086 NEW) # https://cmake.org/cmake/help/v3.14/policy/CMP0086.html +cmake_policy(SET CMP0094 NEW) # https://cmake.org/cmake/help/v3.14/policy/CMP0094.html find_package(SWIG REQUIRED) c4_log("found swig ${SWIG_VERSION}: ${SWIG_EXECUTABLE}") @@ -47,6 +48,7 @@ endif() if(RYML_BUILD_API_PYTHON) c4_log("enabling python3 API") + set(Python3_FIND_VIRTUALENV "FIRST") find_package(Python3 COMPONENTS Interpreter Development REQUIRED) c4_log("found python ${Python3_VERSION}: ${Python3_EXECUTABLE}") # @@ -106,7 +108,7 @@ if(RYML_BUILD_API_PYTHON) c4_set_folder_remote_project_targets("test" ryml-api-test-python3) function(add_python_test script) get_filename_component(script_name ${script} NAME_WE) - set(script ${pydir}/test/${script}) + set(script ${pydir}/ryml/tests/${script}) set(tn ryml-api-test-python3-${script_name}) set(cmd python ${script}) add_custom_target(${tn} @@ -123,7 +125,7 @@ if(RYML_BUILD_API_PYTHON) add_custom_target(ryml-api-bm-python3) add_dependencies(ryml-api-bm ryml-api-bm-python3) c4_set_folder_remote_project_targets("bm" ryml-api-bm-python3) - set(script ${pydir}/test/parse_bm.py) + set(script ${pydir}/ryml/tests/parse_bm.py) c4_add_benchmark_cmd(ryml-api-bm-python3-travis COMMAND python ${script} ${CMAKE_CURRENT_LIST_DIR}/../bm/cases/travis.yml ryml) c4_add_benchmark_cmd(ryml-api-bm-python3-appveyor diff --git a/api/python/.gitignore b/api/python/.gitignore index b02cd8a3..830e7e4d 100644 --- a/api/python/.gitignore +++ b/api/python/.gitignore @@ -136,3 +136,6 @@ dmypy.json # Version file generated by setuptools_scm version.py + +# SWIG produced file +ryml/ryml.py diff --git a/api/python/Makefile b/api/python/Makefile new file mode 100644 index 00000000..0adbb900 --- /dev/null +++ b/api/python/Makefile @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT + +# Use bash even on Windows +SHELL := /bin/bash + +# On Windows the activate script is stored in a different location. +ACTIVATE_SCRIPT := venv/bin/activate +ifeq ($(OS),Windows_NT) +ACTIVATE_SCRIPT := venv/Scripts/activate +endif + +ACTIVATE=[[ -e $(ACTIVATE_SCRIPT) ]] && source $(ACTIVATE_SCRIPT); + +clean: + rm -rf dist *.egg-info + rm -rf ../../build ../../.egg* + rm -rf ryml/*.so ryml/ryml.py ryml/include ryml/lib + +.PHONY: clean + +venv-clean: + rm -rf venv + +.PHONY: venv-clean + +$(ACTIVATE_SCRIPT): requirements.txt Makefile + make venv + @touch $(ACTIVATE_SCRIPT) + +venv: + virtualenv --python=python3 --always-copy venv + # Packaging tooling. + ${ACTIVATE} pip install -U pip twine build + # Setup requirements. + ${ACTIVATE} pip install -v -r requirements.txt + @${ACTIVATE} python -c "from ryml.version import version as v; print('Installed version:', v)" + +.PHONY: venv + +build-sdist: | $(ACTIVATE_SCRIPT) + ${ACTIVATE} (cd ../..; python -m build --sdist --outdir $(PWD)/dist) + +.PHONY: build-sdist + +build-wheel: | $(ACTIVATE_SCRIPT) + rm -rf dist + $(MAKE) build-sdist + @ls -l dist/*.tar.gz + ${ACTIVATE} pip wheel -v dist/*.tar.gz --wheel-dir $(PWD)/dist + +.PHONY: build-wheel + +build: + rm -rf build dist + $(MAKE) build-sdist + $(MAKE) build-wheel + +.PHONY: build + +# PYPI_TEST = --repository-url https://test.pypi.org/legacy/ +PYPI_TEST = --repository testpypi + +upload-test: | $(ACTIVATE_SCRIPT) + make clean + make build-sdist + ${ACTIVATE} twine upload ${PYPI_TEST} dist/* + +.PHONY: upload-test + +upload: | $(ACTIVATE_SCRIPT) + make clean + make build-sdist + ${ACTIVATE} twine upload --verbose dist/* + +.PHONY: upload + +check: | $(ACTIVATE_SCRIPT) + make clean + make build-wheel + ${ACTIVATE} twine check dist/*.whl + +.PHONY: check + +install: | $(ACTIVATE_SCRIPT) + ${ACTIVATE} python setup.py install + +.PHONY: install + +test: | $(ACTIVATE_SCRIPT) + ${ACTIVATE} pytest + +.PHONY: test + +version: | $(ACTIVATE_SCRIPT) + ${ACTIVATE} python setup.py --version + +.PHONY: version diff --git a/api/python/pyproject.toml b/api/python/pyproject.toml deleted file mode 100644 index dcb4030f..00000000 --- a/api/python/pyproject.toml +++ /dev/null @@ -1,2 +0,0 @@ -[build-system] -requires = ["setuptools", "wheel", "ninja", "cmake_build_extension"] diff --git a/api/python/requirements.txt b/api/python/requirements.txt new file mode 100644 index 00000000..955e7830 --- /dev/null +++ b/api/python/requirements.txt @@ -0,0 +1,7 @@ +ruamel.yaml +ninja +pyyaml +prettytable +git+https://github.com/litghost/cmake-build-extension.git@add_support_for_components#egg=cmake-build-extension +wheel +-e ../.. diff --git a/api/python/requirements_dev.txt b/api/python/requirements_dev.txt deleted file mode 100644 index 9e2462b3..00000000 --- a/api/python/requirements_dev.txt +++ /dev/null @@ -1,6 +0,0 @@ -ruamel.yaml -ninja -pyyaml -prettytable -git+git://github.com/litghost/cmake-build-extension.git@add_support_for_components#egg=cmake-build-extension -wheel diff --git a/api/python/setup.py b/api/python/setup.py deleted file mode 100644 index 983f3512..00000000 --- a/api/python/setup.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python -from pathlib import Path -from setuptools import setup -from cmake_build_extension import BuildExtension, CMakeExtension - -# define a CMake package -cmake_args = dict( - name='ryml.ryml', - install_prefix='', - source_dir=str(Path("../../").absolute()), - cmake_component='python', - cmake_configure_options=[ - "-DRYML_BUILD_API:BOOL=ON", - ]) - -try: - ext = CMakeExtension(**cmake_args) -except TypeError: - # FIXME: cmake_build_extension may not support cmake_component. - del cmake_args['cmake_component'] - ext = CMakeExtension(**cmake_args) - -setup(name='rapidyaml', - version='0.1.0', - packages=['ryml'], - ext_modules=[ext], - cmdclass=dict(build_ext=BuildExtension), - author='Joao Paulo Magalhaes', - url='https://github.com/biojppm/rapidyaml', - description='This is a test', - license='MIT', - ) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..c718b57c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools>=42", "setuptools_scm[toml]>=3.4", "wheel", "ninja", "cmake_build_extension"] diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..5dc26a35 --- /dev/null +++ b/setup.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT + +import os +import shutil +import sys + +from pathlib import Path +from distutils import log +from setuptools import setup +from setuptools.command.sdist import sdist as SdistCommand +from cmake_build_extension import BuildExtension, CMakeExtension + +TOP_DIR = (Path(__file__).parent).resolve() + +# Where the Python library is actually found. +PYTHON_DIR = "api/python" + +setup_kw = {} + +# Read in the package version when not in a git repository. +VERSION_FILE = os.path.join(PYTHON_DIR, 'ryml', 'version.py') +if not (TOP_DIR / '.git').exists() and os.path.exists(VERSION_FILE): + exec(open(VERSION_FILE).read()) + setup_kw['version'] = version +else: + setup_kw['use_scm_version']= { + "version_scheme": "post-release", + "local_scheme": "no-local-version", + "write_to": VERSION_FILE, + } + +# Read in the module description from the README.md file. +README_FILE = TOP_DIR / "README.md" +if README_FILE.exists(): + with open(TOP_DIR / "README.md", "r") as fh: + setup_kw['long_description'] = fh.read() + setup_kw['long_description_content_type'] = "text/markdown" + +# define a CMake package +cmake_args = dict( + name='ryml.ryml', + install_prefix='', + source_dir='', + cmake_component='python', + cmake_configure_options=[ + "-DRYML_BUILD_API:BOOL=ON", + # Force cmake to use the Python interpreter we are currently using to + # run setup.py + "-DPython3_EXECUTABLE:FILEPATH="+sys.executable, + ], +) + +try: + ext = CMakeExtension(**cmake_args) +except TypeError: + del cmake_args['cmake_component'] + ext = CMakeExtension(**cmake_args) + + # If the CMakeExtension doesn't support `cmake_component` then we have to + # do some manual cleanup. + _BuildExtension=BuildExtension + class BuildExtension(_BuildExtension): + def build_extension(self, ext): + _BuildExtension.build_extension(self, ext) + ext_dir = Path(self.get_ext_fullpath(ext.name)).parent.absolute() + cmake_install_prefix = ext_dir / ext.install_prefix + assert cmake_install_prefix.exists(), cmake_install_prefix + try: + lib_path = cmake_install_prefix / "lib" + assert lib_path.exists(), lib_path + log.info("Removing everything under: %s", lib_path) + shutil.rmtree(lib_path) + + inc_path = cmake_install_prefix / "include" + assert inc_path.exists(), inc_path + log.info("Removing everything under: %s", inc_path) + shutil.rmtree(inc_path) + + # Windows only + cm_path = cmake_install_prefix / "cmake" + if cm_path.exists(): + log.info("Removing everything under: %s", cm_path) + shutil.rmtree(cm_path) + except: + log.info('Found following installed files:') + for f in cmake_install_prefix.rglob("*"): + log.info(' - %s', f) + raise + +setup( + # Package human readable information + name='rapidyaml', + #author='Joao Paulo Magalhaes', + description='Rapid YAML - a library to parse and emit YAML, and do it fast.', + url='https://github.com/biojppm/rapidyaml', + license='MIT', + license_files=['LICENSE.txt'], + # Package contents control + cmdclass={ + "build_ext": BuildExtension, + }, + package_dir={"": PYTHON_DIR}, + packages=['ryml'], + ext_modules=[ext], + include_package_data=True, + # Requirements + python_requires=">=3.7", + setup_requires=['setuptools_scm'], + # Extra arguments + **setup_kw, +)