Reworking Python packaging.

* Move `setup.py` and `pyproject.toml` to root directory otherwise
   everything breaks.
 * Use `setuptool_scm` for version.
 * Build wheels from sdist files using pip.

.github/workflows/ci.yml

@@ -551,9 +551,8 @@ jobs:
         run: |
             source .github/setenv.sh
             cd api/python
-            pip install -r requirements_dev.txt
+            pip install -v -r requirements.txt
-            python setup.py bdist_wheel
+            find
-            pip install dist/*.whl
       - name: api-shared64-python-test
         run: |
             source .github/setenv.sh

MANIFEST.in

@@ -0,0 +1,3 @@
+# MANIFEST.in must be in root directory.
+# See https://github.com/pypa/setuptools/issues/2615
+graft ext

api/CMakeLists.txt

@@ -7,6 +7,7 @@ c4_log("enabling API")
 cmake_policy(PUSH)
 cmake_policy(SET CMP0078 NEW) # https://cmake.org/cmake/help/v3.14/policy/CMP0078.html
 cmake_policy(SET CMP0086 NEW) # https://cmake.org/cmake/help/v3.14/policy/CMP0086.html
+cmake_policy(SET CMP0094 NEW) # https://cmake.org/cmake/help/v3.14/policy/CMP0094.html
 
 find_package(SWIG REQUIRED)
 c4_log("found swig ${SWIG_VERSION}: ${SWIG_EXECUTABLE}")
@@ -47,6 +48,7 @@ endif()
 
 if(RYML_BUILD_API_PYTHON)
     c4_log("enabling python3 API")
+    set(Python3_FIND_VIRTUALENV "FIRST")
     find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
     c4_log("found python ${Python3_VERSION}: ${Python3_EXECUTABLE}")
     #
@@ -106,7 +108,7 @@ if(RYML_BUILD_API_PYTHON)
         c4_set_folder_remote_project_targets("test" ryml-api-test-python3)
         function(add_python_test script)
             get_filename_component(script_name ${script} NAME_WE)
-            set(script ${pydir}/test/${script})
+            set(script ${pydir}/ryml/tests/${script})
             set(tn ryml-api-test-python3-${script_name})
             set(cmd python ${script})
             add_custom_target(${tn}
@@ -123,7 +125,7 @@ if(RYML_BUILD_API_PYTHON)
         add_custom_target(ryml-api-bm-python3)
         add_dependencies(ryml-api-bm ryml-api-bm-python3)
         c4_set_folder_remote_project_targets("bm" ryml-api-bm-python3)
-        set(script ${pydir}/test/parse_bm.py)
+        set(script ${pydir}/ryml/tests/parse_bm.py)
         c4_add_benchmark_cmd(ryml-api-bm-python3-travis
             COMMAND python ${script} ${CMAKE_CURRENT_LIST_DIR}/../bm/cases/travis.yml ryml)
         c4_add_benchmark_cmd(ryml-api-bm-python3-appveyor

api/python/.gitignore

@@ -136,3 +136,6 @@ dmypy.json
 
 # Version file generated by setuptools_scm
 version.py
+
+# SWIG produced file
+ryml/ryml.py

api/python/Makefile

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MIT
+
+# Use bash even on Windows
+SHELL := /bin/bash
+
+# On Windows the activate script is stored in a different location.
+ACTIVATE_SCRIPT := venv/bin/activate
+ifeq ($(OS),Windows_NT)
+ACTIVATE_SCRIPT := venv/Scripts/activate
+endif
+
+ACTIVATE=[[ -e $(ACTIVATE_SCRIPT) ]] && source $(ACTIVATE_SCRIPT);
+
+clean:
+	rm -rf dist *.egg-info
+	rm -rf ../../build ../../.egg*
+	rm -rf ryml/*.so ryml/ryml.py ryml/include ryml/lib
+
+.PHONY: clean
+
+venv-clean:
+	rm -rf venv
+
+.PHONY: venv-clean
+
+$(ACTIVATE_SCRIPT): requirements.txt Makefile
+	make venv
+	@touch $(ACTIVATE_SCRIPT)
+
+venv:
+	virtualenv --python=python3 --always-copy venv
+	# Packaging tooling.
+	${ACTIVATE} pip install -U pip twine build
+	# Setup requirements.
+	${ACTIVATE} pip install -v -r requirements.txt
+	@${ACTIVATE} python -c "from ryml.version import version as v; print('Installed version:', v)"
+
+.PHONY: venv
+
+build-sdist: | $(ACTIVATE_SCRIPT)
+	${ACTIVATE} (cd ../..; python -m build --sdist --outdir $(PWD)/dist)
+
+.PHONY: build-sdist
+
+build-wheel: | $(ACTIVATE_SCRIPT)
+	rm -rf dist
+	$(MAKE) build-sdist
+	@ls -l dist/*.tar.gz
+	${ACTIVATE} pip wheel -v dist/*.tar.gz --wheel-dir $(PWD)/dist
+
+.PHONY: build-wheel
+
+build:
+	rm -rf build dist
+	$(MAKE) build-sdist
+	$(MAKE) build-wheel
+
+.PHONY: build
+
+# PYPI_TEST = --repository-url https://test.pypi.org/legacy/
+PYPI_TEST = --repository testpypi
+
+upload-test: | $(ACTIVATE_SCRIPT)
+	make clean
+	make build-sdist
+	${ACTIVATE} twine upload ${PYPI_TEST} dist/*
+
+.PHONY: upload-test
+
+upload: | $(ACTIVATE_SCRIPT)
+	make clean
+	make build-sdist
+	${ACTIVATE} twine upload --verbose dist/*
+
+.PHONY: upload
+
+check: | $(ACTIVATE_SCRIPT)
+	make clean
+	make build-wheel
+	${ACTIVATE} twine check dist/*.whl
+
+.PHONY: check
+
+install: | $(ACTIVATE_SCRIPT)
+	${ACTIVATE} python setup.py install
+
+.PHONY: install
+
+test: | $(ACTIVATE_SCRIPT)
+	${ACTIVATE} pytest
+
+.PHONY: test
+
+version: | $(ACTIVATE_SCRIPT)
+	${ACTIVATE} python setup.py --version
+
+.PHONY: version

api/python/pyproject.toml

@@ -1,2 +0,0 @@
-[build-system]
-requires = ["setuptools", "wheel", "ninja", "cmake_build_extension"]

api/python/requirements.txt

@@ -0,0 +1,7 @@
+ruamel.yaml
+ninja
+pyyaml
+prettytable
+git+https://github.com/litghost/cmake-build-extension.git@add_support_for_components#egg=cmake-build-extension
+wheel
+-e ../..

api/python/requirements_dev.txt

@@ -1,6 +0,0 @@
-ruamel.yaml
-ninja
-pyyaml
-prettytable
-git+git://github.com/litghost/cmake-build-extension.git@add_support_for_components#egg=cmake-build-extension
-wheel

api/python/setup.py

@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-from pathlib import Path
-from setuptools import setup
-from cmake_build_extension import BuildExtension, CMakeExtension
-
-# define a CMake package
-cmake_args = dict(
-    name='ryml.ryml',
-    install_prefix='',
-    source_dir=str(Path("../../").absolute()),
-    cmake_component='python',
-    cmake_configure_options=[
-        "-DRYML_BUILD_API:BOOL=ON",
-        ])
-
-try:
-    ext = CMakeExtension(**cmake_args)
-except TypeError:
-    # FIXME: cmake_build_extension may not support cmake_component.
-    del cmake_args['cmake_component']
-    ext = CMakeExtension(**cmake_args)
-
-setup(name='rapidyaml',
-    version='0.1.0',
-    packages=['ryml'],
-    ext_modules=[ext],
-    cmdclass=dict(build_ext=BuildExtension),
-    author='Joao Paulo Magalhaes',
-    url='https://github.com/biojppm/rapidyaml',
-    description='This is a test',
-    license='MIT',
-    )

pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools>=42", "setuptools_scm[toml]>=3.4", "wheel", "ninja", "cmake_build_extension"]

setup.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MIT
+
+import os
+import shutil
+import sys
+
+from pathlib import Path
+from distutils import log
+from setuptools import setup
+from setuptools.command.sdist import sdist as SdistCommand
+from cmake_build_extension import BuildExtension, CMakeExtension
+
+TOP_DIR = (Path(__file__).parent).resolve()
+
+# Where the Python library is actually found.
+PYTHON_DIR = "api/python"
+
+setup_kw = {}
+
+# Read in the package version when not in a git repository.
+VERSION_FILE = os.path.join(PYTHON_DIR, 'ryml', 'version.py')
+if not (TOP_DIR / '.git').exists() and os.path.exists(VERSION_FILE):
+    exec(open(VERSION_FILE).read())
+    setup_kw['version'] = version
+else:
+    setup_kw['use_scm_version']= {
+        "version_scheme": "post-release",
+        "local_scheme": "no-local-version",
+        "write_to": VERSION_FILE,
+    }
+
+# Read in the module description from the README.md file.
+README_FILE = TOP_DIR / "README.md"
+if README_FILE.exists():
+    with open(TOP_DIR / "README.md", "r") as fh:
+        setup_kw['long_description'] = fh.read()
+        setup_kw['long_description_content_type'] = "text/markdown"
+
+# define a CMake package
+cmake_args = dict(
+    name='ryml.ryml',
+    install_prefix='',
+    source_dir='',
+    cmake_component='python',
+    cmake_configure_options=[
+        "-DRYML_BUILD_API:BOOL=ON",
+        # Force cmake to use the Python interpreter we are currently using to
+        # run setup.py
+        "-DPython3_EXECUTABLE:FILEPATH="+sys.executable,
+    ],
+)
+
+try:
+    ext = CMakeExtension(**cmake_args)
+except TypeError:
+    del cmake_args['cmake_component']
+    ext = CMakeExtension(**cmake_args)
+
+    # If the CMakeExtension doesn't support `cmake_component` then we have to
+    # do some manual cleanup.
+    _BuildExtension=BuildExtension
+    class BuildExtension(_BuildExtension):
+        def build_extension(self, ext):
+            _BuildExtension.build_extension(self, ext)
+            ext_dir = Path(self.get_ext_fullpath(ext.name)).parent.absolute()
+            cmake_install_prefix = ext_dir / ext.install_prefix
+            assert cmake_install_prefix.exists(), cmake_install_prefix
+            try:
+                lib_path = cmake_install_prefix / "lib"
+                assert lib_path.exists(), lib_path
+                log.info("Removing everything under: %s", lib_path)
+                shutil.rmtree(lib_path)
+
+                inc_path = cmake_install_prefix / "include"
+                assert inc_path.exists(), inc_path
+                log.info("Removing everything under: %s", inc_path)
+                shutil.rmtree(inc_path)
+
+                # Windows only
+                cm_path = cmake_install_prefix / "cmake"
+                if cm_path.exists():
+                    log.info("Removing everything under: %s", cm_path)
+                    shutil.rmtree(cm_path)
+            except:
+                log.info('Found following installed files:')
+                for f in cmake_install_prefix.rglob("*"):
+                    log.info(' - %s', f)
+                raise
+
+setup(
+    # Package human readable information
+    name='rapidyaml',
+    #author='Joao Paulo Magalhaes',
+    description='Rapid YAML - a library to parse and emit YAML, and do it fast.',
+    url='https://github.com/biojppm/rapidyaml',
+    license='MIT',
+    license_files=['LICENSE.txt'],
+    # Package contents control
+    cmdclass={
+        "build_ext": BuildExtension,
+    },
+    package_dir={"": PYTHON_DIR},
+    packages=['ryml'],
+    ext_modules=[ext],
+    include_package_data=True,
+    # Requirements
+    python_requires=">=3.7",
+    setup_requires=['setuptools_scm'],
+    # Extra arguments
+    **setup_kw,
+)