1
0
Fork 0
mirror of https://github.com/ultrajson/ultrajson.git synced 2024-03-29 07:09:56 +01:00

Use cibuildwheel to build wheels.

This will enable the building of macOS ARM64 compatible wheels, fixing #456 (and
also lets us delete lots of code!!!).
This commit is contained in:
bwoodsend 2021-11-28 10:29:43 +00:00
parent 6833495c6f
commit 3ac5b9dd45
No known key found for this signature in database
GPG Key ID: 52B5C9300BEF567E
5 changed files with 101 additions and 221 deletions

View File

@ -1,89 +0,0 @@
name: Deploy Linux wheels
on:
push:
branches:
- main
pull_request:
paths:
- ".github/workflows/deploy-wheels-linux.yml"
- "scripts/build-manylinux-wheels.sh"
release:
types:
- published
jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
wheel: [
"manylinux2014_aarch64",
"manylinux2014_i686",
"manylinux2014_x86_64",
"musllinux_1_1_aarch64",
"musllinux_1_1_x86_64",
]
python-version: ["pypy3", "3.6", "3.7", "3.8", "3.9", "3.10"]
include:
# Add version-tag variable to existing jobs
- { python-version: "pypy3", version-tag: "pp37-pypy37_pp73" }
- { python-version: "3.6", version-tag: "cp36-cp36m" }
- { python-version: "3.7", version-tag: "cp37-cp37m" }
- { python-version: "3.8", version-tag: "cp38-cp38" }
- { python-version: "3.9", version-tag: "cp39-cp39" }
- { python-version: "3.10", version-tag: "cp310-cp310" }
exclude:
# No PyPy3 on musllinux
- { python-version: "pypy3", wheel: "musllinux_1_1_aarch64" }
- { python-version: "pypy3", wheel: "musllinux_1_1_x86_64" }
steps:
- uses: actions/checkout@v2
- run: |
git fetch --prune --unshallow
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade -q pip pytest wheel setuptools twine
# https://github.com/docker/setup-qemu-action
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Build ${{ matrix.wheel }} wheels
run: |
docker run -v `pwd`:/io quay.io/pypa/${{ matrix.wheel }} /io/scripts/build-manylinux-wheels.sh ${{ matrix.version-tag }}
- name: Upload as build artifacts
uses: actions/upload-artifact@v2
with:
name: wheels
path: dist/*.whl
- name: Publish package to PyPI
if: github.event.action == 'published'
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.pypi_password }}
run: twine upload --skip-existing dist/*.whl
- name: Publish package to TestPyPI
if: |
github.repository == 'ultrajson/ultrajson' &&
github.ref == 'refs/heads/main'
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.test_pypi_password }}
run: |
twine upload --repository-url https://test.pypi.org/legacy/ --skip-existing dist/*.whl

View File

@ -1,4 +1,4 @@
name: Deploy Windows/macOS wheels
name: Deploy Windows/macOS/native Linux wheels
on:
push:
@ -6,7 +6,7 @@ on:
- main
pull_request:
paths:
- ".github/workflows/deploy-wheels-windows-macos.yml"
- ".github/workflows/deploy-wheels-native.yml"
release:
types:
- published
@ -17,34 +17,29 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [windows-latest, macos-latest]
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
architecture: [x64, x86]
exclude:
- {os: macos-latest, architecture: x86}
os: [windows-latest, macOS-latest, ubuntu-latest]
steps:
- uses: actions/checkout@v2
- run: |
git fetch --prune --unshallow
- run: git fetch --prune --unshallow
- uses: actions/setup-python@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
architecture: ${{ matrix.architecture }}
- name: Install cibuildwheel
run: python -m pip install cibuildwheel==2.3.0
- name: Install dependencies
run: |
python -m pip install --upgrade -q pip pytest wheel setuptools twine
- name: Test with pytest
run: |
python -m pip install .
pytest
- name: Build wheel
run: python setup.py -q bdist_wheel
- name: Build wheels
run: python -m cibuildwheel --output-dir dist
# Options are supplied via environment variables:
env:
# Build separate wheels for macOS's different architectures.
CIBW_ARCHS_MACOS: "x86_64 arm64"
# Build only on Linux architectures that don't need qemu emulation.
CIBW_ARCHS_LINUX: "x86_64 i686"
# Don't build with prerelease Python versions.
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6,<3.11"
# Run the test suite after each build.
CIBW_TEST_REQUIRES: "pytest"
CIBW_TEST_COMMAND: pytest {package}/tests
- name: Upload as build artifacts
uses: actions/upload-artifact@v2

View File

@ -0,0 +1,81 @@
name: Deploy QEMU emulated Linux wheels
on:
push:
branches:
- main
pull_request:
paths:
- ".github/workflows/deploy-wheels-linux.yml"
- ".github/workflows/deploy-wheels-qemu.yml"
release:
types:
- published
jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
architecture: ["aarch64"]
python-version:
- pp37
- pp38
- cp36
- cp37
- cp38
- cp39
- cp310
steps:
- uses: actions/checkout@v2
- run: git fetch --prune --unshallow
- uses: actions/setup-python@v2
- name: Install cibuildwheel
run: python -m pip install cibuildwheel==2.3.0
# https://github.com/docker/setup-qemu-action
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Build wheels
run: python -m cibuildwheel --output-dir dist
# Options are supplied via environment variables:
env:
# Build only the currently selected Linux architecture (so we can
# parallelise for speed).
CIBW_ARCHS_LINUX: "${{ matrix.architecture }}"
# Likewise, select only one Python version per job to speed this up.
CIBW_BUILD: "${{ matrix.python-version }}-*"
# Run the test suite after each build.
CIBW_TEST_REQUIRES: "pytest"
CIBW_TEST_COMMAND: pytest {package}/tests
- name: Upload as build artifacts
uses: actions/upload-artifact@v2
with:
name: wheels
path: dist/*.whl
- name: Publish package to PyPI
if: github.event.action == 'published'
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.pypi_password }}
run: twine upload --skip-existing dist/*.whl
- name: Publish package to TestPyPI
if: |
github.repository == 'ultrajson/ultrajson' &&
github.ref == 'refs/heads/main'
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.test_pypi_password }}
run: |
twine upload --repository-url https://test.pypi.org/legacy/ --skip-existing dist/*.whl

View File

@ -1,55 +0,0 @@
#!/usr/bin/env bash
set -e -x
# This is to be run by Docker inside a Docker image.
# You can test it locally on a Linux machine by installing Docker and running from this
# repo's root:
# $ docker run -v `pwd`:/io quay.io/pypa/manylinux2014_x86_64 /io/scripts/build-manylinux-wheels.sh [cp310-cp310]
# The -v gives a directory alias for passing files in and out of the Docker.
# (/io is arbitrary). E.g the setup.py script can be accessed in the Docker via
# /io/setup.py quay.io/pypa/manylinux2014_x86_64 is the full Docker image name. Docker
# downloads it automatically.
# The last argument is a shell command that the Docker will execute. Filenames must be
# from the Docker's perspective.
# Wheels are initially generated as you would usually, but put in a temp directory temp-wheels.
# The pip-cache is optional but can speed up local builds having a real permanent pip-cache dir.
mkdir -p /io/pip-cache
mkdir -p /io/temp-wheels
# Clean out any old existing wheels.
find /io/temp-wheels/ -type f -delete
# Log the Python versions in the image.
ls /opt/python/
build_wheel () {
PYBIN=$1
"${PYBIN}/pip" install -q -U setuptools wheel pytest --cache-dir /io/pip-cache
(cd /io/ && "${PYBIN}/python" -m pip install .)
(cd /io/ && "${PYBIN}/python" -m pytest)
(cd /io/ && "${PYBIN}/python" setup.py -q bdist_wheel -d /io/temp-wheels)
}
if [ -n "$1" ]; then
# Build a single wheel
PYBIN=/opt/python/$1/bin
build_wheel ${PYBIN}
else
# Build many wheels
for PYBIN in /opt/python/cp3{6..10}*/bin; do
build_wheel ${PYBIN}
done
fi
"$PYBIN/pip" install -q auditwheel
# Wheels aren't considered manylinux unless they have been through
# auditwheel. Audited wheels go in /io/dist/.
mkdir -p /io/dist/
for whl in /io/temp-wheels/*.whl; do
auditwheel repair "$whl" -w /io/dist/
done

View File

@ -1,52 +0,0 @@
#!/usr/bin/env python3
import argparse
import os.path
import re
import subprocess
VERSION = re.compile(r":: Python :: (\d\.\d)$")
EXES = []
with open("setup.py") as f:
for line in f:
match = VERSION.search(line)
if match:
major_s, minor_s = match[1].split(".")
major, minor = int(major_s), int(minor_s)
if (major, minor) < (3, 8):
EXES.append(f"cp{major}{minor}-cp{major}{minor}m")
else:
EXES.append(f"cp{major}{minor}-cp{major}{minor}")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("version")
args = parser.parse_args()
pkg = f"ujson=={args.version}"
os.makedirs("dist", exist_ok=True)
for exe in EXES:
pip = f"/opt/python/{exe}/bin/pip"
if subprocess.call(
(
# fmt: off
"docker", "run", "--rm",
# so files are not root-owned
"--user", f"{os.getuid()}:{os.getgid()}",
"--volume", f'{os.path.abspath("dist")}:/dist:rw',
"quay.io/pypa/manylinux2014_x86_64:latest",
"bash", "-euxc",
f"{pip} wheel -w /tmp/wheels --no-deps {pkg} && "
f"auditwheel repair -w /dist /tmp/wheels/*.whl",
# fmt: on
)
):
return 1
return 0
if __name__ == "__main__":
exit(main())