From aba9bf73dff43333e2981587c3dedaaea4cf5423 Mon Sep 17 00:00:00 2001 From: Chang She <759245+changhiskhan@users.noreply.github.com> Date: Fri, 29 Jul 2022 11:24:05 -0700 Subject: [PATCH 1/3] Build linux wheels for typed_python Using manylinux we can build wheels across different python versions. This makes it easier to create binary distributions of typed_python so that it doesn't always need to be built from the source distro when installing. --- .gitignore | 7 ++++++ build_scripts/README.md | 24 ++++++++++++++++++++ build_scripts/create_manylinux_wheels.sh | 29 ++++++++++++++++++++++++ build_scripts/run_build.sh | 19 ++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 build_scripts/README.md create mode 100755 build_scripts/create_manylinux_wheels.sh create mode 100755 build_scripts/run_build.sh diff --git a/.gitignore b/.gitignore index a1ee77226..2dc97bf80 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,13 @@ object_database/web/content/dist /.env* /.test_venv /dist/ +/wheels/ /pip-wheel-metadata/ /testcert.cert /testcert.key + +.idea/ + +# build_scripts sets up temporary symlinks so don't accidentally check them in +Pipfile +Pipfile.lock \ No newline at end of file diff --git a/build_scripts/README.md b/build_scripts/README.md new file mode 100644 index 000000000..a100d3dff --- /dev/null +++ b/build_scripts/README.md @@ -0,0 +1,24 @@ +# Wheel builds for typed_python + +Using [Manylinux](https://github.com/pypa/manylinux) we can make typed_python more portable. + +## Building wheels + +1. Prerequisites: docker +2. Execute the `run_build.sh` script + +This process should create a `./wheels` directory under the typed_python repo root with +4 wheels for python 3.7-3.10 built for linux x86_64. + +To build arm64 wheels, just do `run_build.sh aarch64`. + +## Release to pypi + +1. pip install twine +2. twine upload wheels/*.whl + +## Notes + +Here we're using the CentOS 7 based manylinux2014 image. +We cannot move to the newer `manylinux_x_y` image ([PEP 600](https://peps.python.org/pep-0600/)) +until type_python drops support for python 3.7 diff --git a/build_scripts/create_manylinux_wheels.sh b/build_scripts/create_manylinux_wheels.sh new file mode 100755 index 000000000..ea1c23e98 --- /dev/null +++ b/build_scripts/create_manylinux_wheels.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Run this inside a manylinux docker container + +# Remove build artifacts +rm -rf build dist wheels + +# code dir +TP=${TYPED_PYTHON_DIR:-/opt/apriori/typed_python} + +FLAVOR="cp" # not pypy +ORIG_PATH=$PATH +for VERSION in 7 8 9 10; do + # Manylinux image has supported python installed like /opt/python/cp38-cp38 + PYNAME="${FLAVOR}"3"${VERSION}" + if [[ ${VERSION} == 7 ]]; then SUFFIX="m"; else SUFFIX=""; fi + PYHOME=/opt/python/${PYNAME}-${PYNAME}${SUFFIX} + PATH=${PYHOME}/bin:${ORIG_PATH} + pip install pipenv + PIPFILE="Pipfile_3_${VERSION}" + ln -sf ${PIPFILE} Pipfile + ln -sf "${PIPFILE}.lock" Pipfile.lock + pipenv sync --python $(which python) && pipenv run python setup.py bdist_wheel + rm Pipfile Pipfile.lock +done + +for whl in dist/*.whl; do + auditwheel repair "$whl" -w wheels +done diff --git a/build_scripts/run_build.sh b/build_scripts/run_build.sh new file mode 100755 index 000000000..0304f5aaf --- /dev/null +++ b/build_scripts/run_build.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -ex + +REPO_ROOT=$(realpath $(dirname $(realpath $0))/..) +ARCH=${1:-"x86_64"} +MANYLINUX=${2:-"manylinux2014"} # PEP 599 +BASE_IMAGE="quay.io/pypa/${MANYLINUX}_${ARCH}" + +if [[ $ARCH = "aarch64" ]]; then + DOCKER_PLATFORM="--platform linux/arm64" +fi +docker run \ + --rm \ + ${DOCKER_PLATFORM} \ + --volume ${REPO_ROOT}:/opt/apriori/typed_python \ + --workdir /opt/apriori/typed_python \ + ${BASE_IMAGE} \ + /opt/apriori/typed_python/build_scripts/create_manylinux_wheels.sh From 2675eafae805ce7ff70c180beacd42e9437587eb Mon Sep 17 00:00:00 2001 From: Chang She <759245+changhiskhan@users.noreply.github.com> Date: Fri, 29 Jul 2022 12:06:10 -0700 Subject: [PATCH 2/3] Update README.md --- build_scripts/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/build_scripts/README.md b/build_scripts/README.md index a100d3dff..224fcff0b 100644 --- a/build_scripts/README.md +++ b/build_scripts/README.md @@ -10,8 +10,6 @@ Using [Manylinux](https://github.com/pypa/manylinux) we can make typed_python mo This process should create a `./wheels` directory under the typed_python repo root with 4 wheels for python 3.7-3.10 built for linux x86_64. -To build arm64 wheels, just do `run_build.sh aarch64`. - ## Release to pypi 1. pip install twine From 26c6d2f85a4768c31796579739ee2db001a52e6b Mon Sep 17 00:00:00 2001 From: Chang She <759245+changhiskhan@users.noreply.github.com> Date: Sat, 30 Jul 2022 09:10:08 -0700 Subject: [PATCH 3/3] use build deps from pyproject.toml --- .gitignore | 1 + build_scripts/README.md | 12 ++++++--- build_scripts/create_manylinux_wheel.sh | 34 ++++++++++++++++++++++++ build_scripts/create_manylinux_wheels.sh | 29 -------------------- build_scripts/run_build.sh | 28 ++++++++++++------- 5 files changed, 61 insertions(+), 43 deletions(-) create mode 100755 build_scripts/create_manylinux_wheel.sh delete mode 100755 build_scripts/create_manylinux_wheels.sh diff --git a/.gitignore b/.gitignore index 2dc97bf80..1818f3234 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ object_database/web/content/dist # Emacs \#* \.#* +**/*~ # leading slash means only match at repo-root level /.python-version diff --git a/build_scripts/README.md b/build_scripts/README.md index 224fcff0b..10a668c68 100644 --- a/build_scripts/README.md +++ b/build_scripts/README.md @@ -1,14 +1,16 @@ # Wheel builds for typed_python -Using [Manylinux](https://github.com/pypa/manylinux) we can make typed_python more portable. +Using [Manylinux](https://github.com/pypa/manylinux) we can make typed_python +more portable. ## Building wheels 1. Prerequisites: docker -2. Execute the `run_build.sh` script +2. Run `run_build.sh ` (37 38 39 310 are supported) -This process should create a `./wheels` directory under the typed_python repo root with -4 wheels for python 3.7-3.10 built for linux x86_64. +This process should create a `./wheels` directory under the typed_python repo +root with wheels for python 3.7-3.10 built for linux x86_64 that are uploadable +to pypi ## Release to pypi @@ -20,3 +22,5 @@ This process should create a `./wheels` directory under the typed_python repo ro Here we're using the CentOS 7 based manylinux2014 image. We cannot move to the newer `manylinux_x_y` image ([PEP 600](https://peps.python.org/pep-0600/)) until type_python drops support for python 3.7 + +`typed_python` code does not build on arm64 though manylinux does support it. \ No newline at end of file diff --git a/build_scripts/create_manylinux_wheel.sh b/build_scripts/create_manylinux_wheel.sh new file mode 100755 index 000000000..a75b778f8 --- /dev/null +++ b/build_scripts/create_manylinux_wheel.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Run this inside a manylinux docker container + +# code dir +pushd ${TYPED_PYTHON_DIR:-/opt/apriori/typed_python} + +FLAVOR="cp" # cpython not pypy +ORIG_PATH=$PATH +VERSION=$1 + +VALID_VERSIONS=("37 38 39 310") +if [[ ! "${VALID_VERSIONS[*]}" =~ "$VERSION" ]]; then + echo "Valid versions are 37, 38, 39, 310" + exit 1 +fi + +# Manylinux image has supported python installed like /opt/python/cp38-cp38 +PYNAME="${FLAVOR}${VERSION}" + +# 37 in maintenance so has special suffix +if [[ ${VERSION} == 37 ]]; then SUFFIX="m"; else SUFFIX=""; fi + +# select the right python +PYTHON=/opt/python/${PYNAME}-${PYNAME}${SUFFIX}/bin/python + +# `python -m build` respects build deps in pyproject.toml +# `-w` option builds the wheel +${PYTHON} -m build -w + +# make wheels that pypi recognizes +for whl in dist/*${PYNAME}*.whl; do + auditwheel repair "$whl" -w wheels +done diff --git a/build_scripts/create_manylinux_wheels.sh b/build_scripts/create_manylinux_wheels.sh deleted file mode 100755 index ea1c23e98..000000000 --- a/build_scripts/create_manylinux_wheels.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Run this inside a manylinux docker container - -# Remove build artifacts -rm -rf build dist wheels - -# code dir -TP=${TYPED_PYTHON_DIR:-/opt/apriori/typed_python} - -FLAVOR="cp" # not pypy -ORIG_PATH=$PATH -for VERSION in 7 8 9 10; do - # Manylinux image has supported python installed like /opt/python/cp38-cp38 - PYNAME="${FLAVOR}"3"${VERSION}" - if [[ ${VERSION} == 7 ]]; then SUFFIX="m"; else SUFFIX=""; fi - PYHOME=/opt/python/${PYNAME}-${PYNAME}${SUFFIX} - PATH=${PYHOME}/bin:${ORIG_PATH} - pip install pipenv - PIPFILE="Pipfile_3_${VERSION}" - ln -sf ${PIPFILE} Pipfile - ln -sf "${PIPFILE}.lock" Pipfile.lock - pipenv sync --python $(which python) && pipenv run python setup.py bdist_wheel - rm Pipfile Pipfile.lock -done - -for whl in dist/*.whl; do - auditwheel repair "$whl" -w wheels -done diff --git a/build_scripts/run_build.sh b/build_scripts/run_build.sh index 0304f5aaf..4216614e3 100755 --- a/build_scripts/run_build.sh +++ b/build_scripts/run_build.sh @@ -2,18 +2,26 @@ set -ex +if [[ $# -eq 0 ]]; then + echo "Usage: ./run_build.sh " + echo "Valid 's are 37 38 39 310" + exit 1 +fi + +VERSION=$1 + REPO_ROOT=$(realpath $(dirname $(realpath $0))/..) -ARCH=${1:-"x86_64"} -MANYLINUX=${2:-"manylinux2014"} # PEP 599 +ARCH=${2:-"x86_64"} +MANYLINUX=${3:-"manylinux2014"} # PEP 599 BASE_IMAGE="quay.io/pypa/${MANYLINUX}_${ARCH}" -if [[ $ARCH = "aarch64" ]]; then - DOCKER_PLATFORM="--platform linux/arm64" -fi -docker run \ - --rm \ - ${DOCKER_PLATFORM} \ +# Unfortunately the code is specific to x86_64 arch +#if [[ $ARCH = "aarch64" ]]; then +# +# DOCKER_PLATFORM="--platform linux/arm64" +#fi + +docker run --rm ${DOCKER_PLATFORM} \ --volume ${REPO_ROOT}:/opt/apriori/typed_python \ - --workdir /opt/apriori/typed_python \ ${BASE_IMAGE} \ - /opt/apriori/typed_python/build_scripts/create_manylinux_wheels.sh + /opt/apriori/typed_python/build_scripts/create_manylinux_wheel.sh $VERSION