From b18cb3b5666ff4df76e617d87e2b98074a272baa Mon Sep 17 00:00:00 2001 From: Dobatymo Date: Thu, 25 May 2023 00:30:00 +0800 Subject: [PATCH 1/5] build wheels on github actions only building 64 bit on linux since nasm causes an error currently ``` [ 24%] Building ASM_NASM object simd/CMakeFiles/simd.dir/i386/jchuff-sse2.asm.o /project/libjpeg-turbo/simd/i386/jchuff-sse2.asm:472: error: invalid operand type make[2]: *** [simd/CMakeFiles/simd.dir/i386/jchuff-sse2.asm.o] Error 1 make[1]: *** [simd/CMakeFiles/simd.dir/all] Error 2 ``` --- .github/workflows/pythonpackage.yml | 81 +++++++++++++++++++++++++++++ .gitignore | 4 ++ .gitmodules | 4 ++ get-nasm.py | 28 ++++++++++ jpegtran/jpegtran_build.py | 8 ++- libjpeg-turbo | 1 + linux-build.sh | 10 ++++ pyproject.toml | 3 ++ setup.py | 9 ++-- 9 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/pythonpackage.yml create mode 100644 .gitmodules create mode 100644 get-nasm.py create mode 160000 libjpeg-turbo create mode 100644 linux-build.sh create mode 100644 pyproject.toml diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 0000000..3703d03 --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,81 @@ +name: Python package + +on: + push: + release: + types: + - published + +jobs: + build_wheels_linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: 'true' + - name: Build wheels + uses: pypa/cibuildwheel@v2.12.3 + env: + CIBW_BEFORE_ALL: "sh linux-build.sh" + CIBW_BEFORE_BUILD: echo {project}/binaries + CIBW_SKIP: "cp36-* pp*" + CIBW_ARCHS_LINUX: auto64 + CIBW_REPAIR_WHEEL_COMMAND: "LD_LIBRARY_PATH=/project/binaries auditwheel repair -w {dest_dir} {wheel}" + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + build_wheels_windows: + runs-on: windows-latest + strategy: + matrix: + arch: + - AMD64 + - x86 + steps: + - uses: actions/checkout@v3 + with: + submodules: 'true' + - uses: actions/setup-python@v4 + with: + python-version: '3.8' + - uses: ilammy/msvc-dev-cmd@v1 + with: + arch: ${{ matrix.arch }} + - name: Build dependencies + shell: cmd + run: | + python get-nasm.py ${{ matrix.arch }} + set path=%path%;%CD%\nasm-2.14.02 + mkdir binaries + cd binaries + cmake -G"NMake Makefiles" -DCMAKE_BUILD_TYPE=Release ../libjpeg-turbo + nmake + cd .. + - name: Build wheels + uses: pypa/cibuildwheel@v2.12.3 + env: + CIBW_SKIP: cp36-* pp* + CIBW_ARCHS_WINDOWS: ${{ matrix.arch }} + CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel" + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "python -m delvewheel repair -w {dest_dir} {wheel} --add-path binaries" + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + + upload-pypi: + if: github.event_name == 'release' && github.event.action == 'published' + needs: [build_wheels_linux, build_wheels_windows] + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v2 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_password }} diff --git a/.gitignore b/.gitignore index 0b8476e..e49492a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ *.pyc __pycache__ doc/_* + +build/ +binaries/ +CMakeFiles/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1125f6b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "libjpeg-turbo"] + path = libjpeg-turbo + url = https://github.com/libjpeg-turbo/libjpeg-turbo + branch = main diff --git a/get-nasm.py b/get-nasm.py new file mode 100644 index 0000000..9dcc5ce --- /dev/null +++ b/get-nasm.py @@ -0,0 +1,28 @@ +import urllib.request +from argparse import ArgumentParser +from pathlib import Path +from zipfile import ZipFile + +urls = { + "amd64": "https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/win64/nasm-2.14.02-win64.zip", + "x86": "https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/win32/nasm-2.14.02-win32.zip", +} + + +def main(): + parser = ArgumentParser() + parser.add_argument("arch", type=str.lower, choices=urls.keys()) + parser.add_argument("--zip-name", default="nasm.zip") + args = parser.parse_args() + + url = urls[args.arch] + urllib.request.urlretrieve(url, args.zip_name) + + with ZipFile(args.zip_name, "r") as zip: + zip.extractall() + + assert Path("nasm-2.14.02").exists() + + +if __name__ == "__main__": + main() diff --git a/jpegtran/jpegtran_build.py b/jpegtran/jpegtran_build.py index 76411f6..13a4d5a 100644 --- a/jpegtran/jpegtran_build.py +++ b/jpegtran/jpegtran_build.py @@ -12,13 +12,17 @@ #include "turbojpeg.h" """ +static_lib_dir = 'binaries' + ffi = FFI() ffi.set_source( "_jpegtran", SOURCE, sources=["src/epeg.c"], - include_dirs=["src"], + include_dirs=["src", "libjpeg-turbo", static_lib_dir], define_macros=[("HAVE_UNSIGNED_CHAR", "1")], - libraries=["jpeg", "turbojpeg"]) + libraries=['jpeg', 'turbojpeg'], + library_dirs=[static_lib_dir], +) ffi.cdef(CDEF) if __name__ == "__main__": diff --git a/libjpeg-turbo b/libjpeg-turbo new file mode 160000 index 0000000..3a53627 --- /dev/null +++ b/libjpeg-turbo @@ -0,0 +1 @@ +Subproject commit 3a53627306233013dcec61a90f0e9ed302ea5156 diff --git a/linux-build.sh b/linux-build.sh new file mode 100644 index 0000000..db6eb06 --- /dev/null +++ b/linux-build.sh @@ -0,0 +1,10 @@ +yum makecache +yum -y install nasm +python -m pip install -U pip wheel +python -m pip install -U cmake +mkdir binaries +cd binaries +cmake -G"Unix Makefiles" ../libjpeg-turbo +make +cd .. +ls binaries diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..94304cb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "cffi >= 1.0"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index f3f3f1a..26b7e43 100644 --- a/setup.py +++ b/setup.py @@ -5,11 +5,11 @@ if os.path.exists('README.rst'): if sys.version_info > (3,): - description_long = open('README.rst', encoding="utf-8").read() + long_description = open('README.rst', encoding="utf-8").read() else: - description_long = open('README.rst').read() + long_description = open('README.rst').read() else: - description_long = """ + long_description = """ A Python package for blazingly fast JPEG transformations. Compared to other, more general purpose image processing libraries like `wand-py`_ or `PIL/Pillow`_, the performance gain can, depending on the transformation, be @@ -23,14 +23,13 @@ name='jpegtran-cffi', version="0.5.3.dev", description=("Extremly fast, (mostly) lossless JPEG transformations"), - description_long=description_long, + long_description=long_description, author="Johannes Baiter", url="http://github.com/jbaiter/jpegtran-cffi.git", author_email="johannes.baiter@gmail.com", license='MIT', packages=['jpegtran'], package_data={'jpegtran': ['jpegtran.cdef']}, - setup_requires=['cffi >= 1.0'], install_requires=['cffi >= 1.0'], cffi_modules=["jpegtran/jpegtran_build.py:ffi"] ) From ca7b6a4686b6e1cbc7b4f92da053f1dd7c33e07a Mon Sep 17 00:00:00 2001 From: Dobatymo Date: Sat, 27 May 2023 08:14:35 +0800 Subject: [PATCH 2/5] change name to allow upload --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 26b7e43..95c382d 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ """ setup( - name='jpegtran-cffi', + name='jpegtran-cffi-wheels', version="0.5.3.dev", description=("Extremly fast, (mostly) lossless JPEG transformations"), long_description=long_description, From 9f30820b4415d709dc17939401ffb78047f04bb7 Mon Sep 17 00:00:00 2001 From: Dobatymo Date: Sat, 27 May 2023 09:09:14 +0800 Subject: [PATCH 3/5] fix readme format --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 227e48f..c299669 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ Wand and PIL were too slow to be usable. .. _PIL/PIllow: https://pillow.readthedocs.io .. _Benchmarks: https://jpegtran-cffi.readthedocs.io/en/latest/#benchmarks .. _epeg library: https://github.com/mattes/epeg -.. _libturbojpeg: http://www.libjpeg-turbo.org/About/TurboJPEG +.. _turbojpeg: http://www.libjpeg-turbo.org/About/TurboJPEG .. _libjpeg-turbo: http://www.libjpeg-turbo.org/ .. _CFFI: https://cffi.readthedocs.io .. _spreads: https://spreads.readthedocs.io From 0a9b09c9cd54c75c02184b7bbe43acc6737dbbeb Mon Sep 17 00:00:00 2001 From: Dobatymo Date: Sat, 27 May 2023 09:18:57 +0800 Subject: [PATCH 4/5] use newer pypi github action --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3703d03..5268549 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -75,7 +75,7 @@ jobs: name: artifact path: dist - - uses: pypa/gh-action-pypi-publish@master + - uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.pypi_password }} From 15860736c7572b2191d4ab418cfbab5980df7336 Mon Sep 17 00:00:00 2001 From: Dobatymo Date: Wed, 31 May 2023 20:11:46 +0800 Subject: [PATCH 5/5] install nasm for musllinux builds --- linux-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linux-build.sh b/linux-build.sh index db6eb06..4c0546d 100644 --- a/linux-build.sh +++ b/linux-build.sh @@ -1,5 +1,7 @@ yum makecache yum -y install nasm +apk update +apk add --upgrade nasm python -m pip install -U pip wheel python -m pip install -U cmake mkdir binaries