diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index d71f5dc6..00000000 --- a/.dockerignore +++ /dev/null @@ -1,15 +0,0 @@ -*.pyc -py_mini_racer/extension/v8/ -vendor/depot_tools/ -Dockerfile -Makefile -wheelhouse -dist -build -docker -Dockerfile.build -docker-compose.yml -Makefile -*.whl -**/*.so -.git diff --git a/.flake8 b/.flake8 deleted file mode 100644 index d909fe62..00000000 --- a/.flake8 +++ /dev/null @@ -1,10 +0,0 @@ -[flake8] -ignore = - E203, - E501, - W503 -exclude = - .tox, - docs/conf.py, - build, - py_mini_racer/extension diff --git a/.github/issue_template.md b/.github/issue_template.md index 78126cf8..68c8ef8d 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -1,12 +1,15 @@ ### Steps to reproduce ### Expected behavior + Tell us what should happen ### Actual behavior + Tell us what happens instead ### System configuration + **PyMiniRacer version**: **Python version**: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9a17357a..9010b3b0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,137 +1,395 @@ -name: Build +name: pypi-build + on: - push: - branches: [master] - pull_request: - branches: [master] + # For PRs: + pull_request: {} + # For releases: + push: + branches: + - 'release/*' concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: build-${{ github.head_ref }} + +defaults: + run: + shell: bash jobs: - build: - name: ${{ matrix.config.plat_name }} - strategy: - matrix: - config: - - os: ubuntu-16.04 - plat_name: manylinux1_x86_64 - - os: macos-10.15 - plat_name: macosx_10_10_x86_64 - - os: windows-2019 - plat_name: win_amd64 - fail-fast: true - runs-on: ${{ matrix.config.os }} - timeout-minutes: 180 - steps: - - name: Configure git - run: git config --global core.symlinks true - - - name: Clone repository - uses: actions/checkout@v1 - - - name: Install python 3.6 - uses: actions/setup-python@v1 - with: - python-version: 3.6.x - architecture: x64 - - - name: Install python 2.7 - uses: actions/setup-python@v1 - with: - python-version: 2.7.x - architecture: x64 - - - name: Build wheelhouse - run: | - python3 -m pip install setuptools wheel - mkdir wheelhouse - python3 helpers/build_package.py wheel wheelhouse - shell: bash - - - name: Archive wheelhouse - uses: actions/upload-artifact@v2 - with: - name: package-${{ matrix.config.plat_name }} - path: wheelhouse - - - name: Check the wheel - if: matrix.config.plat_name == 'manylinux1_x86_64' - run: | - python3 -m pip install auditwheel twine readme_renderer[md] - auditwheel show wheelhouse/*.whl - twine check wheelhouse/*.whl - - - name: Test - run: | - python3 -m pip install pytest wheelhouse/*.whl - pytest tests - shell: bash - - build-on-alpine: - name: alpine_x86_64 - runs-on: ubuntu-latest - container: - image: nicolassqreen/azure-pipelines-container-alpine-python:3.12 - timeout-minutes: 180 - steps: - - name: Clone repository - uses: actions/checkout@v1 - - - name: Download V8 sources - uses: docker://python:2 - with: - args: python helpers/v8_build.py --no-build --no-sysroot - - - name: Prepare Aline Linux build environment - run: | - sudo apk -U add samurai llvm lld linux-headers binutils-gold - cp -f /usr/local/bin/gn py_mini_racer/extension/v8/buildtools/linux64/gn - rm -f py_mini_racer/extension/depot_tools/ninja - - - name: Build the extension - run: | - python helpers/v8_build.py --no-update --no-sysroot --target py_mini_racer_shared_lib - cp py_mini_racer/extension/out/libmini_racer.so py_mini_racer/libmini_racer.muslc.so - - - name: Build the wheelhouse - run: | - sudo apk add py3-pip py3-wheel - mkdir wheelhouse - python3 setup.py sdist --dist-dir wheelhouse - - - name: Archive wheelhouse - uses: actions/upload-artifact@v2 - with: - name: package-alpine_x86_64 - path: wheelhouse - - - name: Test - run: | - python3 -m pip install pytest wheelhouse/*.tar.gz - pytest tests - - release: - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') - runs-on: ubuntu-latest - needs: [build-on-alpine, build] - steps: - - name: Download packages - uses: actions/download-artifact@v2 - with: - path: tmp - - - name: Move packages to the wheelhouse - run: | - mkdir wheelhouse - find tmp -name '*.whl' -exec mv {} wheelhouse \; - find tmp -name '*.tar.gz' -exec mv {} wheelhouse \; - shell: bash - - - name: Publish 📦 to PyPI - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.PYPI_TOKEN }} - packages_dir: wheelhouse/ + sdist: + name: Build sdist + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + # Fetch all tags + fetch-depth: 0 + + - name: Install deps + run: python -m pip install --upgrade build + + - name: Build + run: python -m build --sdist + + - uses: actions/upload-artifact@v3 + with: + name: sdist + path: dist/* + if-no-files-found: error + + # We build for Linux using uraimo/run-on-arch-action@v2, which runs a container under + # the runner in order to reach different platforms (notably Alpine with its musl) and + # architectures (notably aarch64) via emulation. uraimo/run-on-arch-action@v2 doesn't + # support Mac or Windows, so we run a separate job for those. + linux-wheels: + name: Build wheel for ${{ matrix.image }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + # To maximize compatibility of generated wheels, we should theoreticaly build + # on the *oldest* supported distros. + # But V8 ships its own bullseye sysroot and links against that, so we may as + # well run on bullseye (even though buster would provide an older supported + # build distro): + - debian:bullseye + - arm64v8/debian:bullseye + # Alpine 3.19 includes a clang new enough for V8 to build (with only minor + # patches!). Builds on 3.19 seem incompatible with <= 3.18 due to libstdc++ + # symbols. (And we can't just run on an old Alpine and update clang from the + # llvm site, because unlike Debian, the llvm project doesn't maintain + # updated packages for old Alpine distros.) + - alpine:3.19 + - arm64v8/alpine:3.19 + exclude: + # The aarch64 builds run really slowly, so let's skip them except for + # releases. + # For more info on this GitHub Actions hack, see: + # https://stackoverflow.com/questions/65384420/how-do-i-make-a-github-action-matrix-element-conditional#answer-73822998 + - image: ${{ !startsWith(github.ref, 'refs/heads/release/') && 'arm64v8/debian:bullseye' }} + - image: ${{ !startsWith(github.ref, 'refs/heads/release/') && 'arm64v8/alpine:3.19' }} + + steps: + - name: Configure git + run: git config --global core.symlinks true + + - uses: actions/checkout@v3 + with: + # Fetch all tags + fetch-depth: 0 + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + # For security reasons, only use sccache on releases: + if: ${{ startsWith(github.ref, 'refs/heads/release/') }} + + - uses: uraimo/run-on-arch-action@v2 + name: Build wheel + with: + arch: none + distro: none + base_image: ${{ matrix.image }} + + setup: | + mkdir -p "${PWD}/wheels" + + dockerRunArgs: | + --volume "${PWD}/wheels:/wheels" + + shell: /bin/sh + + install: | + case "${{ matrix.image }}" in + *debian*) + # Let's download some system packages! + # Note that the precise list of packages we need is intertwined not just + # with V8, but with helpers/build_v8.py, which makes various decisions + # about using system versus V8-provided tools and dependencies. + PACKAGES="" + # First, some basic wheel-building dependencies: + PACKAGES="${PACKAGES} python3" + PACKAGES="${PACKAGES} pip" + PACKAGES="${PACKAGES} python3-venv" + # helpers/v8_build.py uses git to download depot_tools, and + # depot_tools/gclient uses it to grab V8: + PACKAGES="${PACKAGES} git" + # depot_tools/cipd uses curl to download various things including its own + # managed python: + PACKAGES="${PACKAGES} curl" + # the V8 build generation system (GN) uses pkg-config: + PACKAGES="${PACKAGES} pkg-config" + # the clang build script still uses the system lld: + PACKAGES="${PACKAGES} lld" + # we use grep below + PACKAGES="${PACKAGES} grep" + apt-get update -q -y + apt-get install -q -y ${PACKAGES} + + # the V8 build runs on clang by default. On mainstream platforms, V8 + # bundles a working clang, but not on arm64. So we need to bring our own + # clang and llvm and lld. + # bullseye ships an ancient llvm, clang, and lld. Add the latest stable: + LLVM_VERSION=18 + curl https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${LLVM_VERSION} main" \ + > /etc/apt/sources.list.d/llvm.list + apt-get update -q -y + PACKAGES="clang-${LLVM_VERSION} llvm-${LLVM_VERSION} lld-${LLVM_VERSION}" + apt-get install -q -y ${PACKAGES} + # set up non-versioned symlinks to the latest stable clang: + for pkg in ${PACKAGES}; do + for f in $(dpkg -L "${pkg}" | grep "/usr/bin/.*-${LLVM_VERSION}"); do + no18="${f%-${LLVM_VERSION}}" + rm -f "${no18}" + ln -s "${f}" "${no18}" + done + done + ;; + *alpine*) + # Let's download some system packages! + # Note that the precise list of packages we need is intertwined not just + # with V8, but with helpers/build_v8.py, which makes various decisions + # about using system versus V8-provided tools and dependencies. + PACKAGES="" + # First, some basic wheel-building dependencies: + PACKAGES="${PACKAGES} python3" + PACKAGES="${PACKAGES} py3-pip" + PACKAGES="${PACKAGES} py3-virtualenv" + # helpers/v8_build.py uses git to download depot_tools, and + # depot_tools/gclient uses it to grab V8: + PACKAGES="${PACKAGES} git" + # depot_tools/cipd uses curl to download various things including its own + # managed python: + PACKAGES="${PACKAGES} curl" + # depot_tools/vpython3 runs on bash: + PACKAGES="${PACKAGES} bash" + # pip needs the following to build some dependencies on the fly on + # aarch64: + PACKAGES="${PACKAGES} gcc" + PACKAGES="${PACKAGES} python3-dev" + PACKAGES="${PACKAGES} musl-dev" + PACKAGES="${PACKAGES} libffi-dev" + # the V8 build system uses gn to generate build files and on Alpine the + # one it bundles uses glibc (not musl) and thus does not work: + PACKAGES="${PACKAGES} gn" + # We need patch to apply patches to V8: + PACKAGES="${PACKAGES} patch" + # the V8 build runs on clang by default. On mainstream platforms, V8 + # bundles a working clang, but not on Alpine. So get our own clang, llvm, + # and lld: + PACKAGES="${PACKAGES} clang" + PACKAGES="${PACKAGES} llvm" + PACKAGES="${PACKAGES} lld" + # the V8 build system includes a sysroot (/usr/include, /usr/lib, etc) on + # many platforms, but not Alpine. Thus we have to provide extra build-time + # deps: + PACKAGES="${PACKAGES} glib-dev" + # the V8 build system has its own libstdc++, but it doesn't seem to work + # on Alpine, so we get and use the system one: + PACKAGES="${PACKAGES} libc++-dev" + apk update + apk add ${PACKAGES} + ;; + esac + + if [ ! -z "${{ env.ACTIONS_CACHE_URL }}" ]; then + # Set up sccache. + # We can't directly use the binary fetched by Mozilla's GitHub SCCache + # Action, because we're doing a cross-arch build (the job is on x86_64 but + # the container may be on aarch64). So we reproduce the logic from the + # action: + # https://github.com/Mozilla-Actions/sccache-action/blob/main/src/setup.ts + SCCACHE_VERSION="v0.7.7" + SCCACHE_ARCH="$(uname -m)" + SCCACHE_BASE_URL="https://github.com/mozilla/sccache/releases/download" + # Note that the sccache binary is statically linked and thus we can use the + # musl binary on Debian (this is indeed what Mozilla's Github Action does): + SCCACHE_PLATFORM="unknown-linux-musl" + SCCACHE_URL="${SCCACHE_BASE_URL}/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-${SCCACHE_PLATFORM}.tar.gz" + + mkdir /sccache + cd /sccache + curl --location "${SCCACHE_URL}" > sccache.tar.gz + tar --strip-components 1 -xvzf sccache.tar.gz + fi + + run: | + set -e + + if [ ! -z "${{ env.ACTIONS_CACHE_URL }}" ]; then + export SCCACHE_GHA_ENABLED="true" + export SCCACHE_PATH=/sccache/sccache + export ACTIONS_CACHE_URL="${{ env.ACTIONS_CACHE_URL }}" + export ACTIONS_RUNTIME_TOKEN="${{ env.ACTIONS_RUNTIME_TOKEN }}" + fi + + python3 -m venv /venv + . /venv/bin/activate + python3 -m pip install --upgrade build + python3 -m build --wheel + cp dist/*.whl /wheels/ + chmod a+rwx /wheels/*.whl + + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: ./wheels/* + if-no-files-found: error + + - uses: uraimo/run-on-arch-action@v2 + name: Test wheel + with: + arch: none + distro: none + base_image: ${{ matrix.image }} + + setup: | + mkdir -p "${PWD}/wheels" + + dockerRunArgs: | + --volume "${PWD}/wheels:/wheels" + + shell: /bin/sh + + install: | + case "${{ matrix.image }}" in + *debian*) + PACKAGES="" + PACKAGES="${PACKAGES} python3" + PACKAGES="${PACKAGES} pip" + PACKAGES="${PACKAGES} python3-venv" + apt-get update -q -y + apt-get install -q -y ${PACKAGES} + ;; + *alpine*) + PACKAGES="" + PACKAGES="${PACKAGES} python3" + PACKAGES="${PACKAGES} py3-pip" + PACKAGES="${PACKAGES} py3-virtualenv" + # Needed by libmini_racer.so: + PACKAGES="${PACKAGES} libatomic" + apk update + apk add ${PACKAGES} + ;; + esac + + run: | + set -e + + python3 -m venv /venv + . /venv/bin/activate + case "${{ matrix.image }}" in + *debian*) + python3 -m pip install --upgrade hatch hatch-fancy-pypi-readme hatch-mkdocs + hatch run testinstalled:install /wheels/*.whl + hatch run testinstalled:run + ;; + *alpine*) + # Due to https://github.com/indygreg/python-build-standalone/issues/86 + # Hatch's matrix testing doesn't work on Alpine. Just run pytest directly + # on the system Python version: + python3 -m pip install --upgrade pytest + python3 -m pip install /wheels/*.whl + pytest tests + ;; + esac + + # We build for Mac and Windows directly on the supported GitHub-hosted runners. + non-linux-wheels: + name: Build wheel for ${{ matrix.config.os }} + runs-on: ${{ matrix.config.os }} + strategy: + fail-fast: false + matrix: + config: + # Note that to maximize compatibility of generated wheels, we build on the + # *oldest* supported GitHub-hosted runners, per + # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners + - os: windows-2019 # x86_64 + - os: macos-11 # this is the earliest x86_64 runner + - os: macos-14 # this is the earliest arm64 runner + + steps: + - name: Configure git + run: git config --global core.symlinks true + + - uses: actions/checkout@v3 + with: + # Fetch all tags + fetch-depth: 0 + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + # For security reasons, only use sccache on releases: + if: ${{ startsWith(github.ref, 'refs/heads/release') }} + + - uses: actions/setup-python@v5 + with: + # depot_tools still uses distutils which is gone in 3.12: + python-version: '3.11' + + - name: Build wheel + run: | + set -e + python3 -m pip install --upgrade build + python3 -m build --wheel + + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist/* + if-no-files-found: error + + - name: Test wheel + run: | + set -e + python3 -m pip install --upgrade hatch hatch-fancy-pypi-readme hatch-mkdocs + hatch run testinstalled:install dist/*.whl + hatch run testinstalled:run + + release: + name: Create GitHub release + if: startsWith(github.ref, 'refs/heads/release') + needs: + - linux-wheels + - non-linux-wheels + - sdist + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v3 + + - name: Compute release version + run: | + VERSION=${GITHUB_REF_NAME#release/} + echo Version: $VERSION + echo "VERSION=$VERSION" >> $GITHUB_ENV + + - name: Make release + uses: ncipollo/release-action@v1 + with: + artifacts: "wheels/*,sdist/*" + commit: ${{ github.ref }} + tag: ${{ env.VERSION }} + + publish: + name: Upload release to PyPI + if: startsWith(github.ref, 'refs/heads/release') + needs: + - release + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/mini-racer + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - uses: actions/download-artifact@v3 + + - name: Make dist directory + run: mkdir dist && cp wheels/* sdist/* dist/ + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..3fa0ed18 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,24 @@ +name: docs +on: + push: + tags: + - v* + branches: + - main +permissions: + contents: write +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: actions/setup-python@v5 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install hatch hatch-mkdocs + - run: hatch -vv run docs:gh-deploy --force diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index a8b77347..819546a4 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,4 +1,4 @@ -name: Linter Checks +name: Pre-commit checks on: [pull_request] @@ -7,13 +7,41 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: pre-commit|${{ hashFiles('.pre-commit-config.yaml') }} - - uses: pre-commit/action@v1.0.1 + # For clang-tidy and clang-format, let's lint and format with the same release + # of LLVM that we compile with: + - name: Install up-to-date LLVM + run: | + LLVM_VERSION=18 + UBUNTU_VERSION=jammy + curl https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb http://apt.llvm.org/${UBUNTU_VERSION}/ llvm-toolchain-${UBUNTU_VERSION}-${LLVM_VERSION} main" \ + | sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt-get update -q -y + sudo apt-get install -q -y "clang-format-${LLVM_VERSION}" "clang-tidy-${LLVM_VERSION}" + sudo ln -sf "/usr/bin/clang-format-${LLVM_VERSION}" /usr/bin/clang-format + sudo ln -sf "/usr/bin/clang-tidy-${LLVM_VERSION}" /usr/bin/clang-tidy + + - uses: actions/setup-python@v5 with: - token: ${{ secrets.GITHUB_TOKEN }} + # depot_tools still uses distutils which is gone in 3.12: + python-version: 3.11 + + - name: Install deps + run: python -m pip install --upgrade hatch hatch-mkdocs packaging httplib2 + + # needed for clang-tidy: + - name: Grab V8 source + run: python helpers/v8_build.py --fetch-only + + # Hatch seems to fall over itself due to race conditions on the first pre-commit + # run in an environment. (Specifically, there are errors in directory and symlink + # creation within the virtual env.) Help it by just initializing docs serially + # beforehand: + - name: Set up hatch docs env + run: hatch env create docs + + # Same as above, for the mypy env: + - name: Set up hatch types env + run: hatch env create types + + - uses: pre-commit/action@v3.0.1 diff --git a/.gitignore b/.gitignore index 61da2b06..1405308c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ __pycache__/ *.dylib *.dll +# generated V8 build artifacts +icudtl.dat +snapshot_blob.bin + # Distribution / packaging .Python env/ @@ -26,6 +30,7 @@ var/ *.egg-info/ .installed.cfg *.egg +site/ # PyInstaller # Usually these files are written by a python script from a template @@ -39,7 +44,6 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ -.tox/ .coverage .coverage.* .cache @@ -61,17 +65,7 @@ docs/_build/ # PyBuilder target/ -# V8 vendor -py_mini_racer/extension/.cipd -py_mini_racer/extension/.gclient* -py_mini_racer/extension/v8 -py_mini_racer/extension/out -py_mini_racer/extension/depot_tools -py_mini_racer/extension/build -py_mini_racer/extension/build_overrides -py_mini_racer/extension/buildtools -py_mini_racer/extension/testing -py_mini_racer/extension/third_party -py_mini_racer/extension/tools +# V8 build +v8_workspace *.iml diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index fb55a6ce..00000000 --- a/.isort.cfg +++ /dev/null @@ -1,14 +0,0 @@ -[settings] -combine_as_imports = True -default_section = THIRDPARTY -force_grid_wrap = 0 -include_trailing_comma = True -known_first_party = py_mini_racer -multi_line_output = 3 -skip_glob = - .venv.* -skip = - .asv, - .tox, - docs, - py_mini_racer/extension diff --git a/.mdformat.toml b/.mdformat.toml new file mode 100644 index 00000000..05229142 --- /dev/null +++ b/.mdformat.toml @@ -0,0 +1 @@ +wrap = 88 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f16754e..01243f9d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,28 +1,70 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: -- repo: https://github.com/PyCQA/flake8 - rev: 3.7.9 - hooks: - - id: flake8 -- repo: https://github.com/timothycrosley/isort - rev: 5.6.4 +- repo: local hooks: - - id: isort + - id: hatch-fmt + name: Hatch format and lint + entry: hatch fmt # runs Ruff, which includes both linting and formatting. + language: system + types: [python] + pass_filenames: false + - id: hatch-docs-format + name: Hatch format docs + entry: hatch run docs:fmt # runs mdformat + language: system + - id: hatch-docs-build + name: Hatch build docs + entry: hatch run docs:build --strict # checks for, e.g., bad links + language: system + pass_filenames: false + - id: hatch-types-check + name: Hatch check types + entry: hatch run types:check + language: system + types: [python] + files: 'src/py_mini_racer' - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - exclude: \.patch$ + exclude: \.(patch|md)$ - id: end-of-file-fixer - exclude: \.patch$ + exclude: \.(patch|md)$ - id: check-yaml - id: check-added-large-files -- repo: local +- repo: https://github.com/pocc/pre-commit-hooks + rev: v1.3.5 hooks: - - id: clang-format - name: Check C++ format - entry: clang-format - language: system - files: py_mini_racer/extension/mini_racer_extension.cc - args: ['-style=Chromium', '-i'] + - id: clang-format + args: + - --style=Chromium + - -i + - id: clang-tidy + args: + # Things we're disabling: + # lvmlibc-* is intended for the llvm project itself + # llvm-header-guard is likewise hard-coded for the llvm project itself + # llvm-include-order fights clang-format --style=Chromium + # fuchsia has some odd opinions, so disable it globally. + # altera-* is designed for FPGAs and gives weird performance advice. + # we aren't using Google's absl lib, so disable its suggestions. + - >- + -checks=*, + -llvmlibc-*, + -llvm-header-guard, + -llvm-include-order, + -fuchsia-*, + -altera-*, + -abseil-* + - -warnings-as-errors=* + # Interpret .h as C++: + - -extra-arg-before=-xc++ + - -extra-arg=-std=c++20 + - -extra-arg=-stdlib=libstdc++ + - -extra-arg=-isystem + - -extra-arg=v8_workspace/v8/include + # The real build knows the sandbox is enabled but clang-tidy needs to be + # informed: + - -extra-arg=-DV8_ENABLE_SANDBOX + - -extra-arg=-DV8_COMPRESS_POINTERS diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..6cb05587 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,508 @@ +# Architecture + +This document contains some notes about the design of PyMiniRacer. + +## Security goals + +**First and foremost, PyMiniRacer makes no guarantees or warrantees, as noted in the +license.** This section documents the security *goals* of PyMiniRacer. Anything that +doesn't meet these goals should be considered to be a bug (but with no warrantee or even +a guaranteed path to remediation). + +### PyMiniRacer should *be able to* run untrusted JavaScript code + +The ability for PyMiniRacer to run untrusted JavaScript code +[was an original design goal for Sqreen](https://news.ycombinator.com/item?id=39754885#39813985) +in developing PyMiniRacer, and continues to be a design goal today. + +To that end, PyMiniRacer provides: + +1. The innate sandboxing properties of V8. V8 is trusted by billions of folks to run + untrusted JavaScript every day, as a part of Chrome and other web browsers. It has + many features like the [security sandbox](https://v8.dev/blog/sandbox) and + undergoes close security scrutiny. + +1. The ability to create multiple `MiniRacer` instances which each have separate V8 + isolates, to separate different blobs of untrusted code from each other. + +1. Optional timeouts and memory constraints on code being executed. + +Caveats: + +1. The continual security research is V8 under yields a corresponding + [stream of vulnerability reports](https://cve.mitre.org/cgi-bin/cvekey.cgi?keyword=v8). + + +1. ... and while V8 *as embedded in a web browser* will typically receive (funded!) + updates to correct those vulnerabilities, PyMiniRacer is unlikely to see as + aggressive and consistent an update schedule. + +1. ... and of course PyMiniRacer itself may have vulnerabilities. + [This has happened before](https://nvd.nist.gov/vuln/detail/CVE-2020-25489). + +1. ... and even if PyMiniRacer is updated to accomodate a vulnerability fix in itself or + V8, it is incumbent upon Python applications which integrate it to actually + redeploy with the new PyMiniRacer version. + +If running potentially adversarial JavaScript code in a high-security environment, it +might be a better choice to run code using a purpose-built isolation environment such as +containers on [gVisor](https://gvisor.dev/), than to rely on PyMiniRacer for isolation. + +### JavaScript-to-Python callbacks may breach any isolation boundary + +The `MiniRacer.wrap_py_function` method allows PyMiniRacer users to expose Python +functions *they write* to JavaScript. This creates an extension framework which +essentially breaches the isolation boundary provided by V8. + +This feature should only be used if the underlying JavaScript code *is* trusted, or if +the author is certain the exposed Python function is safe for calls from untrusted code. +(I.e., if you expose a Python function which allows reading arbitrary files from disk, +this would obviously be bad if the JavaScript code which may call it is itself +untrusted.) + +## Brief catalog of key components + +### `docs/` + +This is the [`mkdocs` ](https://www.mkdocs.org/) site for PyMiniRacer. To maximize +compatibility with standard open-source repository layout, this directory is just a +bunch of stubs which include files from the package root. + +### `hatch_build.py` + +This is a Hatch build hook which builds Python wheels, by calling `helpers/v8_build.py`. + +### `helpers/v8_build.py` + +This is the PyMiniRacer V8 build wrapper. Building V8 for many platforms (Windows, Mac, +glibc Linux, musl Linux) *and* architectures (x86_64, aarch64) is hard, especially since +V8 is primarily intended to be built by Google engineers on a somewhat different set of +of platforms (i.e., those Chrome runs on), and typically via cross-compiled from +relatively curated build hosts. So this file is complicated and full of `if` statements. + +### `src/v8_py_frontend/` + +This is a small frontend for V8, written in C++. It manages initialization, context, +marshals and unmarshals inputs and outputs through V8's type system, etc. The front-end +exposes simple functions and types which are friendly to the Python `ctypes` system. +These simple C++ functions in turn call the C++ V8 APIs. + +As noted below, `v8_py_frontend` is *not* a Python extension (it does *not* include +`Python.h` or link `libpython`, and it does not touch Python types). + +### (Compiled) `src/py_mini_racer/libmini_racer.so`, `src/py_mini_racer/mini_racer.dll`, `src/py_mini_racer/libmini_racer.dylib` + +These files (*which one* depends on the platform) contain the compiled V8 build, +complete with the frontend from `src/v8_py_frontend`. + +### (Compiled) `src/py_mini_racer/icudtl.dat` + +This is a build-time-generated internationalization artifact, used +[at runtime by V8](https://v8.dev/docs/i18n) and thus shipped with PyMiniRacer. + +### (Compiled) `src/py_mini_racer/snapshot_blob.bin` + +This is a build-time-generated +[startup snapshot](https://v8.dev/blog/custom-startup-snapshots), used at runtime by V8 +and thus shipped with PyMiniRacer. This is a snapshot of the JavaScript heap including +JavaScript built-ins, which accelerates JS engine startup. + +### `src/py_mini_racer/` + +This is the pure-Python implementation of PyMiniRacer. This loads the +(Python-independent) PyMiniRacer dynamic-link library (`.dll` on windows, `.so` on +Linux, `.dylib` on MacOS) and uses the Python `ctypes` system to call methods within it, +to manage V8 context and actually evaluate JavaScript code. + +### `.github/workflows/build.yml` + +This is the primary build script for PyMiniRacer, implemented as a GitHub Actions +workflow. + +## Design decisions + +These are listed in a topological sort, from most-fundamental to most-derived decisions. + +In theory, answers to questions in the vein of "Why is it done this way?" belong in this +section. + +### Minimize the interface with V8 + +V8 is extremely complex and is under continual, heavy development. Such development can +result in interface changes, which may in turn break PyMiniRacer. + +To mitigate the risk of breakage with new V8 builds, we seek to minimize the "API +surface area" between PyMiniRacer and V8. This means we seek to limit "advanced" use of +both: + +1. The V8 C++ API, and +1. The V8 build system (GN) and build options. + +Our success at minimizing the interface with the V8 build system can be measured by: + +1. The number of times the text `v8::` appears in `src/v8_py_frontend`, and +1. The length of `helpers/v8.build.py` (467 lines as of this writing!). Making V8 build + on multiple platforms takes a lot of trickery... + +### Build V8 from source + +The V8 project does not produce stable binary distributions, i.e., static or dynamic +libraries. (In Linux terms, this would probably look like dpkgs and rpms with names like +`libv8` and `libv8-dev`.) Instead, any project (like NodeJS, Chromium, or... +PyMiniRacer!) which wants to integrate V8 must first build it. + +### Build PyPI wheels + +Because V8 takes so long to build (about 2-3 hours at present on the free GitHub Actions +runners, and >12 hours when emulating `aarch64` on them), we want to build wheels for +PyPI. We don't want folks to have to build V8 when they `pip install mini-racer`! + +We build wheels for many operating systems and architectures based on popular demand via +GitHib issues. Currently the list is +`{x86_64, aarch64} × {Debian Linux, Alpine Linux, Mac, Windows}` (but skipping Windows +`aarch64` for now since there is not yet either a GitHub Actions runner, or emulation +layer for it). + +### Use the free GitHub Actions hosted runners + +PyMiniRacer is not a funded project, so we run on the free GitHub Actions hosted +runners. These currently let us build for many key platforms (including via emulation). + +This also lets contributors easily run the same build automation by simply forking the +PyMiniRacer repo and running the workflows (for free!) within their own forks. + +### Don't interface with the CPython API (don't make an extension) + +We'd rather avoid directly interfacing with the CPython API, for a couple reasons: + +1. **API flux**: Similar to the above note about V8, the CPython API is complex and + always in flux, *although not as much as V8*). +1. **Version proliferation**: there are a ton of active Python versions (as of this + writing, PyMiniRacer supports 3.8, 3.9, 3.10, 3.11, and 3.12, and also there's + CPython and PyPy). PyMiniRacer *already* includes builds for 7 target architectures + (see above); if we factor in 5x Python versions and 2x Python interpreters, we will + wind up with 70 wheels, all on a free GitHub Actions runner! + +So, *instead of* an extension module (which includes `Python.h` and links against +`libpython`), we build an ordinary Python-independent C++ library, and use `ctypes` to +access it. + +Consequently, `libmini_racer.so` isn't specific to Python, and the code barely mentions +Python. One could in theory use it from any other language which knows how to call C +APIs, such as Java, Go, C#, ... or just C. No one does so as of this writing. + +### Use `uraimo/run-on-arch-action` + +So, we need to build wheels for multiple architectures. For Windows and Mac (`x86_64` on +Windows, and both `x86_64` and `aarch64` on Mac) we can can use GitHub hosted runners +as-is. For Linux builds (Debian and Alpine, and `x86_64` and `aarch64`), we use the +fantastic GitHub Action workflow step +[`uraimo/run-on-arch-action`](https://github.com/uraimo/run-on-arch-action), which lets +us build a docker container on the fly and run it on QEMU. + +### Don't use `cibuildwheel` + +Many modern Python projects which need to build wheels with native code use +[the `cibuildwheel` project](https://github.com/pypa/cBbuildwheel) to manange their +builds. However, `cibuildwheel` isn't a perfect fit here. Because we are building +Python-independent dynamic-link libraries instead of Python extension modules modules +for the reasons noted above, we aren't linking with any particular Python ABI. Thus we +need *only* `(operating systems × architectures)` builds, whereas `cibuildwheel` +generates `(operating systems × architecture × Python flavors × Python versions)` +wheels. +[That's a ton of wheels](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)! +Given that it takes hours to *days* to build PyMiniRacer for *one* target OS and +architecture, doing redundant builds is undesirable. + +It might be possible to use `cibuildwheel` with PyMiniRacer by segmenting the build of +the dynamic-link library (i.e., `libmini_racer.so`) from the actual wheel build. That +is, we could have the following separate components: + +1. Create a separate Github Actions workflow to build the `libmini_racer.so` binary + (i.e., the hard part). Publish that as a release, using the GitHub release artifact + management system as a distribution mechanism. +1. The wheel build step could then simply download a pre-built binary from the latest + GitHub release. We could use `cibuildwheel` to manage this step. This would + generate many redundant wheels (because the wheels we'd generate for, say, CPython + 3.9 and 3.10 would be identical), but it wouldn't matter because it would be cheap + and automatic. + +This is similar to how the Ruby [`mini_racer`](https://github.com/rubyjs/mini_racer) and +[`libv8-node`](https://github.com/rubyjs/libv8-node) projects, which inspired +PyMiniRacer, work together today. + +To sum up, to use `cibuildwheel`, we would still need our own *separate* +multi-architecture build workflow for V8, *ahead of* the `cibuildwheel` step. So +`cibuildwheel` could potentially simplify the actual wheel distribution for us, but it +wouldn't simplify the overall workflow management. + +### Use `sccache` to patch around build timeouts + +As of this writing, the Linux `aarch64` builds run on emulation becaues GitHub Actions +has no free hosted `aarch64` runners for Linux. This makes them so slow, they struggle +to complete at all. They take about 24 hours to run. The GitHub Actions +[job timeout is only 6 hours](https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration#usage-limits), +so we have to restart the jobs multiple times. We rely on +[`sccache`](https://github.com/mozilla/sccache) to catch the build up to prior progress. + +It would in theory be less ugly to segment the build into small interlinked jobs of less +than 6 hours each so they each succeed, but for now it's simpler to just manually +restart the failed jobs, each time loading from the build cache and making progress, +until they finally succeed. Hopefully at some point GitHub will provide native `aarch64` +Linux runners, which will alleviate this problem. + +Hopefully, +[per this GitHub community discussion thread](https://github.com/orgs/community/discussions/19197), +we will get a free Linux `aarch64` runner in 2024 and can dispense with +cross-architecture emulation. + +### Build V8 *with* our frontend (`v8_py_frontend`) as a snuck-in component + +We could *just* get a static library (i.e., `libv8.a`) from the V8 build, and link that +into a dynamic-link library (i.e., `libmini_racer.so`) ourselves. + +However: + +1. We do have *more* C++ files to compile (the C++ code in `src/v8_py_frontend`) +1. Because we're not making a true Python extension module (see above), we aren't using + Python's `setuptools` `Extension` infrastructure to perform a build. + +This leaves us needing *some* platform-independent C++ toolchain. + +V8 already has such a toolchain, based on Ninja and Generated Ninja files (GN). We +already have to set it up to build V8 from source (see above for why!). + +Rather than bringing in yet another toolchain, we sneak `v8_py_frontend` into the V8 +tree itself, as a "custom dep". We then instruct GN to build it as if it were an +ordinary part of V8. + +The result is a dynamic-link library which contains an ordinary release build of V8, +plus our Python `ctypes`-friendly frontend. + +### Buggy or adversarial JavaScript shouldn't be able to crash or otherwise disrupt things + +Per the security goals above, we want PyMiniRacer to be able to run untrusted JavaScript +code safely. This means we can't trust JavaScript to "behave". Intentionally bad (i.e, +adversarial) or **un**intentionally bad (i.e., buggy) JavaScript should not be able to: + +1. Crash PyMiniRacer, +1. Read arbitrary memory, or +1. Use infinite CPU or memory resources + +For the latter, the PyMiniRacer Python API exposes optional constraints on memory usage +as well as timeouts. The former two rules are enforced by the design of the C++ side of +PyMiniRacer, and of course V8 itself. + +### Don't trust JavaScript with memory management of C++ objects + +JavaScript is a garbage-collected language, and like many such languages it +[offers](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/FinalizationRegistry) +best-effort finalizer functionality, into which you can inject code which gets called +when the runtime is disposing of an object. + +However, with +[V8-based JavaScript](https://stackoverflow.com/questions/24107280/v8-weakcallback-never-gets-called), +actually relying on this functionality to trigger callbacks to C++ to clean things up is +heartily discouraged. Exploratory attempts to make this with PyMiniRacer actually didn't +work at all. + +Even if we *could* get V8 to call us back reliably to tear down objects (e.g., by +exposing an explicit teardown function to JavaScript), it would be hard to create a +design which does so safely. V8 (per our security goals) may be running adversarial +JavaScript which might try and use a reference after we free it, exploiting a +use-after-free bug. + +### Any raw C++ object pointers and references given to JavaScript must outlive the `v8::Isolate` + +Due to the above rule, we can't rely on V8 to tell us when it's *done with* any +references we give it, until the `v8::Isolate` is torn down. So clearly the only thing +we *can* do is ensure any raw pointers or references we hand to V8 are valid until after +the `v8::Isolate` is torn down. + +### Use JavaScript integer IDs to track any allocated objects on the C++ side + +The above said, we still have cases where we want to tell JavaScript about objects which +have shorter lifecycles than the `v8::Isolate` itself. E.g., a function callback from +JavaScript to C++ (and thus to Python) might only be used as a single `Promise.then` +callback. If a long-running program were to create tons of `Promise`s, we'd want to +garbage collect the callbacks as we go, without waiting for the whole `v8::Isolate` to +exit. + +We can treat that case by "laundering" our raw C++ pointers and references through C++ +maps (i.e., `std::unordered_map>`), giving V8 JavaScript +only IDs into the map. We can convert IDs back into C++ pointers when JavaScript calls +us back, after checking that they're still valid. (And we use `std::shared_ptr` to avoid +tear-down race conditions wherein a map entry is removed in one thread while we're still +using an object in another.) + +In this manner, the C++ side can be authoritative about when objects are torn down. It +can delete C++ objects and remove them from the map whenever it sees fit. If JavaScript +tries to use the ID after that point, such usage can be easily spotted and safely +ignored. + +### Buggy Python shouldn't be able to crash C++ + +Similar to, but with a lower priority than the above rule regarding bad *JavaScript*, +bad *Python* should not be able to crash the Python interpretter through PyMiniRacer. +This is a common design principle for Python; bad code should *not* result in +segmentation faults, sending developers scrambling to C/C++ debugging of core files, +etc. Extension modules should uphold this principle. + +This applies only to **un**intentionally bad (i.e., buggy) Python code. PyMiniRacer does +not and cannot protect itself from intentionally bad (i.e., adversarial) Python code. A +determined Python programmer can always crash Python with ease without any help from +PyMiniRacer. Try it!: `import ctypes; ctypes.cast(0x1, ctypes.c_char_p).value` + +### Minimize trust of Python in *automatic* memory management of C++ objects + +Python is also a garbage-collected language, and like JavaScript, it +[offers](https://docs.python.org/3/reference/datamodel.html#object.__del__) best-effort +finalizer functionality. + +Like in JavaScript code, relying on Python's finalizer functionality is +[heartily discouraged](https://stackoverflow.com/questions/6104535/i-dont-understand-this-python-del-behaviour#answer-6104568). +We can, at best, use `__del__` as a shortcut signaling we can go ahead and free +something to help reduce memory usage, but we shouldn't rely on it. + +Since, unlike JavaScript, we *do* trust Python code, we can create explicit Python APIs +to manage object lifecycle. The Pythonic way to do that is with +[context managers](https://docs.python.org/3/reference/datamodel.html#context-managers). + +Thus, for example, the MiniRacer Python `_Context` object, which wraps exactly one C++ +`MiniRacer::Context` object, provides *both* a `__del__` finalizer for easy cleanup +which always works "eventually", and an explicit context manager interface for +PyMiniRacer users who want strong guarantees about teardown. + +### Minimize trust of Python in handing C/C++ pointers + +The `ctypes` module lets Python directly wrangle C/C++ pointers. This can be used to +send, receive, and mutate data shared between Python and C. + +This is obviously somewhat dangerous. Array overruns are an obvious problem. +Use-after-free is more insidious: imagine the C++ side of PyMiniRacer returns a pointer +to an object to Python, Python stores that pointer, the C++ frees the object, and then +Python tries to use the pointer. This will work *sometimes* and crash—or worse, read +incorrect data—at other times. + +### Use Python integer IDs to track any allocated objects on the C++ side + +Thus, combining all the above rules, we wind up with a similar rule for Python as we +have for JavaScript. Wherever possible, we avoid interchanging raw pointers between C++ +and Python. Instead, we interchange integer IDs. The C++ side of PyMiniRacer can convert +integer IDs to raw pointers using a map, after validating that the IDs are still valid. + +### ... except for `BinaryValueHandle` pointers + +We break the above rule for `BinaryValueHandle` pointers. PyMiniRacer uses +`BinaryValueHandle` to exchange most data between Python and C++. Python directly reads +the contents of `BinaryValueHandle` pointers, to read primitive values (e.g., booleans, +integers, and strings). + +We do this for theoretical performance reasons which have not yet been validated. To be +consistent with the rest of PyMiniRacer's design, we *could* create an API like: + +1. C++ generates a numeric `value_id` and stores a BinaryValue in a + `std::unordered_map>`. +1. C++ gives Python that `value_id` to Python. +1. To get any data Python has to call APIs like `mr_value_type(context_id, value_id)`, + `mr_value_as_bool(context_id, value_id)`, + `mr_value_as_string_len(context_id, value_id)`, + `mr_value_as_string(context_id, value_id, buf, buflen)`, ... +1. Eventually Python calls `mr_value_free(context_id, value_id)` which wipes out the map + entry, thus freeing the `BinaryValue`. + +_**Note: We don't do this. The above is _not_ how PyMiniRacer actually handles +values.**_ + +This is surely slower than direct pointer access, but no performance analysis has been +done to see if it matters. It might be interesting to try the above and benchmark it. It +would be nice to switch to that model if it's sufficiently performant. + +For now at least, we instead use raw pointers for this case. + +We still don't fully trust Python with the lifecyce of `BinaryValueHandle` pointers; +when Python passes these pointers back to C++, we still check validity by looking up the +pointer as a key into a map (which then lets the C++ side of PyMiniRacer find the *rest* +of the `BinaryValue` object). The C++ `MiniRacer::BinaryValueFactory` can +authoritatively destruct any dangling `BinaryValue` objects when it exits. + +This last especially helps with an odd scenario introduced by Python `__del__`: the +order in which Python calls `__del__` on a collection of objects is neither guaranteed +nor very predictable. When a Python program drops references to a Python `MiniRacer` +object, it's common for Python to call `_Context.__del__` before it calls +`ValHandle.__del__`, thus destroying *the container for* the value before it destroys +the value itself. The C++ side of PyMiniRacer can easily detect this scenario: First, +when destroying the `MiniRacer::Context`, it sees straggling `BinaryValue`s and destroys +them. Then, when Python asks C++ to destroy the straggling `BinaryValueHandle`s, the C++ +`mr_free_value` API sees the `MiniRacer::Context` is already gone, and ignores the +redundant request. + +The above scenario does imply a possibility for dangling pointer access: if Python calls +`_Context.__del__` then tries to read the memory addressed by the raw +`BinaryValueHandle` pointers, it will be committing a use-after-free error. We mitigate +this problem by hiding `BinaryValueHandle` within PyMiniRacer's Python code, and by +giving `ValHandle` (our Python wrapper of `BinaryValueHandle`) a reference to the +`_Context`, preventing the context from being finalized until the `ValHandle` is *also* +in Python's garbage list and on its way out. + +### Only touch (most of) the `v8::Isolate` from within the message loop + +While a `v8::Isolate` is generally a thread-aware and multi-threaded object, most of its +methods are not thread-safe. The same goes for most `v8` objects. It is, generally, only +safe to touch things belonging to a `v8::Isolate` if you hold the `v8::Locker` lock. (To +make matters more interesting, documentation about what things might be safe to do +*without* the lock is pretty scarce. You find out when your unsafe code crashes. Which, +you know, might not happen until years after you wrote the unsafe code. C++ is fun!) + +The "don't touch the `v8::Isolate` without holding the `v8::Locker`" rule is made +particularly hard to follow since we also need to run a message loop thread to service +background work in v8. That message loop, of course, itself needs the `v8::Locker`. +Unfortunately, the message loop can wait indefinitely for new work, and yet doesn't give +up the lock while doing that waiting. + +This poses a conundrum: the message loop hogs the isolate lock potentially indefinitely, +and yet other threads (i.e., Python threads) need that lock so they can poke at +`v8::Isolate`-owned objects too. + +We resolve the conundrum by leveraging part of the `v8::Isolate` itself, using a trick +similar to what NodeJS does: everything that needs to touch a `v8::Isolate` should +simply run from the `v8::Isolate`'s own message loop. If you want to run JS code, +manipulate an object, *or even delete a V8 object*, you must submit a task to the +message loop. Then nothing but the message loop itself should need to hold the +`v8::Locker` lock, because only the message loop ever touches the `v8::Isolate`. + +To make this somewhat easier we have created `MiniRacer::IsolateManager`, which provides +an easy API to submit tasks, whose callbacks accept as their first-and-only argument a +`v8::Isolate*`. Such tasks can freely work on the isolate until they exit. (Obviously, +saving a copy of the pointer and using it later would defeat the point; don't do that.) + +One odd tidbit of PyMiniRacer is that *even object destruction* has to use the above +pattern. For example, it is (probably) not safe to free a `v8::Persistent` without +holding the isolate lock, so when a non-message-loop thread needs to destroy a wrapped +V8 value, we enqueue a pretty trivial task for the message loop: +`isolate_manager->Run([persistent]() { delete persistent; })`. + +See [here](https://groups.google.com/g/v8-users/c/glG3-3pufCo) for some discussion of +this design on the v8-users mailing list. + +### If any C++ code creates an Isolate task, it's responsible for awaiting its completion before teardown + +The pattern, described above—of enqueuing all kinds of tasks for the v8 message pump, +including object destruction work—creates an interesting memory management problem for +PyMiniRacer. Such tasks typically create a reference cycle: the creator of the task +(like, say, the `MiniRacer::Context::MakeJSCallback`) bundles into the task references +to various other objects including, often, `this`. Those objects often themselves +contain references to the `MiniRacer::IsolateManager`, which transitively contains a +reference to the `v8::Isolate` and its message queue. Since the message queue contains a +reference to the task, we've just created a reference cycle! + +To avoid either use-after-free or memory leak bugs upon teardown of a +`MiniRacer::Context`, we must enforce the following rule: + +**If you call `MiniRacer::IsolateManager::Run(xyz)`, you are reponsible for ensuring +that task is done before any objects you bound into the function closure xyz (including +and especially `this`) are destroyed.** + +The most common way we ensure this is waiting on the `std::future` returned by +`MiniRacer::IsolateManager::Run(xyz)`. When that future settles, the task is done, and +it's safe to continue tearing down any references the task may hold. diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 00000000..6633603f --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,15 @@ +# Credits + +## Authors + +- Jean-Baptiste AVIAT +- Boris FELD +- Selim MENOUAR +- Nicolas VIVET + +## Contributors + +- messense +- Ben Creech + +_Why not add your name to the list?_ diff --git a/AUTHORS.rst b/AUTHORS.rst deleted file mode 100644 index e9b31730..00000000 --- a/AUTHORS.rst +++ /dev/null @@ -1,18 +0,0 @@ -======= -Credits -======= - -Authors -------- - -* Jean-Baptiste AVIAT -* Boris FELD -* Selim MENOUAR -* Nicolas VIVET - -Contributors ------------- - -* messense https://github.com/messense - -Why not add your name to the list? diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..204c086c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,263 @@ +# Contributing + +Contributions are welcome, and they are greatly appreciated! Every little bit helps, and +credit will always be given. + +## Types of Contributions + +You can contribute in many ways: + +### Report Bugs + +Report bugs at . + +If you are reporting a bug, please include: + +- Your operating system name and version. +- Any details about your local setup that might be helpful in troubleshooting. +- Detailed steps to reproduce the bug. + +### Fix Bugs + +Look through the GitHub issues for bugs. Anything tagged with "bug" is open to whoever +wants to implement it. + +## Implement Features + +Look through the GitHub issues for features. Anything tagged with "feature" is open to +whoever wants to implement it. + +## Write Documentation + +Python Mini Racer could always use more documentation, whether as part of the official +Python Mini Racer docs, in docstrings, or even on the web in blog posts, articles, and +such. + +## Submit Feedback + +The best way to send feedback is to file an issue at +. + +If you are proposing a feature: + +- Explain in detail how it would work. +- Keep the scope as narrow as possible, to make it easier to implement. +- Remember that this is a volunteer-driven project, and that contributions are welcome + :) + +## Get Started! + +Ready to contribute? Here's how to set up `PyMiniRacer` for local development. + +!!! warning + Building this package from source takes several GB of disk space and takes 1-2 hours. + +1. Do a quick scan through [the architecture guide](ARCHITECTURE.md) before diving in. + +1. Fork the `PyMiniRacer` repo on GitHub. + +1. If you plan to change C++ code you should probably install at least `clang-format` + and `clang-tidy` from [the latest stable LLVM](https://releases.llvm.org/). While + the `PyMiniRacer` build uses its own compiler (!) on most systems, our pre-commit + rules rely on the system `clang-format` and `clang-tidy`. If your versions of those + utilities do not match the ones `PyMiniRacer` uses on GitHub Actions, you may see + spurious pre-commit errors. + + If you're on a Debian-related Linux distribution using the LLVM project's standard + apt packages, note that you will likely have to override `/usr/bin/clang-format` + and `/usr/bin/clang-tidy` to point to the latest version, i.e., + `/usr/bin/clang-format-18` and `/usr/bin/clang-tidy-18`, respectively. + + You can always silence local pre-commit errors with the `-n` argument to + `git commit`. We check `pre-commit`s on every pull request using GitHub Actions, so + you can check for errors there instead. + +1. Run some of the following: + + ```sh + # Set up a virtualenv: + $ python -m venv ~/my_venv + $ . ~/my_venv/bin/activate + $ pip install pre-commit hatch hatch-mkdocs + + # Set up a local clone of your fork: + $ git clone git@github.com:your_name_here/PyMiniRacer.git + $ cd PyMiniRacer/ + $ pre-commit install # install our pre-commit hooks + + # Build and test stuff: + $ hatch run docs:serve # build the docs you're reading now! + $ hatch build # this may take 1-2 hours! + $ hatch run test:run + ``` + + You can also play with your build in the Python REPL, as follows: + + ```sh + $ hatch shell + $ python + >>> from py_mini_racer import MiniRacer + >>> mr = MiniRacer() + >>> mr.eval('6*7') + 42 + >>> exit() + $ exit + ``` + + As a shortcut for iterative development, you can skip the `hatch build` and matrix + tests, and do: + + ```sh + $ python helpers/v8_build.py [--skip-fetch] # will install a DLL into src/py_mini_racer + $ PYTHONPATH=src pytest # run the tests + $ PYTHONPATH=src python # play with the build in the REPL + >>> from py_mini_racer import MiniRacer + >>> ... + ``` + +1. Create a branch for local development:: + + ```sh + $ git checkout -b name-of-your-bugfix-or-feature + ``` + + Now you can make your changes locally. + +1. When you're done making changes, check that your changes pass the linter and the + tests, including testing other Python versions: + + ```sh + $ pre-commit run # run formatters and linters + $ hatch run docs:serve # look at the docs if you changed them! + $ hatch build # this may take 1-2 hours! + $ hatch run test:run + $ hatch run test:run-coverage # with coverage! + ``` + +1. Commit your changes and push your branch to GitHub:: + + ```sh + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + ``` + +1. (Optional) Run the GitHub Actions build workflow on your fork to ensure that all + architectures work. + +1. Submit a pull request through the GitHub website. + +## Tests + +If you want to run the tests, you need to build the package first: + +```sh + $ hatch build +``` + +Then run: + +```sh + $ hatch run test +``` + +Or for the full test matrix: + +```sh + $ hatch run test:run + $ hatch run test:run-coverage # with coverage! +``` + +## Pull Request Guidelines + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +1. If the pull request adds functionality, the docs should be updated. Put your new + functionality into a function with a docstring, and add the feature to the list in + README.md. +1. The pull request should work for the entire test matrix of Python versions + (`hatch run tests:run`). + +## Releasing `PyMiniRacer` + +Releases for `PyMiniRacer` should be done by GitHub Actions on the official project +repository. + +### Ordinary releases + +To make an ordinary release from `main`: + +1. Merge all changes into `main` on the official repository. + +1. Pick the next revision number: + + ```sh + $ git fetch + $ git ls-remote origin | grep refs/heads/release + # observe the next available release version + ``` + +1. Create a `feature/...` branch, and: + + 1. Update `HISTORY.md` with a summary of changes since the last release. + + 1. Update `src/py_mini_racer/__about__.py` with the new revision number. + + 1. Create and merge a pull request for this branch into `main`. + +1. Create a `release/...` branch: + + ```sh + $ git checkout main + $ git pull + NEXT_RELEASE=the next tag, starting with the letter v. E.g., "v0.12.1". + $ git checkout -b "release/${NEXT_RELEASE}" + $ git push --set-upstream origin "release/${NEXT_RELEASE}" + ``` + +1. Observe the build process on GitHub Actions. It should build and push docs and upload + wheels to PyPI automatically. + + !!! warning + As of this writing, the `aarch64` Linux builds are slow because they're running on + emulation. They time out on the first try (and second and third and...) after 6 + hours. If you "restart failed jobs", they will quickly catch up to where where they + left off due to [`sccache`](https://github.com/mozilla/sccache). The jobs should + *eventually* complete within the time limit. You can observe their slow progress + using the Ninja build status (e.g., `[1645/2312] CXX obj/v8_foo_bar.o`). + +### Hotfix releases + +To hotfix a prior release: + +1. Prepare the fix as a `feature` branch as normal, and merge it into `main`. + +1. Pick the next revision number. This will typically be a patch-version update on the + current release name. + +1. Create a new release branch for the hotfix: + + ```sh + BASE_RELEASE=the release you are hotfixing, starting with the letter v. E.g., "v0.12.0". + NEXT_RELEASE=the next tag, starting with the letter v. E.g., "v0.12.1". + $ git checkout "release/${BASE_RELEASE}" + $ git pull + $ git checkout -b "release/${NEXT_RELEASE}" + ``` + +1. Hotfix the commit(s) created in step #1 onto the new release branch. + +1. Create a version update commit: + + 1. Update `HISTORY.md` with a summary of changes since the last release. + + 1. Update `src/py_mini_racer/__about__.py` with the new revision number. + +1. Commit and push the new release branch to GitHub. + +1. Merge this branch into `main`. *All content on release branches should be included in + `main`.* + +1. Observe the build process on GitHub Actions. It should build and push docs and upload + wheels to PyPI automatically. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index b13bd07e..00000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,112 +0,0 @@ -============ -Contributing -============ - -Contributions are welcome, and they are greatly appreciated! Every -little bit helps, and credit will always be given. - -You can contribute in many ways: - -Types of Contributions ----------------------- - -Report Bugs -~~~~~~~~~~~ - -Report bugs at https://github.com/sqreen/PyMiniRacer/issues. - -If you are reporting a bug, please include: - -* Your operating system name and version. -* Any details about your local setup that might be helpful in troubleshooting. -* Detailed steps to reproduce the bug. - -Fix Bugs -~~~~~~~~ - -Look through the GitHub issues for bugs. Anything tagged with "bug" -is open to whoever wants to implement it. - -Implement Features -~~~~~~~~~~~~~~~~~~ - -Look through the GitHub issues for features. Anything tagged with "feature" -is open to whoever wants to implement it. - -Write Documentation -~~~~~~~~~~~~~~~~~~~ - -Python Mini Racer could always use more documentation, whether as part of the -official Python Mini Racer docs, in docstrings, or even on the web in blog posts, -articles, and such. - -Submit Feedback -~~~~~~~~~~~~~~~ - -The best way to send feedback is to file an issue at https://github.com/sqreen/PyMiniRacer/issues. - -If you are proposing a feature: - -* Explain in detail how it would work. -* Keep the scope as narrow as possible, to make it easier to implement. -* Remember that this is a volunteer-driven project, and that contributions - are welcome :) - -Get Started! ------------- - -Ready to contribute? Here's how to set up `PyMiniRacer` for local development. - -1. Fork the `PyMiniRacer` repo on GitHub. -2. Clone your fork locally:: - - $ git clone git@github.com:your_name_here/PyMiniRacer.git - -3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: - - $ mkvirtualenv PyMiniRacer - $ cd PyMiniRacer/ - $ python helpers/v8_build.py - $ python setup.py develop - -**Warning**: building this package from source takes several GB of disk space and takes ~60 minutes. - -4. Create a branch for local development:: - - $ git checkout -b name-of-your-bugfix-or-feature - - Now you can make your changes locally. - -5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: - - $ flake8 py_mini_racer tests - $ python setup.py test - $ tox - - To get flake8 and tox, just pip install them into your virtualenv. - -6. Commit your changes and push your branch to GitHub:: - - $ git add . - $ git commit -m "Your detailed description of your changes." - $ git push origin name-of-your-bugfix-or-feature - -7. Submit a pull request through the GitHub website. - -Pull Request Guidelines ------------------------ - -Before you submit a pull request, check that it meets these guidelines: - -1. The pull request should include tests. -2. If the pull request adds functionality, the docs should be updated. Put - your new functionality into a function with a docstring, and add the - feature to the list in README.rst. -3. The pull request should work for Python 2.7, at least 3.5 and for PyPy. - -Tips ----- - -To run a subset of tests:: - - $ python -m unittest tests.test_eval diff --git a/HISTORY.md b/HISTORY.md new file mode 100644 index 00000000..ed1bf0be --- /dev/null +++ b/HISTORY.md @@ -0,0 +1,282 @@ +# History + +## 0.12.4 (2024-06-16) + +- Upgrade to V8 12.6 from 12.4. + +## 0.12.3 (2024-05-25) + +- Fix potential hang if JavaScript calls a function produced by `wrap_py_function` while + we're tearing it down. + +## 0.12.2 (2024-05-20) + +- Add optional context manager and `.close()` semantics to Python `MiniRacer` class. + +- Fixed a potential hang on MiniRacer teardown if MiniRacer is executing a microtask + which loops infinitely. + +- Switch C++ side of MiniRacer to a more straightforward object lifecycle management + model. + +## 0.12.1 (2024-05-18) + +- Update to V8 12.4. This includes fixes for CVE-2024-3159, CVE-2024-3156, and + CVE-2024-2625. These vulnerabilities in V8 would impact PyMiniRacer users who are + running untrusted and adversarial JavaScript code. + +## 0.12.0 (2024-04-29) + +- Added support for installing callbacks from JS back into Python, using + MiniRacer.wrap_py_function. + +- Refactored the Python implementation into many internal files. This should mostly not + present a breaking change, except for code which reaches into internal + (`_`-prefixed) variables. + +## 0.11.1 (2024-04-08) + +- Fixed Python crash on long-running microtasks, introduced in v0.8.1 (before which + long-running microtasks would probably not run at all). + +- Fixed memory leak on the exception object reported when an `eval` times out. + +- Hardened the memory management of JS value interchange, context tracking, and + asynchronous task tracking between C++ and Python. + +- Added exhaustive typing (now with a MyPy pre-commit to verify!) + +- Added a test asserting that [the v8 sandbox](https://v8.dev/blog/sandbox) is enabled + on all platforms we build wheels for. + +## 0.11.0 (2024-04-03) + +- Added a `MutableMapping` (`dict`-like) interface for all derivatives of JS Objects, + and a `MutableSequence` (`list`-like) interface for JS Arrays. You can now use + Pythonic idioms to read and write Object properties and Array elements in Python, + including recursively (i.e., you can read Objects embedded in other objects, and + embed your own). + +- Added ability to directly call `JSFunction` objects from Python. E.g., + `mr.eval("a => a*a")(4)` parses the given number-squaring code into a function, + returns a handle to that function to Python, calls it with the number `4`, and + recieves the result of `16`. + +- Added a `JSUndefined` Python object to model JavaScript `undefined`. This is needed to + properly implement the above interface for reading Object and Array elements. + *Unfortunately, this may present a breaking change for users who assume JavaScript + `undefined` is modeled as Python `None`.* + +- Removed an old optimization for `eval` on simple no-argument function calls (i.e., + `myfunc()`). The optimization only delivered about a 17% speedup on no-op calls (and + helped relatively *less* on calls which actually did work), and for the purpose of + optimizing repeated calls to the same function, it's now redundant with extracting + and calling the function from Python, e.g., `mr.eval("myfunc")()`. + +- Hardening (meaning "fixing potential but not-yet-seen bugs") related to freeing + `BinaryValue` instances (which convey data from C++ to Python). + +- More hardening related to race conditions on teardown of the `MiniRacer` object in the + unlikely condition that `eval` operations are still executing on the C++ side, and + abandoned on the Python side, when Python attempts to garbage collect the + `MiniRacer` object. + +## 0.10.0 (2024-03-31) + +- Updated to V8 12.3 from V8 12.2 now that Chromium stable is on 12.3. + +- Added Python-side support for JS Promises. You can now return a JS Promise from code + executed by `MiniRacer.eval`, and PyMiniRacer will convert it to a Python object + which has a blocking `promise.get()` method, and also supports `await promise` in + `async` Python functions. + +- Added a `setTimeout` and `clearTimeout`. These common functions live in the Web API + standard, not the ECMAScript standard, and thus don't come with V8, but they're so + ubiquitious we now ship an implemention with `PyMiniRacer`. + +## 0.9.0 (2024-03-30) + +- Revamped JS execution model to be out-of-thread. Python/C++ interaction now happens + via callbacks. + +- Consequently, Control+C (`KeyboardInterrupt`) now interrupts JS execution. + +- Hardened C++-side thread safety model, resolving potential race conditions introduced + in v0.8.1 (but not actually reported as happening anywhere). + +- Further improved JS exception reporting; exception reports now show the offending code + where possible. + +- Introduced `timeout_sec` parameter to `eval`, `call`, and `execute` to replace the + `timeout`, which unfortunately uses milliseconds (unlike the Python standard + library). In the future we may emit deprecation warnings for use of `timeout`. + +## 0.8.1 (2024-03-23) + +- A series of C++ changes which should not impact the behavior of PyMiniRacer: +- Refactoring how we use V8 by inverting the control flow. Before we had function + evaluations which ran and drained the message loop. Now we have an always-running + message loop into which we inject function evaluations. This seems to be the + preferred way to use V8. This is not expected to cause any behavior changes (but, in + tests, makes + [microtask competion](https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API/Microtask_guide) + more consistent). +- Refactoring the C++ implementation into multiple components to make startup and + teardown logic more robust. +- Added tests for the existing fast-function-call path. +- Also, simplified Python conversion of C++ evaluation results. + +## 0.8.0 (2024-03-18) + +- General overhaul of C++ implementation to better adhere to modern best practice. This + should have no visible impact except for the following notes... +- Exposed the hard memory limit as a context-specific (as opposed to `eval`-specific) + limit, since that's how it worked all along anyway. The `max_memory` `eval` argument + still works for backwards compatibility purposes. +- Correct message type of some exceptions to `str` instead of `bytes` (they should all + be `str` now). +- Added better messages for JS parse errors. +- Added backtraces for more JS errors. +- Added some really basic Python typing. + +## 0.7.0 (2024-03-06) + +- Update V8 to 12.2 +- Drop Python 2 support +- Fix small Python 3.12 issue and add testing for Python 3.9-3.12 +- Add aarch64 support for Mac and Linux +- Revamp DLL loading to be compliant with Python 3.9-style resource loading. This may + present a small breaking change for advanced usage; the `EXTENSION_PATH` and + `EXTENSION_NAME` module variables, and `MiniRacer.v8_flags` and `MiniRacer.ext` + class variable have all been removed. +- Add support for the [ECMAScript internalization API](https://v8.dev/docs/i18n) and + thus [the ECMA `Intl` API](https://tc39.es/ecma402/) +- Use [fast startup snapshots](https://v8.dev/blog/custom-startup-snapshots) +- Switch from setuptools to Hatch +- Switch from tox to Hatch +- Switch from flake8 and isort to Hatch's wrapper of Ruff +- Switch from Sphinx to mkdocs (and hatch-mkdocs) +- Switch from unittest to pytest +- Add ARCHITECTURE.md and lots of code comments + +## 0.6.0 (2020-04-20) + +- Update V8 to 8.9 +- Optimize function calls without arguments +- Switch V8 to single threaded mode to avoid crashes after fork +- Switch to strict mode by default +- Revamp documentation + +## 0.5.0 (2020-02-25) + +- Update V8 to 8.8 + +## 0.4.0 (2020-09-22) + +- Universal wheels for Linux, Mac and Windows +- Fallback to source package for Alpine Linux + +## 0.3.0 (2020-06-29) + +- Introduce a strict mode +- Fix array conversion when size changes dynamically (CVE-2020-25489) + +## 0.2.0 (2020-03-11) + +- Support for Alpine Linux +- Avoid pip private modules in setup.py + +## 0.2.0b1 (2020-01-09) + +- Support for Windows 64 bits +- Support for Python 3.8 +- Upgrade V8 to 7.8 +- Support soft memory limits + +## 0.1.18 (2019-01-04) + +- Support memory and time limits + +## 0.1.17 (2018-19-12) + +- Upgrade libv8 +- Fix a memory leak + +## 0.1.16 (2018-07-11) + +- Add wheel for Python without PyMalloc + +## 0.1.15 (2018-06-18) + +- Add wheel for Python 3.7 + +## 0.1.14 (2018-05-25) + +- Add support for pip 10 +- Update package metadata + +## 0.1.13 (2018-03-15) + +- Add heap_stats function +- Fix issue with returned strings containing null bytes + +## 0.1.12 (2018-17-04) + +- Remove dependency to enum + +## 0.1.11 (2017-07-11) + +- Add compatibility for centos6 + +## 0.1.10 (2017-03-31) + +- Add the possibility to pass a custom JSON encoder in call. + +## 0.1.9 (2017-03-24) + +- Fix the compilation for Ubuntu 12.04 and glibc \< 2.17. + +## 0.1.8 (2017-03-02) + +- Update targets build for better compatibility with old Mac OS X and linux platforms. + +## 0.1.7 (2016-10-04) + +- Improve general performances of the JS execution. +- Add the possibility to build a different version of V8 (for example with debug + symbols). +- Fix a conflict that could happens between statically linked libraries and dynamic + ones. + +## 0.1.6 (2016-08-12) + +- Add error message when py_mini_racer sdist fails to build asking to update pip in + order to download the pre-compiled wheel instead of the source distribution. + +## 0.1.5 (2016-08-04) + +- Build py_mini_racer against a static Python. When built against a shared library + python, it doesn't work with a static Python. + +## 0.1.4 (2016-08-04) + +- Ensure JSEvalException message is converted to unicode + +## 0.1.3 (2016-08-04) + +- Fix extension loading for python3 +- Add a make target for building distributions (sdist + wheels) +- Fix eval conversion for python 3 + +## 0.1.2 (2016-08-03) + +- Fix date support +- Fix Dockerfile for generating python3 wheels + +## 0.1.1 (2016-08-02) + +- Fix sdist distribution. + +## 0.1.0 (2016-08-01) + +- First release on PyPI. diff --git a/HISTORY.rst b/HISTORY.rst deleted file mode 100644 index 82c8278f..00000000 --- a/HISTORY.rst +++ /dev/null @@ -1,125 +0,0 @@ -.. :changelog: - -History -------- - -0.6.0 (2020-04-20) -''''''''''''''''''' -* Update V8 to 8.9 -* Optimize function calls without arguments -* Switch V8 to single threaded mode to avoid crashes after fork -* Switch to strict mode by default -* Revamp documentation - -0.5.0 (2020-02-25) -''''''''''''''''''' -* Update V8 to 8.8 - -0.4.0 (2020-09-22) -''''''''''''''''''' -* Universal wheels for Linux, Mac and Windows -* Fallback to source package for Alpine Linux - -0.3.0 (2020-06-29) -''''''''''''''''''' -* Introduce a strict mode -* Fix array conversion when size changes dynamically (CVE-2020-25489) - -0.2.0 (2020-03-11) -''''''''''''''''''' -* Support for Alpine Linux -* Avoid pip private modules in setup.py - -0.2.0b1 (2020-01-09) -''''''''''''''''''''' -* Support for Windows 64 bits -* Support for Python 3.8 -* Upgrade V8 to 7.8 -* Support soft memory limits - -0.1.18 (2019-01-04) -'''''''''''''''''''' -* Support memory and time limits - -0.1.17 (2018-19-12) -'''''''''''''''''''' -* Upgrade libv8 -* Fix a memory leak - -0.1.16 (2018-07-11) -'''''''''''''''''''' -* Add wheel for Python without PyMalloc - -0.1.15 (2018-06-18) -'''''''''''''''''''' -* Add wheel for Python 3.7 - - -0.1.14 (2018-05-25) -'''''''''''''''''''' -* Add support for pip 10 -* Update package metadata - -0.1.13 (2018-03-15) -'''''''''''''''''''' -* Add heap_stats function -* Fix issue with returned strings containing null bytes - -0.1.12 (2018-17-04) -'''''''''''''''''''' -* Remove dependency to enum - -0.1.11 (2017-07-11) -'''''''''''''''''''' -* Add compatibility for centos6 - -0.1.10 (2017-03-31) -'''''''''''''''''''' -* Add the possibility to pass a custom JSON encoder in call. - -0.1.9 (2017-03-24) -''''''''''''''''''' -* Fix the compilation for Ubuntu 12.04 and glibc < 2.17. - -0.1.8 (2017-03-02) -''''''''''''''''''' -* Update targets build for better compatibility with old Mac OS X and linux platforms. - -0.1.7 (2016-10-04) -''''''''''''''''''' -* Improve general performances of the JS execution. -* Add the possibility to build a different version of V8 (for example with debug symbols). -* Fix a conflict that could happens between statically linked libraries and dynamic ones. - -0.1.6 (2016-08-12) -''''''''''''''''''' -* Add error message when py_mini_racer sdist fails to build asking to update pip in order to download the pre-compiled wheel instead of the source distribution. - -0.1.5 (2016-08-04) -''''''''''''''''''' -* Build py_mini_racer against a static Python. When built against a shared library python, it doesn't work with a static Python. - -0.1.4 (2016-08-04) -''''''''''''''''''' -* Ensure JSEvalException message is converted to unicode - -0.1.3 (2016-08-04) -''''''''''''''''''' -* Fix extension loading for python3 -* Add a make target for building distributions (sdist + wheels) -* Fix eval conversion for python 3 - -0.1.2 (2016-08-03) -''''''''''''''''''' -* Fix date support -* Fix Dockerfile for generating python3 wheels - - -0.1.1 (2016-08-02) -''''''''''''''''''' -* Fix sdist distribution. - - -0.1.0 (2016-08-01) -''''''''''''''''''' -* First release on PyPI. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index b99a907a..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,15 +0,0 @@ -include AUTHORS.rst -include CONTRIBUTING.rst -include HISTORY.rst -include LICENSE -include README.rst - -include py_mini_racer/libmini_racer.glibc.so py_mini_racer/libmini_racer.muslc.so -include py_mini_racer/libmini_racer.dylib -include py_mini_racer/mini_racer.dll - -recursive-include tests * -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] - -recursive-include docs *.rst conf.py diff --git a/Makefile b/Makefile deleted file mode 100644 index 351a3910..00000000 --- a/Makefile +++ /dev/null @@ -1,89 +0,0 @@ -.PHONY: clean-pyc clean-build docs clean -define BROWSER_PYSCRIPT -import os, webbrowser, sys -try: - from urllib import pathname2url -except: - from urllib.request import pathname2url - -webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) -endef -export BROWSER_PYSCRIPT -BROWSER := python -c "$$BROWSER_PYSCRIPT" - -help: - @echo "clean - remove all build, test, coverage and Python artifacts" - @echo "clean-build - remove build artifacts" - @echo "clean-pyc - remove Python file artifacts" - @echo "clean-test - remove test and coverage artifacts" - @echo "lint - check style with flake8" - @echo "test - run tests quickly with the default Python" - @echo "test-all - run tests on every Python version with tox" - @echo "coverage - check code coverage quickly with the default Python" - @echo "docs - generate Sphinx HTML documentation, including API docs" - @echo "release - package and upload a release" - @echo "dist - package" - @echo "install - install the package to the active Python's site-packages" - -clean: clean-build clean-pyc clean-test - -clean-build: - rm -fr build/ - rm -fr dist/ - rm -fr .eggs/ - rm -fr venv* - rm -Rf py_mini_racer/*.so - find . -name '*.egg-info' -exec rm -fr {} + - find . -name '*.egg' -exec rm -f {} + - -clean-pyc: - find . -name '*.pyc' -exec rm -f {} + - find . -name '*.pyo' -exec rm -f {} + - find . -name '*~' -exec rm -f {} + - find . -name '__pycache__' -exec rm -fr {} + - -clean-test: - rm -fr .tox/ - rm -f .coverage - rm -fr htmlcov/ - -lint: - flake8 py_mini_racer tests - -test: - python setup.py test - -test-all: - tox - -coverage: - coverage run --source py_mini_racer setup.py test - coverage report -m - coverage html - $(BROWSER) htmlcov/index.html - -docs: - rm -f docs/py_mini_racer.rst - rm -f docs/modules.rst - sphinx-apidoc -o docs/ py_mini_racer - $(MAKE) -C docs clean - $(MAKE) -C docs html - $(BROWSER) docs/_build/html/index.html - -servedocs: docs - watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . - -release: clean - python setup.py sdist upload - python setup.py bdist_wheel upload - -dist: clean - python setup.py sdist - python setup.py bdist_wheel - ls -l dist - -upload: dist - twine upload dist/* - -install: clean - python setup.py install diff --git a/README.md b/README.md new file mode 100644 index 00000000..9218ff8b --- /dev/null +++ b/README.md @@ -0,0 +1,248 @@ +[![PyPI status indicator](https://img.shields.io/pypi/v/mini_racer.svg)](https://pypi.python.org/pypi/mini_racer) +[![Github workflow status indicator](https://github.com/bpcreech/PyMiniRacer/actions/workflows/build.yml/badge.svg)](https://github.com/bpcreech/PyMiniRacer/actions/workflows/build.yml) +[![ISC License](https://img.shields.io/badge/License-ISC-blue.svg)](https://opensource.org/licenses/ISC) + +Minimal, modern embedded V8 for Python. + +![MiniRacer logo: a V8 with a very snakey 8](py_mini_racer.png) + +[Full documentation](https://bpcreech.com/PyMiniRacer/). + +## Features + +- Latest ECMAScript support +- Web Assembly support +- Unicode support +- Thread safe +- Re-usable contexts + +MiniRacer can be easily used by Django or Flask projects to minify assets, run babel or +WASM modules. + +## New home! (As of March 2024) + +PyMiniRacer was created by [Sqreen](https://github.com/sqreen), and originally lived at + with the PyPI package +[`py-mini-racer`](https://pypi.org/project/py-mini-racer/). + +As of March 2024, after a few years without updates, [I](https://bpcreech.com) have +reached out to the original Sqreen team. We agreed that I should fork PyMiniRacer, +giving it a new home at with a new PyPI +package [`mini-racer`](https://pypi.org/project/mini-racer/) (*note: no `py-`*). It now +has [a new version](https://bpcreech.com/PyMiniRacer/history/#070-2024-03-06) for the +first time since 2021! + +## Examples + +MiniRacer is straightforward to use: + +```sh + $ pip install mini-racer +``` + +and then: + +```python + $ python3 + >>> from py_mini_racer import MiniRacer + >>> ctx = MiniRacer() + >>> ctx.eval("1+1") + 2 + >>> ctx.eval("var x = {company: 'Sqreen'}; x.company") + 'Sqreen' + >>> print(ctx.eval("'❤'")) + ❤ + >>> ctx.eval("var fun = () => ({ foo: 1 });") +``` + +Variables are kept inside of a context: + +```python + >>> ctx.eval("x.company") + 'Sqreen' +``` + +You can evaluate whole scripts within JavaScript, or define and return JavaScript +function objects and call them from Python (*new in v0.11.0*): + +```python + >>> square = ctx.eval("a => a*a") + >>> square(4) + 16 +``` + +JavaScript Objects and Arrays are modeled in Python as dictionaries and lists (or, more +precisely, +[`MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping) +and +[`MutableSequence`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableSequence) +instances), respectively (*new in v0.11.0*): + +```python + >>> obj = ctx.eval("var obj = {'foo': 'bar'}; obj") + >>> obj["foo"] + 'bar' + >>> list(obj.keys()) + ['foo'] + >>> arr = ctx.eval("var arr = ['a', 'b']; arr") + >>> arr[1] + 'b' + >>> 'a' in arr + True + >>> arr.append(obj) + >>> ctx.eval("JSON.stringify(arr)") + '["a","b",{"foo":"bar"}]' +``` + +Meanwhile, `call` uses JSON to transfer data between JavaScript and Python, and converts +data in bulk: + +```python + >>> ctx.call("fun") + {'foo': 1} +``` + +Composite values are serialized using JSON. Use a custom JSON encoder when sending +non-JSON encodable parameters: + +```python + import json + + from datetime import datetime + + class CustomEncoder(json.JSONEncoder): + + def default(self, obj): + if isinstance(obj, datetime): + return obj.isoformat() + + return json.JSONEncoder.default(self, obj) +``` + +```python + >>> ctx.eval("var f = function(args) { return args; }") + >>> ctx.call("f", datetime.now(), encoder=CustomEncoder) + '2017-03-31T16:51:02.474118' +``` + +MiniRacer is ES6 capable: + +```python + >>> ctx.execute("[1,2,3].includes(5)") + False +``` + +MiniRacer supports asynchronous execution using JS `Promise` instances (*new in +v0.10.0*): + +```python + >>> promise = ctx.eval( + ... "new Promise((res, rej) => setTimeout(() => res(42), 10000))") + >>> promise.get() # blocks for 10 seconds, and then: + 42 +``` + +You can use JS `Promise` instances with Python `async` (*new in v0.10.0*): + +```python + >>> import asyncio + >>> async def demo(): + ... promise = ctx.eval( + ... "new Promise((res, rej) => setTimeout(() => res(42), 10000))") + ... return await promise + ... + >>> asyncio.run(demo()) # blocks for 10 seconds, and then: + 42 +``` + +JavaScript `null` and `undefined` are modeled in Python as `None` and `JSUndefined`, +respectively: + +```python + >>> list(ctx.eval("[undefined, null]")) + [JSUndefined, None] +``` + +You can install callbacks from JavaScript to Python (*new in v0.12.0*): + +```python + >>> async def read_file(fn): + ... with open(fn) as f: # (or aiofiles would be even better here) + ... return f.read() + ... + >>> async def get_dictionary(): + ... async with ctx.wrap_py_function(read_file) as jsfunc: + ... # "Install" our JS function on the global "this" object: + ... ctx.eval('this')['read_file'] = jsfunc + ... d = await ctx.eval('this.read_file("/usr/share/dict/words")') + ... return d.split() + ... + >>> dictionary = asyncio.run(get_dictionary()) + >>> print(dictionary[0:10]) + ['A', 'AA', 'AAA', "AA's", 'AB', 'ABC', "ABC's", 'ABCs', 'ABM', "ABM's"] +``` + +*Note that adding Python callbacks may degrade the security properties of PyMiniRacer! +See [PyMiniRacer's security goals](ARCHITECTURE.md#security-goals).* + +MiniRacer supports [the ECMA `Intl` API](https://tc39.es/ecma402/): + +```python + # Indonesian dates! + >>> ctx.eval('Intl.DateTimeFormat(["ban", "id"]).format(new Date())') + '16/3/2024' +``` + +V8 heap information can be retrieved: + +```python + >>> ctx.heap_stats() + {'total_physical_size': 1613896, + 'used_heap_size': 1512520, + 'total_heap_size': 3997696, + 'total_heap_size_executable': 3145728, + 'heap_size_limit': 1501560832} +``` + +A WASM example is available in the +[`tests`](https://github.com/bpcreech/PyMiniRacer/blob/master/tests/test_wasm.py). + +## Compatibility + +PyMiniRacer is compatible with Python 3.8-3.12 and is based on `ctypes`. + +PyMiniRacer is distributed using [wheels](https://pythonwheels.com/) on +[PyPI](https://pypi.org/). The wheels are intended to provide compatibility with: + +| OS | x86_64 | aarch64 | +| ------------------------------- | ------ | ------- | +| macOS ≥ 10.9 | ✓ | ✓ | +| Windows ≥ 10 | ✓ | ✖ | +| Ubuntu ≥ 20.04 | ✓ | ✓ | +| Debian ≥ 11 | ✓ | ✓ | +| RHEL ≥ 9 | ✓ | ✓ | +| other Linuxes with glibc ≥ 2.31 | ✓ | ✓ | +| Alpine ≥ 3.19 | ✓ | ✓ | +| other Linux with musl ≥ 1.2 | ✓ | ✓ | + +If you have a up-to-date pip and it doesn't use a wheel, you might have an environment +for which no wheel is built. Please open an issue. + +## Developing and releasing PyMiniRacer + +See [the contribution guide](CONTRIBUTING.md). + +## Credits + +Built with love by [Sqreen](https://www.sqreen.com). + +PyMiniRacer launch was described in +[`this blog post`](https://web.archive.org/web/20230526172627/https://blog.sqreen.com/embedding-javascript-into-python/). + +PyMiniRacer is inspired by [mini_racer](https://github.com/SamSaffron/mini_racer), built +for the Ruby world by Sam Saffron. + +In 2024, PyMiniRacer was revived, and adopted by [Ben Creech](https://bpcreech.com). +Upon discussion with the original Sqreen authors, we decided to re-launch PyMiniRacer as +a fork under and +. diff --git a/README.rst b/README.rst deleted file mode 100644 index 520191ad..00000000 --- a/README.rst +++ /dev/null @@ -1,212 +0,0 @@ -.. image:: https://img.shields.io/pypi/v/py_mini_racer.svg - :target: https://pypi.python.org/pypi/py_mini_racer - -.. image:: https://github.com/sqreen/PyMiniRacer/actions/workflows/build.yml/badge.svg - :target: https://github.com/sqreen/PyMiniRacer/actions/workflows/build.yml - -.. image:: https://img.shields.io/badge/License-ISC-blue.svg - :target: https://opensource.org/licenses/ISC - -Minimal, modern embedded V8 for Python. - -.. image:: data/py_mini_racer.png - :align: center - -Features --------- - -* Latest ECMAScript support -* Web Assembly support -* Unicode support -* Thread safe -* Re-usable contexts - -MiniRacer can be easily used by Django or Flask projects to minify assets, run -babel or WASM modules. - -Examples --------- - -MiniRacer is straightforward to use: - -.. code-block:: python - - >>> from py_mini_racer import MiniRacer - >>> ctx = MiniRacer() - >>> ctx.eval("1+1") - 2 - >>> ctx.eval("var x = {company: 'Sqreen'}; x.company") - 'Sqreen' - >>> print(ctx.eval("'\N{HEAVY BLACK HEART}'")) - ❤ - >>> ctx.eval("var fun = () => ({ foo: 1 });") - -Variables are kept inside of a context: - -.. code-block:: python - - >>> ctx.eval("x.company") - 'Sqreen' - - -While ``eval`` only supports returning primitive data types such as -strings, ``call`` supports returning composite types such as objects: - -.. code-block:: python - - >>> ctx.call("fun") - {'foo': 1} - - -Composite values are serialized using JSON. -Use a custom JSON encoder when sending non-JSON encodable parameters: - -.. code-block:: python - - import json - - from datetime import datetime - - class CustomEncoder(json.JSONEncoder): - - def default(self, obj): - if isinstance(obj, datetime): - return obj.isoformat() - - return json.JSONEncoder.default(self, obj) - - -.. code-block:: python - - >>> ctx.eval("var f = function(args) { return args; }") - >>> ctx.call("f", datetime.now(), encoder=CustomEncoder) - '2017-03-31T16:51:02.474118' - - -MiniRacer is ES6 capable: - -.. code-block:: python - - >>> ctx.execute("[1,2,3].includes(5)") - False - -V8 heap information can be retrieved: - -.. code-block:: python - - >>> ctx.heap_stats() - {'total_physical_size': 1613896, - 'used_heap_size': 1512520, - 'total_heap_size': 3997696, - 'total_heap_size_executable': 3145728, - 'heap_size_limit': 1501560832} - - -A WASM example is available in the `tests`_. - -.. _`tests`: https://github.com/sqreen/PyMiniRacer/blob/master/tests/test_wasm.py - - -Compatibility -------------- - -PyMiniRacer is compatible with Python 2 & 3 and based on ctypes. - -The binary builds have been tested on x86_64 with: - -* macOS >= 10.13 -* Ubuntu >= 16.04 -* Debian >= 9 -* CentOS >= 7 -* Alpine >= 3.11 -* Windows 10 - -It should work on any Linux with a libc >= 2.12 and a wheel compatible pip (>= 8.1). - -If you're running Alpine Linux, you may need to install required dependencies manually using the following command: - -.. code-block:: bash - - $ apk add libgcc libstdc++ - -If you have a up-to-date pip and it doesn't use a wheel, you might have an environment for which no wheel is built. Please open an issue. - -Installation ------------- - -We built Python wheels (prebuilt binaries) for macOS 64 bits, Linux 64 bits and Windows 64 bits. - -.. code:: bash - - $ pip install py-mini-racer - -Build ------ - -**Warning**: building this package from source takes several GB of disk space and takes ~60 minutes. - -First check that your current Python executable is version 2.7. This is required -by the V8 build system. - -.. code:: bash - - $ python --version - Python 2.7.16 - -You can build the extension with the following command: - -.. code:: bash - - $ python helpers/v8_build.py - -You can generate a wheel for whatever Python version with the command: - -.. code:: bash - - $ python3 helpers/build_package.py wheel dist - -It will then build V8, the extension, and generates a wheel for your current -Python version. The V8 builds are cached in the ``py_mini_racer/extension/v8/`` -directory. - -Notes for building on macOS -''''''''''''''''''''''''''' - -The legacy Python binary builds (OSX 10.6) need to be downloaded from: - https://www.python.org/downloads/ - -They will allow to build a wheel compatible with former OSX versions. - -Tests ------ - -If you want to run the tests, you need to build the extension first, first install pytest: - -.. code-block:: bash - - $ python -m pip install pytest - -Then launch: - -.. code:: bash - - $ python -m pytest tests - -Credits -------- - -Built with love by Sqreen_. - -.. _Sqreen: https://www.sqreen.com - -PyMiniRacer launch was described in `this blog post`_. - -.. _`this blog post`: https://blog.sqreen.com/embedding-javascript-into-python/ - -PyMiniRacer is inspired by mini_racer_, built for the Ruby world by Sam Saffron. - -.. _`mini_racer`: https://github.com/SamSaffron/mini_racer - -`Cookiecutter-pypackage`_ was used as this package skeleton. - -.. _`Cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage diff --git a/data/favicon.ico b/data/favicon.ico new file mode 100644 index 00000000..40eec185 Binary files /dev/null and b/data/favicon.ico differ diff --git a/data/py_mini_racer.png b/data/py_mini_racer.png deleted file mode 100644 index 7654b211..00000000 Binary files a/data/py_mini_racer.png and /dev/null differ diff --git a/data/py_mini_racer.png b/data/py_mini_racer.png new file mode 120000 index 00000000..8d99473f --- /dev/null +++ b/data/py_mini_racer.png @@ -0,0 +1 @@ +../py_mini_racer.png \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index fb12ca59..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,177 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/py_mini_racer.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/py_mini_racer.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/py_mini_racer" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/py_mini_racer" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 00000000..60a8d8fd --- /dev/null +++ b/docs/api.md @@ -0,0 +1,3 @@ +# API Reference + +::: py_mini_racer diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index 83492fe7..00000000 --- a/docs/api.rst +++ /dev/null @@ -1,5 +0,0 @@ -API Reference -------------- - -.. autoclass:: py_mini_racer.MiniRacer - :members: eval, execute, call, set_soft_memory_limit, was_soft_memory_limit_reached, low_memory_notification, heap_stats, v8_version diff --git a/docs/architecture.md b/docs/architecture.md new file mode 120000 index 00000000..6763c822 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1 @@ +../ARCHITECTURE.md \ No newline at end of file diff --git a/docs/authors.md b/docs/authors.md new file mode 120000 index 00000000..3234d6e0 --- /dev/null +++ b/docs/authors.md @@ -0,0 +1 @@ +../AUTHORS.md \ No newline at end of file diff --git a/docs/authors.rst b/docs/authors.rst deleted file mode 100644 index e122f914..00000000 --- a/docs/authors.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../AUTHORS.rst diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100755 index bc6b3e21..00000000 --- a/docs/conf.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# py_mini_racer documentation build configuration file, created by -# sphinx-quickstart on Tue Jul 9 22:26:36 2013. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os - -# If extensions (or modules to document with autodoc) are in another -# directory, add these directories to sys.path here. If the directory is -# relative to the documentation root, use os.path.abspath to make it -# absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# Get the project root dir, which is the parent dir of this -cwd = os.getcwd() -project_root = os.path.dirname(cwd) - -# Insert the project root dir as the first element in the PYTHONPATH. -# This lets us ensure that the source package is imported, and that its -# version is used. -sys.path.insert(0, project_root) - -from py_mini_racer import __about__ - -# -- General configuration --------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] - -autodoc_member_order = 'bysource' - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'PyMiniRacer' -copyright = u'2019, Sqreen' - -# The version info for the project you're documenting, acts as replacement -# for |version| and |release|, also used in various other places throughout -# the built documents. -# -# The short X.Y version. -version = __about__.__version__ -# The full version, including alpha/beta/rc tags. -release = __about__.__version__ - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to -# some non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built -# documents. -#keep_warnings = False - - -# -- Options for HTML output ------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'alabaster' - -# Theme options are theme-specific and customize the look and feel of a -# theme further. For a list of options available for each theme, see the -# documentation. -html_theme_options = { - 'logo_name': False, - 'github_user': 'sqreen', - 'github_repo': 'PyMiniRacer', - 'github_button': True, -} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as -# html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the -# top of the sidebar. -html_logo = 'data/py_mini_racer.png' - -# The name of an image file (within the static path) to use as favicon -# of the docs. This file should be a Windows icon file (.ico) being -# 16x16 or 32x32 pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) -# here, relative to this directory. They are copied after the builtin -# static files, so a file named "default.css" will overwrite the builtin -# "default.css". -html_static_path = ['data'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page -# bottom, using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - - -# Additional templates that should be rendered to pages, maps page names -# to template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. -# Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. -# Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages -# will contain a tag referring to it. The value of this option -# must be the base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'py_mini_racerdoc' - - -# -- Options for LaTeX output ------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - #'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - #'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass -# [howto/manual]). -latex_documents = [ - ('index', 'py_mini_racer.tex', - u'PyMiniRacer Documentation', - u'Boris FELD', 'manual'), -] - -# The name of an image file (relative to this directory) to place at -# the top of the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings -# are parts, not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output ------------------------------------ - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'py_mini_racer', - u'PyMiniRacer Documentation', - [u'Sqreen'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ---------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'py_mini_racer', - u'PyMiniRacer Documentation', - u'Sqreen', - 'py_mini_racer', - 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False diff --git a/docs/contributing.md b/docs/contributing.md new file mode 120000 index 00000000..44fcc634 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1 @@ +../CONTRIBUTING.md \ No newline at end of file diff --git a/docs/contributing.rst b/docs/contributing.rst deleted file mode 100644 index e582053e..00000000 --- a/docs/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../CONTRIBUTING.rst diff --git a/docs/history.md b/docs/history.md new file mode 120000 index 00000000..a5333ae4 --- /dev/null +++ b/docs/history.md @@ -0,0 +1 @@ +../HISTORY.md \ No newline at end of file diff --git a/docs/history.rst b/docs/history.rst deleted file mode 100644 index 25064996..00000000 --- a/docs/history.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../HISTORY.rst diff --git a/docs/index.md b/docs/index.md new file mode 120000 index 00000000..32d46ee8 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index db166057..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. include:: readme.rst - -API Reference -------------- - -.. toctree:: - :maxdepth: 2 - - api - -Additional Information ----------------------- - -.. toctree:: - :maxdepth: 2 - - contributing - authors - history diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 42cd866d..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,242 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\py_mini_racer.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\py_mini_racer.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end diff --git a/docs/readme.rst b/docs/readme.rst deleted file mode 100644 index 8fb07a4e..00000000 --- a/docs/readme.rst +++ /dev/null @@ -1,3 +0,0 @@ -.. include:: ../README.rst - :start-after: center - :end-before: Credits diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 00000000..91076fea --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,39 @@ +from os.path import dirname +from sys import path as syspath +from typing import Iterable + +from hatchling.builders.config import BuilderConfig +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from packaging.tags import Tag + +# During env initialization the PYTHONPATH doesn't include our helpers, so +# give it some help: +syspath.append(dirname(__file__)) +from helpers.v8_build import build_v8, clean_v8, get_platform_tag + + +class V8BuildHook(BuildHookInterface[BuilderConfig]): + def clean(self, versions: Iterable[str]) -> None: + del versions + + clean_v8("src/py_mini_racer") + + def initialize(self, version: str, build_data): + del version + + artifacts = build_v8(out_path="src/py_mini_racer") + + build_data.setdefault("force_include", {}).update(artifacts) + + # From https://stackoverflow.com/questions/76450587/python-wheel-that-includes-shared-library-is-built-as-pure-python-platform-indep + # We have to tell Hatch that we're building a non-pure-Python wheel, even + # though there are no extension modules (instead, there is native code in a + # dynamic-link library in the package): + build_data["pure_python"] = False + + # Because we aren't building an extension module (just pure Python and a + # Python-independent DLL), any single wheel we create is broadly compatible + # with different Python interpreters. Just mark py3 with any ABI: + tag = Tag("py3", "none", get_platform_tag()) + + build_data["tag"] = str(tag) diff --git a/py_mini_racer/extension/__init__.py b/helpers/__init__.py similarity index 100% rename from py_mini_racer/extension/__init__.py rename to helpers/__init__.py diff --git a/helpers/babel.py b/helpers/babel.py index 7cd76081..3aa8cbed 100644 --- a/helpers/babel.py +++ b/helpers/babel.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python -""" Transform the input stream using babel.transform """ +"""Transform the input stream using babel.transform""" + import os import sys @@ -7,35 +7,35 @@ def babel_transform(es_string): - """ Transform the provided string using babel.transform """ + """Transform the provided string using babel.transform""" - path_to_babel = os.path.join(os.path.dirname(__file__), '..', 'tests', - 'fixtures', 'babel.js') + path_to_babel = os.path.join( + os.path.dirname(__file__), "..", "tests", "fixtures", "babel.js" + ) - babel_source = open(path_to_babel, "r").read() + with open(path_to_babel) as f: + babel_source = f.read() # Initializes PyMiniRacer ctx = py_mini_racer.MiniRacer() # Parse babel - ctx.eval("""var self = this; %s """ % babel_source) + ctx.eval(f"var self = this; {babel_source}" "") # Transform stuff :) - val = "babel.transform(`%s`)['code']" % es_string - res = ctx.eval(val) - return res - + val = f"babel.transform(`{es_string}`)['code']" + return ctx.eval(val) -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) != 1: name = sys.argv[0] - sys.stderr.write("Usage: cat es6file.js | %s\n" % name) - sys.stderr.write("Example: echo [1,2,3].map(n => n + 1); | %s\n" % name) + sys.stderr.write(f"Usage: cat es6file.js | {name}\n") + sys.stderr.write(f"Example: echo [1,2,3].map(n => n + 1); | {name}\n") sys.exit(-1) es6_data = sys.stdin.read() res = babel_transform(es6_data) - print(res) + sys.stdout.write(res) diff --git a/helpers/build_package.py b/helpers/build_package.py deleted file mode 100644 index 307d45f9..00000000 --- a/helpers/build_package.py +++ /dev/null @@ -1,56 +0,0 @@ -import glob -import os -import shutil -import sys - -from setuptools.build_meta import ( - build_sdist as setuptools_build_sdist, - build_wheel as setuptools_build_wheel, - get_requires_for_build_sdist, - get_requires_for_build_wheel, - prepare_metadata_for_build_wheel, -) -from v8_build import build_v8 - - -def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): - config_settings = config_settings or {} - # Our wheel is compatible both with python 2 & python 3 - config_settings["--global-option"] = options = ["--python-tag", "py2.py3"] - # Clean previous version of the lib - for pattern in ("py_mini_racer/*.so", "py_mini_racer/*.dylib", "py_mini_racer/*.dll"): - for filename in glob.glob(pattern): - print("removing {}".format(filename)) - os.unlink(filename) - # Build V8 - build_v8("py_mini_racer_shared_lib") - # Build the wheel - if os.name == "posix" and sys.platform == "darwin": - shutil.copyfile("py_mini_racer/extension/out/libmini_racer.dylib", "py_mini_racer/libmini_racer.dylib") - options.extend(["--plat-name", "macosx_10_10_x86_64"]) - elif sys.platform == "win32": - shutil.copyfile("py_mini_racer/extension/out/mini_racer.dll", "py_mini_racer/mini_racer.dll") - options.extend(["--plat-name", "win_amd64"]) - else: - shutil.copyfile("py_mini_racer/extension/out/libmini_racer.so", "py_mini_racer/libmini_racer.glibc.so") - options.extend(["--plat-name", "manylinux1_x86_64"]) - return setuptools_build_wheel(wheel_directory, config_settings=config_settings, metadata_directory=metadata_directory) - - -def build_sdist(sdist_directory, config_settings=None): - return setuptools_build_sdist(sdist_directory) - - -__all__ = [ - "get_requires_for_build_wheel", - "get_requires_for_build_sdist", - "prepare_metadata_for_build_wheel", - "build_wheel", - "build_sdist", -] - -if __name__ == "__main__": - if sys.argv[1] == "wheel": - build_wheel(sys.argv[2]) - else: - build_sdist(sys.argv[2]) diff --git a/helpers/no-aarch64-linux-gnu-target.patch b/helpers/no-aarch64-linux-gnu-target.patch new file mode 100644 index 00000000..22040f24 --- /dev/null +++ b/helpers/no-aarch64-linux-gnu-target.patch @@ -0,0 +1,13 @@ +--- build/config/compiler/BUILD.gn.orig 2024-03-04 23:25:24.939930259 -0500 ++++ build/config/compiler/BUILD.gn 2024-03-04 23:25:54.208173956 -0500 +@@ -1191,8 +1191,8 @@ + } else if (current_cpu == "arm64") { + if (is_clang && !is_android && !is_nacl && !is_fuchsia && + !(is_chromeos_lacros && is_chromeos_device)) { +- cflags += [ "--target=aarch64-linux-gnu" ] +- ldflags += [ "--target=aarch64-linux-gnu" ] ++ # cflags += [ "--target=aarch64-linux-gnu" ] ++ # ldflags += [ "--target=aarch64-linux-gnu" ] + } + } else if (current_cpu == "mipsel" && !is_nacl) { + ldflags += [ "-Wl,--hash-style=sysv" ] diff --git a/helpers/split-threshold-for-reg-with-hint.patch b/helpers/split-threshold-for-reg-with-hint.patch new file mode 100644 index 00000000..78baae19 --- /dev/null +++ b/helpers/split-threshold-for-reg-with-hint.patch @@ -0,0 +1,11 @@ +--- build/config/compiler/BUILD.gn.orig 2024-06-15 12:35:34.335313711 -0400 ++++ build/config/compiler/BUILD.gn 2024-06-15 12:36:00.967346881 -0400 +@@ -571,7 +571,7 @@ + # TODO(crbug.com/40283598): This causes binary size growth and potentially + # other problems. + # TODO(crbug.com/40284925): This isn't supported by Cronet's mainline llvm version. +- if (default_toolchain != "//build/toolchain/cros:target" && ++ if (false && default_toolchain != "//build/toolchain/cros:target" && + !llvm_android_mainline) { + cflags += [ + "-mllvm", diff --git a/helpers/v8_build.py b/helpers/v8_build.py index 17b7f55e..b8edfa7b 100644 --- a/helpers/v8_build.py +++ b/helpers/v8_build.py @@ -1,319 +1,467 @@ -# -*- coding: utf-8 -*-" -import argparse -import errno -import glob -import json -import logging -import os -import os.path -import subprocess -import sys -from contextlib import contextmanager - -logging.basicConfig() -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.DEBUG) -ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) -V8_VERSION = "branch-heads/8.9" +from argparse import ArgumentParser +from errno import EEXIST +from functools import lru_cache +from logging import DEBUG, basicConfig, getLogger +from os import environ, makedirs, pathsep, remove, symlink, unlink +from os.path import abspath, dirname, exists, isdir, isfile +from os.path import join as pathjoin +from platform import machine +from re import match +from shlex import join as shlexjoin +from shutil import copyfile, rmtree +from subprocess import check_call +from sys import executable, platform + +from packaging.tags import platform_tags + +basicConfig() +LOGGER = getLogger(__name__) +LOGGER.setLevel(DEBUG) +ROOT_DIR = dirname(abspath(__file__)) +V8_VERSION = "branch-heads/12.6" def local_path(path="."): - """ Return path relative to this file - """ - return os.path.abspath(os.path.join(ROOT_DIR, path)) + """Return path relative to this file.""" + return abspath(pathjoin(ROOT_DIR, path)) -PATCHES_PATH = local_path('../patches') +@lru_cache(maxsize=None) +def is_win(): + return platform.startswith("win") -def call(cmd): - LOGGER.debug("Calling: '%s' from working directory %s", cmd, os.getcwd()) - current_env = os.environ - depot_tools_env = os.pathsep.join([local_path("../py_mini_racer/extension/depot_tools"), os.environ['PATH']]) - current_env['PATH'] = depot_tools_env - current_env['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0' - return subprocess.check_call(cmd, shell=True, env=current_env) +@lru_cache(maxsize=None) +def is_linux(): + return platform == "linux" -@contextmanager -def chdir(new_path, make=False): - old_path = os.getcwd() +@lru_cache(maxsize=None) +def is_mac(): + return platform == "darwin" - if make is True: - try: - os.mkdir(new_path) - except OSError: - pass - try: - yield os.chdir(new_path) - finally: - os.chdir(old_path) +class UnknownArchError(RuntimeError): + def __init__(self, arch): + super().__init__(f"Unknown arch {arch!r}") -def install_depot_tools(): - if not os.path.isdir(local_path("../py_mini_racer/extension/depot_tools")): - LOGGER.debug("Cloning depot tools") - call("git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git {}".format(local_path("../py_mini_racer/extension/depot_tools"))) - else: - LOGGER.debug("Using already cloned depot tools") +@lru_cache(maxsize=None) +def get_v8_target_cpu(): + m = machine().lower() + if m in ("arm64", "aarch64"): + return "arm64" + if m == "arm": + return "arm" + if (not m) or (match("(x|i[3-6]?)86$", m) is not None): + return "ia32" + if m in ("x86_64", "amd64"): + return "x64" + if m == "s390x": + return "s390x" + if m == "ppc64": + return "ppc64" + raise UnknownArchError(m) -def prepare_workdir(): - directories = ["build", "build_overrides", "buildtools", "testing", - "third_party", "tools"] - with chdir(local_path("../py_mini_racer/extension")): - for item in directories: - if not os.path.exists(item): - symlink_force(os.path.join("v8", item), item) +@lru_cache(maxsize=None) +def get_dll_filename(): + if is_mac(): + return "libmini_racer.dylib" -def ensure_v8_src(revision): - """ Ensure that v8 src are presents and up-to-date - """ - path = local_path("../py_mini_racer/extension") + if is_win(): + return "mini_racer.dll" - if not os.path.isfile(local_path("../py_mini_racer/extension/.gclient")): - fetch_v8(path) - else: - update_v8(path) + return "libmini_racer.so" - checkout_v8_version(local_path("../py_mini_racer/extension/v8"), revision) - dependencies_sync(path) +@lru_cache(maxsize=None) +def get_data_files_list(): + """List the files which v8 builds and then needs at runtime.""" -def fetch_v8(path): - """ Fetch v8 - """ - with chdir(os.path.abspath(path), make=True): - call("fetch --nohooks v8") + return ( + # V8 i18n data: + "icudtl.dat", + # V8 fast-startup snapshot; a dump of the heap after loading built-in JS + # modules: + "snapshot_blob.bin", + # And obviously, the V8 build itself: + get_dll_filename(), + ) -def update_v8(path): - """ Update v8 repository - """ - with chdir(path): - call("gclient fetch") +@lru_cache(maxsize=None) +def is_musl(): + # Alpine uses musl for libc, instead of glibc. This breaks many assumptions in the + # V8 build, so we have to reconfigure various things when running on musl libc. + # Determining if we're on musl (or Alpine) is surprisingly complicated; the best + # way seems to be to check the dynamic linker ependencies of the current Python + # executable for musl! packaging.tags.platform_tags (which is used by pip et al) + # does this for us: + return any("musllinux" in t for t in platform_tags()) -def checkout_v8_version(path, revision): - """ Ensure that we have the right version +@lru_cache(maxsize=None) +def get_platform_tag(): + """Return a pip platform tag indicating compatibility of the mini_racer binary. + + See https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/. """ - with chdir(path): - call("git checkout {} -- .".format(revision)) + if is_mac(): + # pip seems finicky about platform tags with larger macos versions, so just + # tell arm64 is 11.0 and everything is is 10.9: + if get_v8_target_cpu() == "arm64": + return "macosx_11_0_arm64" -def dependencies_sync(path): - """ Sync v8 build dependencies - """ - with chdir(path): - call("gclient sync") + return "macosx_10_9_x86_64" + # return the first, meaning the most-specific, platform tag: + return next(platform_tags()) -def run_hooks(path): - """ Run v8 build hooks - """ - with chdir(path): - call("gclient runhooks") - - -def gen_makefiles(build_path, no_sysroot=False): - with chdir(local_path("../py_mini_racer/extension")): - build_path = local_path(build_path) - if not os.path.exists(build_path): - os.makedirs(build_path) - LOGGER.debug("Writing args.gn in %s", build_path) - with open(os.path.join(build_path, "args.gn"), "w") as f: - opts = { - "proprietary_codecs": "false", - "toolkit_views": "false", - "use_aura": "false", - "use_dbus": "false", - "use_gio": "false", - "use_glib": "false", - "use_ozone": "false", - "use_udev": "false", - "is_desktop_linux": "false", - - "is_cfi": "false", - "is_debug": "false", - "is_component_build": "false", - - "symbol_level": "0", - "strip_debug_info": "true", - "treat_warnings_as_errors": "true", - - "v8_monolithic": "false", - "v8_use_external_startup_data": "false", - "v8_enable_i18n_support": "false", - - "v8_untrusted_code_mitigations": "false", - # See https://v8.dev/docs/untrusted-code-mitigations - - # See cc_wrapper - "clang_use_chrome_plugins": "false", - } - if no_sysroot: - opts.update({ - "treat_warnings_as_errors": "false", - "use_sysroot": "false", - "clang_use_chrome_plugins": "false", - "clang_base_path": "\"/usr\"", - "use_custom_libcxx": "false", - "use_gold": "true", - "use_lld": "false", - }) - sccache = os.environ.get('SCCACHE') - if sccache is not None: - opts["cc_wrapper"] = json.dumps(sccache) - f.write("# This file is auto generated by v8_build.py") - f.write("\n".join("{}={}".format(a, b) for (a, b) in opts.items())) - f.write("\n") - extra_args = os.getenv("GN_ARGS") - if extra_args: - f.write("\n".join(extra_args.split())) - f.write("\n") - call("gn gen {}".format(local_path(build_path))) - - -def make(build_path, target, cmd_prefix=""): - """ Create a release of v8 - """ - with chdir(local_path("../py_mini_racer/extension")): - call("{} ninja -vv -C {} {}".format(cmd_prefix, local_path(build_path), target)) +@lru_cache(maxsize=None) +def get_workspace_path(): + return local_path(pathjoin("..", "v8_workspace")) -def patch_v8(): - """ Apply patch on v8 - """ - path = local_path("../py_mini_racer/extension/v8") - patches_paths = PATCHES_PATH - apply_patches(path, patches_paths) +@lru_cache(maxsize=None) +def get_depot_tools_path(): + return pathjoin(get_workspace_path(), "depot_tools") -def symlink_force(target, link_name): - LOGGER.debug("Creating symlink to %s on %s", target, link_name) - if sys.platform == "win32": - call(["mklink", "/d", os.path.abspath(link_name), os.path.abspath(target)]) + +@lru_cache(maxsize=None) +def get_v8_path(): + return pathjoin(get_workspace_path(), "v8") + + +def unlink_if_exists(f): + if exists(f): + unlink(f) + + +def run(*args, cwd, depot_tools_first=True): + LOGGER.debug("Calling: '%s' from working directory %s", shlexjoin(args), cwd) + env = environ.copy() + + if depot_tools_first: + env["PATH"] = pathsep.join([get_depot_tools_path(), environ["PATH"]]) + else: + env["PATH"] = pathsep.join([environ["PATH"], get_depot_tools_path()]) + + env["DEPOT_TOOLS_WIN_TOOLCHAIN"] = "0" + # vpython is V8's Python environment manager; it downloads Python binaries + # dynamically. This doesn't work on Alpine (because it downloads a glibc binary, + # but needs a musl binary), so let's just disable it on all environments: + env["VPYTHON_BYPASS"] = "manually managed python not supported by chrome operations" + # Goma is a remote build system which we aren't using. depot_tools/autoninja.py + # tries to run the goma client, which is checked into depot_tools as a glibc binary. + # This fails on musl (on Alpine), so let's just disable the thing: + env["GOMA_DISABLED"] = "1" + + return check_call(args, env=env, cwd=cwd) + + +def ensure_depot_tools(): + if isdir(get_depot_tools_path()): + LOGGER.debug("Using already cloned depot tools") + return + + LOGGER.debug("Cloning depot tools") + makedirs(f"{get_workspace_path()}", exist_ok=True) + run( + "git", + "clone", + "https://chromium.googlesource.com/chromium/tools/depot_tools.git", + cwd=get_workspace_path(), + ) + + # depot_tools will auto-update when we run various commands. This creates extra + # dependencies, e.g., on goma (which has trouble running on Alpine due to musl). + # We just created a fresh single-use depot_tools checkout. There is no reason to + # update it, so let's just disable that functionality: + open(pathjoin(get_depot_tools_path(), ".disable_auto_update"), "w").close() + + if is_win(): + # Create git.bit and maybe other shortcuts used by the Windows V8 build tools: + run( + pathjoin(get_depot_tools_path(), "bootstrap", "win_tools.bat"), + cwd=get_depot_tools_path(), + ) + + +def ensure_v8_src(revision): + """Ensure that v8 src are present and up-to-date.""" + + # We create our own .gclient config instead of creating it via fetch.py so we can + # control (non-)installation of a sysroot. + gclient_file = pathjoin(get_workspace_path(), ".gclient") + if not isfile(gclient_file): + makedirs(get_workspace_path(), exist_ok=True) + if is_musl(): + # Prevent fetching of a useless Debian sysroot on Alpine. + # We disable use of the sysroot below (see "use_sysroot"), so this is just + # an optimization to preempt the download. + # (Note that "musl" is not a valid OS in the depot_tools deps system; + # "musl" here is just a placeholder to mean "*not* the thing you think is + # called 'linux'".) + # Syntax from https://source.chromium.org/chromium/chromium/src/+/main:docs/ios/running_against_tot_webkit.md + target_os = """\ +target_os = ["musl"] +target_os_only = "True" +""" + else: + target_os = "" + + with open(gclient_file, "w") as f: + f.write( + f"""\ +solutions = [ + {{ "name" : "v8", + "url" : "https://chromium.googlesource.com/v8/v8.git", + "deps_file" : "DEPS", + "managed" : False, + "custom_deps" : {{}}, + "custom_vars": {{}}, + }}, +] +{target_os}\ +""" + ) + + run( + executable, + pathjoin(get_depot_tools_path(), "gclient.py"), + "sync", + "--revision", + f"v8@{revision}", + cwd=get_workspace_path(), + ) + + ensure_symlink( + local_path(pathjoin("..", "src", "v8_py_frontend")), + pathjoin(get_v8_path(), "custom_deps", "mini_racer"), + ) + + +def apply_patch(path, patch_filename): + applied_patches_filename = local_path(".applied_patches") + + if not exists(applied_patches_filename): + open(applied_patches_filename, "w").close() + + with open(applied_patches_filename, "r+") as f: + applied_patches = set(f.read().splitlines()) + + if patch_filename in applied_patches: + return + + run( + "patch", + "-p0", + "-i", + patch_filename, + cwd=path, + ) + + f.write(patch_filename + "\n") + + +def run_build(build_dir): + """Run the actual v8 build.""" + + # As explained in the design principles in ARCHITECTURE.md, we want to reduce the + # surface area of the V8 build system which PyMiniRacer depends upon. To accomodate + # that goal, we run with as few non-default build options as possible. + + # The standard developer guide for V8 suggests we use the v8/tools/dev/v8gen.py + # tool to both generate args.gn, and run gn to generate Ninja build files. + + # Unfortunately, v8/tools/dev/v8gen.py is unhappy about being run on non-amd64 + # architecture (it seems to think we're always cross-compiling from amd64). Thus + # we reproduce what it does, which for our simple case devolves to just generating + # args.gn with minimal arguments, and running "gn gen ...". + + opts = { + # These following settings are based those found for the "x64.release" + # configuration. This can be verified by running: + # tools/mb/mb.py lookup -b x64.release -m developer_default + # ... from within the v8 directory. + "dcheck_always_on": "false", + "is_debug": "false", + "target_cpu": f'"{get_v8_target_cpu()}"', + "v8_target_cpu": f'"{get_v8_target_cpu()}"', + # We sneak our C++ frontend into V8 as a symlinked "custom_dep" so + # that we can reuse the V8 build system to make our dynamic link + # library: + "v8_custom_deps": '"//custom_deps/mini_racer"', + } + + if (is_linux() and get_v8_target_cpu() == "arm64") or is_musl(): + # The V8 build process includes its own clang binary, but not for aarch64 on + # Linux glibc, and not for Alpine (musl) at all. + # Per tools/dev/gm.py, use the the system clang instead: + opts["clang_base_path"] = '"/usr"' + + opts["clang_use_chrome_plugins"] = "false" + # Because we use a different clang, more warnings pop up. Ignore them: + opts["treat_warnings_as_errors"] = "false" + + # V8 currently uses a clang flag -split-threshold-for-reg-with-hint=0 which + # doen't exist on Alpine's mainline llvm yet. Disable it: + if is_musl(): + apply_patch( + get_v8_path(), + local_path("split-threshold-for-reg-with-hint.patch"), + ) + + if is_musl() and get_v8_target_cpu() == "arm64": + # The V8 build unhelpfully sets the clang flag --target=aarch64-linux-gnu + # on musl. The --target flag is useful when we're cross-compiling (which we're + # not) and we aren't on aarch64-linux-gnu, we're actually on what clang calls + # aarch64-alpine-linux-musl. + # This patch just disables the spurious cflags and ldflags: + apply_patch( + get_v8_path(), + local_path("no-aarch64-linux-gnu-target.patch"), + ) + + if is_musl(): + # On various OSes, the V8 build process brings in a whole copy of the sysroot + # (/usr/include, /usr/lib, etc). Unfortunately on Alpine it tries to use a + # Debian sysroot, which doesn't work. Disable it: + opts["use_sysroot"] = "false" + + # V8 includes its own libc++ whose headers don't seem to work on Alpine: + opts["use_custom_libcxx"] = "false" + + # We optionally use SCCACHE to speed up builds (or restart them on failure): + sccache_path = environ.get("SCCACHE_PATH") + if sccache_path is not None: + opts["cc_wrapper"] = f'"{sccache_path}"' + + makedirs(build_dir, exist_ok=True) + + with open(pathjoin(build_dir, "args.gn"), "w") as f: + f.write("# This file is auto-generated by v8_build.py") + f.write("\n".join(f"{n}={v}" for n, v in opts.items())) + f.write("\n") + + # Now generate Ninja build files: + if is_musl(): + # depot_tools doesn't include a musl-compatible GN, so use the system one: + gn_bin = ("/usr/bin/gn",) else: - try: - os.symlink(target, link_name) - except OSError as e: - if e.errno == errno.EEXIST: - os.remove(link_name) - os.symlink(target, link_name) - else: - raise e - - -def fixup_libtinfo(dir): - dirs = ['/lib64', '/usr/lib64', '/lib', '/usr/lib'] - - v5_locs = ["{}/libtinfo.so.5".format(d) for d in dirs] - found_v5 = next((f for f in v5_locs if os.path.isfile(f)), None) - if found_v5 and os.stat(found_v5).st_size > 100: - return '' - - v6_locs = ["{}/libtinfo.so.6".format(d) for d in dirs] - found_v6 = next((f for f in v6_locs if os.path.isfile(f)), None) - if not found_v6: - return '' - - symlink_force(found_v6, os.path.join(dir, 'libtinfo.so.5')) - return "LD_LIBRARY_PATH='{}:{}'"\ - .format(dir, os.getenv('LD_LIBRARY_PATH', '')) - - -def apply_patches(path, patches_path): - with chdir(path): - - if not os.path.isfile('.applied_patches'): - open('.applied_patches', 'w').close() - - with open('.applied_patches', 'r+') as applied_patches_file: - applied_patches = set(applied_patches_file.read().splitlines()) - - for patch in glob.glob(os.path.join(patches_path, '*.patch')): - if patch not in applied_patches: - call("patch -p1 -N < {}".format(patch)) - applied_patches_file.write(patch + "\n") - - -def patch_sysroot(): - with chdir(local_path("../py_mini_racer/extension/v8/build/linux/debian_sid_amd64-sysroot")): - with open("usr/include/glob.h", "r") as f: - header = f.read() - s, e = header.split("sysroot-creator.sh.", 1) - LOGGER.debug("Patching sysroot /usr/include/glob.h") - with open("usr/include/glob.h", "w") as f: - f.write(s) - f.write("sysroot-creator.sh.") - f.write(""" -__asm__(".symver glob, glob@GLIBC_2.2.5"); -__asm__(".symver glob64, glob64@GLIBC_2.2.5"); - """) - LOGGER.debug("Patching sysroot /usr/include/string.h") - with open("usr/include/string.h", "a") as f: - f.write(""" -__asm__(".symver _sys_errlist, _sys_errlist@GLIBC_2.4"); -__asm__(".symver _sys_nerr, _sys_nerr@GLIBC_2.4"); -__asm__(".symver fmemopen, fmemopen@GLIBC_2.2.5"); -__asm__(".symver memcpy, memcpy@GLIBC_2.2.5"); -__asm__(".symver posix_spawn, posix_spawn@GLIBC_2.2.5"); -__asm__(".symver posix_spawnp, posix_spawnp@GLIBC_2.2.5"); -__asm__(".symver sys_errlist, sys_errlist@GLIBC_2.4"); -__asm__(".symver sys_nerr, sys_nerr@GLIBC_2.4"); - """) - with open("usr/include/math.h", "r") as f: - header = f.read() - s, e = header.split("sysroot-creator.sh.", 1) - LOGGER.debug("Patching sysroot /usr/include/math.h") - with open("usr/include/math.h", "w") as f: - f.write(s) - f.write("sysroot-creator.sh.") - f.write(""" -__asm__(".symver exp2f, exp2f@GLIBC_2.2.5"); -__asm__(".symver expf, expf@GLIBC_2.2.5"); -__asm__(".symver lgamma, lgamma@GLIBC_2.2.5"); -__asm__(".symver lgammaf, lgammaf@GLIBC_2.2.5"); -__asm__(".symver lgammal, lgammal@GLIBC_2.2.5"); -__asm__(".symver log2f, log2f@GLIBC_2.2.5"); -__asm__(".symver logf, logf@GLIBC_2.2.5"); -__asm__(".symver powf, powf@GLIBC_2.2.5"); - """) - - -def build_v8(target=None, build_path=None, revision=None, no_build=False, - no_sysroot=False, no_update=False): - if target is None: - target = "v8" - if build_path is None: - # Must be relative to local_path() - build_path = "../py_mini_racer/extension/out" - if revision is None: - revision = V8_VERSION - install_depot_tools() - if not no_update: + gn_bin = ( + executable, + pathjoin(get_depot_tools_path(), "gn.py"), + ) + + run( + *gn_bin, + "gen", + build_dir, + "--check", + cwd=get_v8_path(), + ) + + # Finally, actually do the build: + if is_musl(): + # depot_tools doesn't include a musl-compatible ninja, so use the system one: + ninja_bin = ("/usr/bin/ninja",) + else: + ninja_bin = ( + executable, + pathjoin(get_depot_tools_path(), "ninja.py"), + ) + + run( + *ninja_bin, + # "-vv", # this is so spammy GitHub Actions struggles to show all the output + "-C", + build_dir, + pathjoin("custom_deps", "mini_racer"), + cwd=get_v8_path(), + ) + + +def ensure_symlink(target, link_name): + LOGGER.debug("Creating symlink to %s on %s", target, link_name) + try: + symlink(target, link_name) + except OSError as e: + if e.errno == EEXIST: + remove(link_name) + symlink(target, link_name) + else: + raise + + +def build_v8( + out_path, + *, + revision=None, + fetch_only=False, + skip_fetch=False, +): + revision = revision or V8_VERSION + + ensure_depot_tools() + + if not skip_fetch: ensure_v8_src(revision) - patch_v8() - if not no_sysroot and sys.platform.startswith("linux"): - patch_sysroot() - prepare_workdir() - if not no_build: - checkout_path = local_path("../py_mini_racer/extension/v8") - cmd_prefix = fixup_libtinfo(checkout_path) - gen_makefiles(build_path, no_sysroot=no_sysroot) - make(build_path, target, cmd_prefix) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--target", default="v8", help="Ninja target") - parser.add_argument("--build-path", default="../py_mini_racer/extension/out", help="Build destination directory (relative to the current path)") + + if fetch_only: + return + + build_dir = pathjoin(get_v8_path(), "out.gn", "build") + + run_build(build_dir) + + # Fish out the build artifacts: + makedirs(out_path, exist_ok=True) + + # Create a map of actual files to in-package filenames, for Hatch to use + # when building the wheel: + artifacts = {} + + for f in get_data_files_list(): + src = pathjoin(build_dir, f) + dst = pathjoin(out_path, f) + + LOGGER.debug("Copying build artifact %s to %s", src, dst) + unlink_if_exists(dst) + copyfile(src, dst) + + artifacts[dst] = pathjoin("py_mini_racer", f) + + LOGGER.debug("Build complete!") + + return artifacts + + +def clean_v8(out_path): + for f in get_data_files_list(): + unlink_if_exists(pathjoin(out_path, f)) + + rmtree(get_workspace_path(), ignore_errors=True) + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument( + "--out-path", + default=pathjoin("src", "py_mini_racer"), + help="Build destination directory", + ) parser.add_argument("--v8-revision", default=V8_VERSION) - parser.add_argument("--no-build", action="store_true", help="Only prepare workdir") - parser.add_argument("--no-update", action="store_true", help="Do not update the workdir") - parser.add_argument("--no-sysroot", action="store_true", help="Do not use the V8 build sysroot") + parser.add_argument("--fetch-only", action="store_true", help="Only fetch V8") + parser.add_argument("--skip-fetch", action="store_true", help="Do not fetch V8") args = parser.parse_args() - build_v8(target=args.target, build_path=args.build_path, revision=args.v8_revision, - no_build=args.no_build, no_update=args.no_update, no_sysroot=args.no_sysroot) + build_v8( + out_path=args.out_path, + revision=args.v8_revision, + fetch_only=args.fetch_only, + skip_fetch=args.skip_fetch, + ) diff --git a/helpers/wheel_pymalloc.py b/helpers/wheel_pymalloc.py deleted file mode 100644 index 9d5ab063..00000000 --- a/helpers/wheel_pymalloc.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Script to add without pymalloc version of the wheel""" - -import os -import re -import sys - -from auditwheel.wheeltools import InWheelCtx, _dist_info_dir -from wheel import pkginfo - - -def get_filenames(directory): - """Get all the file to copy""" - for filename in os.listdir(directory): - if re.search(r"cp\d{2}mu?-manylinux1_\S+\.whl", filename): - yield filename - - -def copy_file(filename, destination): - """Copy the file and put the correct tag""" - - print("Updating file %s" % filename) - out_dir = os.path.abspath(destination) - - tags = filename[:-4].split("-") - - tags[-2] = tags[-2].replace("m", "") - - new_name = "-".join(tags) + ".whl" - wheel_flag = "-".join(tags[2:]) - - with InWheelCtx(os.path.join(destination, filename)) as ctx: - info_fname = os.path.join(_dist_info_dir(ctx.path), 'WHEEL') - infos = pkginfo.read_pkg_info(info_fname) - print("Current Tags: ", ", ".join([v for k, v in infos.items() - if k == "Tag"])) - print("Adding Tag", wheel_flag) - del infos['Tag'] - infos.add_header('Tag', wheel_flag) - pkginfo.write_pkg_info(info_fname, infos) - - ctx.out_wheel = os.path.join(out_dir, new_name) - - print("Saving new wheel into %s" % ctx.out_wheel) - - -def main(): - if len(sys.argv) == 2: - directory = sys.argv[1] - else: - directory = "dist" - for filename in get_filenames(directory): - copy_file(filename, directory) - - -if __name__ == "__main__": - main() diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..7b59e13f --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,39 @@ +site_name: PyMiniRacer +site_url: https://bpcreech.com/PyMiniRacer +theme: + name: material + custom_dir: data + logo: py_mini_racer.png + favicon: favicon.ico + palette: + primary: green +nav: + - Home: index.md + - API Reference: api.md + - Contributing: contributing.md + - Architecture: architecture.md + - Credits: authors.md + - History: history.md + +plugins: + - search + - mkdocstrings: + handlers: + python: + paths: [src] # search packages in the src folder + +hooks: + - mkdocs_hooks.py + +markdown_extensions: + - admonition + - sane_lists + - toc: + permalink: true + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences diff --git a/mkdocs_hooks.py b/mkdocs_hooks.py new file mode 100644 index 00000000..dd177e03 --- /dev/null +++ b/mkdocs_hooks.py @@ -0,0 +1,17 @@ +"""Hooks for mkdocs. + +See https://www.mkdocs.org/user-guide/configuration/#hooks. +""" + + +def on_page_markdown(markdown, **kwargs): + del kwargs + + # The markdown files in the root of the repo refer to each other by name so that + # links between them work on the GitHub code view. + # However, when we bring these pages into the mkdocs site, we lower-case the files + # so the URLs look better. To make the cross-links between the pages work, do light + # surgery on them: + return markdown.replace("ARCHITECTURE.md", "architecture.md").replace( + "CONTRIBUTING.md", "contributing.md" + ) diff --git a/py_mini_racer.png b/py_mini_racer.png new file mode 100644 index 00000000..174e1904 Binary files /dev/null and b/py_mini_racer.png differ diff --git a/py_mini_racer/__about__.py b/py_mini_racer/__about__.py deleted file mode 100644 index dc3b4d9d..00000000 --- a/py_mini_racer/__about__.py +++ /dev/null @@ -1,3 +0,0 @@ -__author__ = 'Sqreen' -__email__ = 'support@sqreen.com' -__version__ = '0.6.0' diff --git a/py_mini_racer/__init__.py b/py_mini_racer/__init__.py deleted file mode 100755 index fa4d8d0e..00000000 --- a/py_mini_racer/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from . import py_mini_racer - -MiniRacer = py_mini_racer.MiniRacer - -__all__ = ['py_mini_racer', 'MiniRacer'] diff --git a/py_mini_racer/extension/.editorconfig b/py_mini_racer/extension/.editorconfig deleted file mode 100644 index 9d08a1a8..00000000 --- a/py_mini_racer/extension/.editorconfig +++ /dev/null @@ -1,9 +0,0 @@ -root = true - -[*] -charset = utf-8 -indent_style = space -indent_size = 2 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true diff --git a/py_mini_racer/extension/.gitignore b/py_mini_racer/extension/.gitignore deleted file mode 100644 index 8ba96d90..00000000 --- a/py_mini_racer/extension/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/py_mini_racer.config -/py_mini_racer.creator -/py_mini_racer.creator.user -/py_mini_racer.files -/py_mini_racer.includes diff --git a/py_mini_racer/extension/.gn b/py_mini_racer/extension/.gn deleted file mode 100644 index c6fefbae..00000000 --- a/py_mini_racer/extension/.gn +++ /dev/null @@ -1 +0,0 @@ -buildconfig = "//build/config/BUILDCONFIG.gn" diff --git a/py_mini_racer/extension/BUILD.gn b/py_mini_racer/extension/BUILD.gn deleted file mode 100644 index 63653740..00000000 --- a/py_mini_racer/extension/BUILD.gn +++ /dev/null @@ -1,36 +0,0 @@ -config("py_mini_racer_config") { - configs = [ "//v8:external_config", "//v8:toolchain", "//v8:features" ] - include_dirs = [ "v8", "$target_gen_dir/v8" ] - cflags = [] -} - -static_library("py_mini_racer_static_lib") { - complete_static_lib = true - output_name = "mini_racer" - sources = [ - "mini_racer_extension.cc", - ] - deps = [ - "//build/config:shared_library_deps", - "//v8:v8", - "//v8:v8_libbase", - "//v8:v8_libplatform", - "//v8:v8_libsampler", - ] - configs += [ ":py_mini_racer_config" ] -} - -shared_library("py_mini_racer_shared_lib") { - output_name = "mini_racer" - sources = [ - "mini_racer_extension.cc", - ] - deps = [ - "//build/config:shared_library_deps", - "//v8:v8", - "//v8:v8_libbase", - "//v8:v8_libplatform", - "//v8:v8_libsampler", - ] - configs += [ ":py_mini_racer_config" ] -} diff --git a/py_mini_racer/extension/mini_racer_extension.cc b/py_mini_racer/extension/mini_racer_extension.cc deleted file mode 100644 index 9d22b216..00000000 --- a/py_mini_racer/extension/mini_racer_extension.cc +++ /dev/null @@ -1,812 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#ifdef V8_OS_WIN -#define LIB_EXPORT __declspec(dllexport) -#else // V8_OS_WIN -#define LIB_EXPORT __attribute__((visibility("default"))) -#endif - -template -static inline T* xalloc(T*& ptr, size_t x = sizeof(T)) { - void* tmp = malloc(x); - if (tmp == NULL) { - fprintf(stderr, "malloc failed. Aborting"); - abort(); - } - ptr = static_cast(tmp); - return static_cast(ptr); -} - -using namespace v8; - -class ArrayBufferAllocator : public v8::ArrayBuffer::Allocator { - public: - virtual void* Allocate(size_t length) { - void* data = AllocateUninitialized(length); - return data == NULL ? data : memset(data, 0, length); - } - virtual void* AllocateUninitialized(size_t length) { return malloc(length); } - virtual void Free(void* data, size_t) { free(data); } -}; - -struct ContextInfo { - Isolate* isolate; - Persistent* context; - ArrayBufferAllocator* allocator; - std::map> backing_stores; - bool interrupted; - size_t soft_memory_limit; - bool soft_memory_limit_reached; - size_t hard_memory_limit; - bool hard_memory_limit_reached; -}; - -struct EvalResult { - bool parsed; - bool executed; - bool terminated; - bool timed_out; - Persistent* value; - Persistent* message; - Persistent* backtrace; - - ~EvalResult() { - kill_value(value); - kill_value(message); - kill_value(backtrace); - } - - private: - static void kill_value(Persistent* val) { - if (!val) { - return; - } - val->Reset(); - delete val; - } -}; - -typedef struct { - ContextInfo* context_info; - const char* eval; - int eval_len; - unsigned long timeout; - EvalResult* result; - size_t max_memory; -} EvalParams; - -enum BinaryTypes { - type_invalid = 0, - type_null = 1, - type_bool = 2, - type_integer = 3, - type_double = 4, - type_str_utf8 = 5, - // type_array = 6, // deprecated - // type_hash = 7, // deprecated - type_date = 8, - type_symbol = 9, - type_object = 10, - - type_function = 100, - type_shared_array_buffer = 101, - type_array_buffer = 102, - - type_execute_exception = 200, - type_parse_exception = 201, - type_oom_exception = 202, - type_timeout_exception = 203, -}; - -struct BinaryValue { - union { - void* ptr_val; - char* str_val; - uint32_t int_val; - double double_val; - }; - enum BinaryTypes type = type_invalid; - size_t len; -}; - -void BinaryValueFree(ContextInfo* context_info, BinaryValue* v) { - if (!v) { - return; - } - switch (v->type) { - case type_execute_exception: - case type_parse_exception: - case type_oom_exception: - case type_timeout_exception: - case type_str_utf8: - free(v->str_val); - break; - case type_bool: - case type_double: - case type_date: - case type_null: - case type_integer: - case type_function: // no value implemented - case type_symbol: - case type_object: - case type_invalid: - // the other types are scalar values - break; - case type_shared_array_buffer: - case type_array_buffer: - context_info->backing_stores.erase(v); - break; - } - free(v); -} - -enum IsolateData { - CONTEXT_INFO, -}; - -static std::unique_ptr current_platform = NULL; -static std::mutex platform_lock; - -static void gc_callback(Isolate* isolate, GCType type, GCCallbackFlags flags) { - ContextInfo* context_info = (ContextInfo*)isolate->GetData(CONTEXT_INFO); - - if (context_info == nullptr) { - return; - } - - HeapStatistics stats; - isolate->GetHeapStatistics(&stats); - size_t used = stats.used_heap_size(); - - context_info->soft_memory_limit_reached = - (used > context_info->soft_memory_limit); - isolate->MemoryPressureNotification((context_info->soft_memory_limit_reached) - ? v8::MemoryPressureLevel::kModerate - : v8::MemoryPressureLevel::kNone); - if (used > context_info->hard_memory_limit) { - context_info->hard_memory_limit_reached = true; - isolate->TerminateExecution(); - } -} - -static void init_v8(char const* flags) { - // no need to wait for the lock if already initialized - if (current_platform != NULL) - return; - - platform_lock.lock(); - - if (current_platform == NULL) { - V8::InitializeICU(); - if (flags != NULL) { - V8::SetFlagsFromString(flags); - } - if (flags != NULL && strstr(flags, "--single-threaded") != NULL) { - current_platform = platform::NewSingleThreadedDefaultPlatform(); - } else { - current_platform = platform::NewDefaultPlatform(); - } - V8::InitializePlatform(current_platform.get()); - V8::Initialize(); - } - - platform_lock.unlock(); -} - -static void breaker(std::timed_mutex& breaker_mutex, void* d) { - EvalParams* data = (EvalParams*)d; - - if (!breaker_mutex.try_lock_for(std::chrono::milliseconds(data->timeout))) { - data->result->timed_out = true; - data->context_info->isolate->TerminateExecution(); - } -} - -static void set_hard_memory_limit(ContextInfo* context_info, size_t limit) { - context_info->hard_memory_limit = limit; - context_info->hard_memory_limit_reached = false; -} - -static bool maybe_fast_call(const char* eval, int eval_len) { - // Does the eval string ends with '()'? - // TODO check if the string is an identifier - return (eval_len > 2 && eval[eval_len - 2] == '(' && - eval[eval_len - 1] == ')'); -} - -static void* nogvl_context_eval(void* arg) { - EvalParams* eval_params = (EvalParams*)arg; - EvalResult* result = eval_params->result; - Isolate* isolate = eval_params->context_info->isolate; - Isolate::Scope isolate_scope(isolate); - HandleScope handle_scope(isolate); - - TryCatch trycatch(isolate); - - Local context = eval_params->context_info->context->Get(isolate); - - Context::Scope context_scope(context); - - set_hard_memory_limit(eval_params->context_info, eval_params->max_memory); - - result->parsed = false; - result->executed = false; - result->terminated = false; - result->timed_out = false; - result->value = NULL; - - std::timed_mutex breaker_mutex; - std::thread breaker_thread; - - // timeout limit - auto timeout = eval_params->timeout; - if (timeout > 0) { - breaker_mutex.lock(); - breaker_thread = - std::thread(&breaker, std::ref(breaker_mutex), (void*)eval_params); - } - // memory limit - if (eval_params->max_memory > 0) { - isolate->AddGCEpilogueCallback(gc_callback); - } - - MaybeLocal maybe_value; - - // Is it a single function call? - if (maybe_fast_call(eval_params->eval, eval_params->eval_len)) { - Local identifier; - Local func; - - // Let's check if the value is a callable identifier - result->parsed = - String::NewFromUtf8(isolate, eval_params->eval, NewStringType::kNormal, - eval_params->eval_len - 2) - .ToLocal(&identifier) && - context->Global()->Get(context, identifier).ToLocal(&func) && - func->IsFunction(); - - if (result->parsed) { - // Call the identifier - maybe_value = Local::Cast(func)->Call( - context, v8::Undefined(isolate), 0, {}); - result->executed = !maybe_value.IsEmpty(); - } - } - - // Fallback on a slower full eval - if (!result->executed) { - Local eval; - Local