diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1da2561..23b364d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,87 +1,259 @@ -name: build +name: Build, deploy, and release on: - merge_group: - pull_request: push: - branches: - - master - workflow_dispatch: + pull_request: jobs: - build: - name: build + pre-commit: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 + prepare: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.extract-version.outputs.version }} + pandoc_version: ${{ steps.extract-version.outputs.pandoc_version }} + wasm_cache_key: ${{ steps.extract-version.outputs.wasm_cache_key }} + wasm_cache_hit: ${{ steps.cache-wasm.outputs.cache-hit }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Extract versions for ${{ github.ref }} + id: extract-version + run: | + VERSION=$(jq -r .version package.json) + PANDOC_VERSION=$(cat pandoc-version.txt) + { + echo "version=${VERSION}"; + echo "pandoc_version=${PANDOC_VERSION}"; + echo "wasm_cache_key=wasm-${PANDOC_VERSION}-${{ hashFiles('patch/pandoc.patch') }}"; + } >> "$GITHUB_OUTPUT" + # Check if we already have the optimized WASM in cache + - name: Check WASM cache + id: cache-wasm + uses: actions/cache/restore@v4 + with: + path: dist/pandoc.wasm + key: ${{ steps.extract-version.outputs.wasm_cache_key }} + + build-wasm: + needs: prepare + if: needs.prepare.outputs.wasm_cache_hit != 'true' runs-on: ubuntu-latest - permissions: - pages: write - id-token: write steps: + - name: Checkout code + uses: actions/checkout@v4 + + # Cache Haskell tools + - name: Cache Haskell tools + id: cache-tools + uses: actions/cache@v4 + with: + path: | + ~/.cabal + ~/.ghc-wasm + key: haskell-tools-${{ needs.prepare.outputs.pandoc_version }} + restore-keys: | + haskell-tools- - - name: setup-alex-happy + - name: Setup build tools + if: steps.cache-tools.outputs.cache-hit != 'true' run: | - pushd "$(mktemp -d)" - cabal path --installdir >> "$GITHUB_PATH" + temp_dir=$(mktemp -d) + pushd "$temp_dir" cabal update - cabal install \ - alex \ - happy + cabal install alex happy + echo "$HOME/.cabal/bin" >> "$GITHUB_PATH" popd - - name: setup-ghc-wasm + - name: Setup GHC-WASM + if: steps.cache-tools.outputs.cache-hit != 'true' run: | - pushd "$(mktemp -d)" - curl -f -L --retry 5 https://gitlab.haskell.org/haskell-wasm/ghc-wasm-meta/-/archive/master/ghc-wasm-meta-master.tar.gz | tar xz --strip-components=1 + temp_dir=$(mktemp -d) + pushd "$temp_dir" + curl -f -L --retry 5 https://gitlab.haskell.org/haskell-wasm/ghc-wasm-meta/-/archive/92ff0eb8541eb0a6097922e3532c3fd44d2f7db4/ghc-wasm-meta-92ff0eb8541eb0a6097922e3532c3fd44d2f7db4.tar.gz | tar xz --strip-components=1 FLAVOUR=9.12 ./setup.sh ~/.ghc-wasm/add_to_github_path.sh popd - - name: checkout - uses: actions/checkout@v4 + - name: Add cached tools to PATH + if: steps.cache-tools.outputs.cache-hit == 'true' + run: | + echo "$HOME/.cabal/bin" >> "$GITHUB_PATH" + ~/.ghc-wasm/add_to_github_path.sh - - name: checkout + - name: Checkout Pandoc uses: actions/checkout@v4 with: - repository: haskell-wasm/pandoc - ref: wasm + repository: jgm/pandoc + ref: ${{ needs.prepare.outputs.pandoc_version }} path: pandoc - - name: gen-plan-json + - name: Patch Pandoc + run: | + pushd pandoc + patch -p1 < ../patch/pandoc.patch + popd + + - name: Generate Cabal plan run: | pushd pandoc wasm32-wasi-cabal build pandoc-cli --dry-run popd - - name: wasm-cabal-cache + # Cache Cabal dependencies and build artifacts + - name: Cache Cabal dependencies uses: actions/cache@v4 with: - key: wasm-cabal-cache-${{ hashFiles('pandoc/dist-newstyle/cache/plan.json') }} - restore-keys: wasm-cabal-cache- path: | ~/.ghc-wasm/.cabal/store pandoc/dist-newstyle + key: wasm-cabal-cache-${{ needs.prepare.outputs.pandoc_version }}-${{ hashFiles('pandoc/dist-newstyle/cache/plan.json') }} + restore-keys: | + wasm-cabal-cache-${{ needs.prepare.outputs.pandoc_version }}- + wasm-cabal-cache- - - name: build + - name: Build Pandoc WASM run: | pushd pandoc wasm32-wasi-cabal build pandoc-cli popd - - name: dist + - name: Optimize WASM run: | - mkdir dist - wasm-opt --low-memory-unused --converge --gufa --flatten --rereloop -Oz $(find pandoc -type f -name pandoc.wasm) -o dist/pandoc.wasm - cp frontend/*.html frontend/*.js dist + mkdir -p dist + WASM_PATH=$(find pandoc -name pandoc.wasm -type f) + wasm-opt --low-memory-unused --converge --gufa --flatten --rereloop -Oz "$WASM_PATH" -o dist/pandoc.wasm + cp src/*.js dist/ - - name: test + - name: Test build run: | - wasmtime run --dir $PWD::/ -- dist/pandoc.wasm pandoc/README.md -o pandoc/README.rst - head --lines=20 pandoc/README.rst + wasmtime run --dir "$PWD"::/ -- dist/pandoc.wasm pandoc/README.md -o pandoc/README.rst + head -20 pandoc/README.rst - - name: upload-pages-artifact - uses: actions/upload-pages-artifact@v3 + - name: Save to cache + uses: actions/cache/save@v4 + with: + path: dist/pandoc.wasm + key: ${{ needs.prepare.outputs.wasm_cache_key }} + + - name: Upload artifact + uses: actions/upload-artifact@v4 with: + name: wasm-build path: dist - retention-days: 90 - - name: deploy-pages + post-process: + needs: [prepare, build-wasm] + if: always() # Run even if build-wasm is skipped + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + # Either get the artifact from build-wasm or from cache + - name: Download built artifact + if: needs.prepare.outputs.wasm_cache_hit != 'true' + uses: actions/download-artifact@v4 + with: + name: wasm-build + path: dist + + # This step only runs if we hit the cache + - name: Restore from cache + if: needs.prepare.outputs.wasm_cache_hit == 'true' + uses: actions/cache/restore@v4 + with: + path: dist/pandoc.wasm + key: ${{ needs.prepare.outputs.wasm_cache_key }} + fail-on-cache-miss: true + + # Combine with JS files - they're not part of the cache key + # but we need them in the artifact + - name: Ensure JS files are included + run: | + cp src/*.js dist/ + + - name: Upload final artifact + uses: actions/upload-artifact@v4 + with: + name: wasm-pandoc-${{ needs.prepare.outputs.version }} + path: dist + + deploy-pages: + needs: [prepare, post-process] + if: always() && !startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: wasm-pandoc-${{ needs.prepare.outputs.version }} + path: dist + + - name: Prepare demo + run: | + cp dist/pandoc.wasm demo/ + + - name: Upload pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: demo + + - name: Deploy to Pages uses: actions/deploy-pages@v4 + + release: + needs: [prepare, post-process] + if: always() && startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: wasm-pandoc-${{ needs.prepare.outputs.version }} + path: dist + + - name: Add metadata files + run: cp {package.json,README.md,LICENSE} dist/ + + - name: Create release package + run: | + pushd dist + zip -r ../wasm-pandoc-${{ needs.prepare.outputs.version }}.zip . + popd + + - name: Upload release asset + uses: softprops/action-gh-release@v2 + with: + files: wasm-pandoc-${{ needs.prepare.outputs.version }}.zip + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + registry-url: "https://registry.npmjs.org" + + - name: Publish to NPM + run: | + cd dist + npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d782400 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/pandoc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..8082c6f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-yaml + # Check YAML files for syntax errors, particularly useful for GitHub Actions workflows + files: \.ya?ml$ + - id: trailing-whitespace + # Ensure no trailing whitespace + - id: end-of-file-fixer + # Ensure files end with a newline + - id: check-added-large-files + # Prevent large files from being committed + args: ['--maxkb=500'] + - id: check-merge-conflict + # Check for merge conflict strings + - id: check-json + # Validate JSON files + - id: detect-private-key + # Prevent accidental commit of private keys + +- repo: https://github.com/rhysd/actionlint + rev: v1.7.7 + hooks: + - id: actionlint + # Lint GitHub Actions workflows + files: ^\.github/workflows/ + +- repo: https://github.com/biomejs/pre-commit + rev: v1.9.4 + hooks: + - id: biome-check + # #entry: biome check --files-ignore-unknown=true --no-errors-on-unmatched --fix --unsafe + # additional_dependencies: ["@biomejs/biome@1.9.2"] diff --git a/README.md b/README.md index fec2f1a..1eb54bf 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,76 @@ -# `pandoc-wasm` +# `wasm-pandoc` -[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/#haskell-wasm:matrix.terrorjack.com) +**Looking for maintainer:** Johannes Wilm has temporarily taken over maintainership of this package due to there being no package on NPM. +However, he knows very little about wasm and haskell and would like for someone else to take this package again. -The latest version of `pandoc` CLI compiled as a standalone -`wasm32-wasi` module that can be run by engines like `wasmtime` as -well as browsers. -## [Live demo](https://tweag.github.io/pandoc-wasm) +The latest version of `pandoc` CLI compiled as a standalone `wasm32-wasi` module that can be run by browsers. + +## [Live demo](https://fiduswriter.github.io/wasm-pandoc) Stdin on the left, stdout on the right, command line arguments at the bottom. No convert button, output is produced dynamically as input changes. -You're also more than welcome to fetch the -[`pandoc.wasm`](https://tweag.github.io/pandoc-wasm/pandoc.wasm) -module and make your own customized app. `pandoc.wasm` is fully -`wasm32-wasi` compliant and doesn't make use of any JSFFI feature in -the ghc wasm backend. -## Building +## To use + +1. Make `wasm-pandoc` a dependency in your project.json. + +2. In your bundler mark "wasm" as an asset/resource. For example in rspack, in your config file: + +```js +module.exports = { + ... + module: { + ... + rules: [ + ... + { + test: /\.(wasm)$/, + type: "asset/resource" + } + ... + ] + ... + } + ... +} +``` + +3. Import `pandoc` from `wasm-pandoc` like this: + +```js +import { pandoc } from "wasm-pandoc" +``` + +4. Execute it like this (it's async): + +```js +const output = await pandoc( + '-s -f json -t markdown', // command line switches + inputFileContents, // string for text formats or blob for binary formats + [ // Additional files - for example bibliography or images + { + filename: 'image13.png', + contents: ..., // string for text formats or blob for binary formats + }, + ... + ] +) + +console.log(output) + +{ + out: '...', + mediaFiles: Map {'media': Map {'image1.jpg' => Blob, 'image2.png' => Blob, ...}} +} -`pandoc.wasm` is built with 9.12 flavour of ghc wasm backend in CI, -which can be installed via -[`ghc-wasm-meta`](https://gitlab.haskell.org/haskell-wasm/ghc-wasm-meta). You -need at least 9.10 since it's the earliest major version with (my -non-official) backports for ghc wasm backend's Template Haskell & ghci -support. +``` -It's built using my -[fork](https://github.com/haskell-wasm/pandoc/tree/wasm) which is -based on latest `pandoc` release and patches dependencies, cabal -config as well as some module code to make things compilable to wasm: +`out` will either be a string (for text formats) or a Blob for binary formats of the main output. `mediaFiles` will be a map of all additional dirs/files that pandoc has created during the process. -- No http client/server functionality. `wasip1` doesn't have proper - sockets support anyway, and support for future versions of wasi is - not on my radar for now. -- No lua support. lua requires `setjmp`/`longjmp` which already work - in `wasi-libc` to some extent, but that requires wasm exception - handling feature which is not supported by `wasmtime` yet. -Other functionalities should just work, if not feel free to file a bug -report :) ## Acknowledgements @@ -48,9 +78,10 @@ Thanks to John MacFarlane and all the contributors who made `pandoc` possible: a fantastic tool that has benefited many developers and is a source of pride for the Haskell community! -Thanks to all past efforts of using `asterius` to compile `pandoc` to -wasm, including but not limited to: +Thanks to all efforts to make `pandoc` run with wasm, including but not limited to: +- amesgen [`Don't patch out network`](https://github.com/haskell-wasm/pandoc/pull/1) +- Cheng Shao [`pandoc-wasm`](https://github.com/tweag/pandoc-wasm) - George Stagg's [`pandoc-wasm`](https://github.com/georgestagg/pandoc-wasm) - Yuto Takahashi's [`wasm-pandoc`](https://github.com/y-taka-23/wasm-pandoc) -- My legacy asterius pandoc [demo](https://asterius.netlify.app/demo/pandoc/pandoc.html) +- TerrorJack's asterius pandoc [demo](https://asterius.netlify.app/demo/pandoc/pandoc.html) diff --git a/biome.json b/biome.json new file mode 100644 index 0000000..305d79a --- /dev/null +++ b/biome.json @@ -0,0 +1,150 @@ +{ + "formatter": { + "enabled": true, + "useEditorconfig": true, + "formatWithErrors": false, + "indentStyle": "space", + "indentWidth": 4, + "lineEnding": "lf", + "lineWidth": 80, + "attributePosition": "auto", + "bracketSpacing": true, + "ignore": [ + "**/.transpile-cache/", + "**/venv/", + "**/static-transpile/", + "**/static-libs/", + "**/static-collected/", + "**/node-modules/", + "**/testing/", + "**/.eslintrc.mjs", + "**/*.json", + "**/*.html", + "**/.direnv", + "**/venv", + "**/.transpile", + "**/.babelrc", + "**/sw-template.js" + ] + }, + "linter": { + "enabled": true, + "rules": { + "recommended": false, + "complexity": { + "noExtraBooleanCast": "error", + "noMultipleSpacesInRegularExpressionLiterals": "error", + "noUselessCatch": "off", + "noUselessConstructor": "off", + "noUselessLabel": "error", + "noUselessLoneBlockStatements": "error", + "noUselessRename": "error", + "noUselessStringConcat": "error", + "noUselessTernary": "off", + "noUselessUndefinedInitialization": "error", + "noVoid": "error", + "noWith": "error", + "useLiteralKeys": "off" + }, + "correctness": { + "noConstAssign": "error", + "noConstantCondition": "error", + "noEmptyCharacterClassInRegex": "error", + "noEmptyPattern": "error", + "noGlobalObjectCalls": "error", + "noInnerDeclarations": "error", + "noInvalidConstructorSuper": "error", + "noInvalidUseBeforeDeclaration": "off", + "noNewSymbol": "error", + "noNonoctalDecimalEscape": "error", + "noPrecisionLoss": "error", + "noSelfAssign": "error", + "noSetterReturn": "error", + "noSwitchDeclarations": "error", + "noUndeclaredVariables": "error", + "noUnreachable": "error", + "noUnreachableSuper": "error", + "noUnsafeFinally": "error", + "noUnsafeOptionalChaining": "error", + "noUnusedLabels": "error", + "noUnusedVariables": "error", + "useArrayLiterals": "error", + "useIsNan": "error", + "useValidForDirection": "error", + "useYield": "error" + }, + "security": { "noGlobalEval": "error" }, + "style": { + "noArguments": "off", + "noCommaOperator": "error", + "noNegationElse": "off", + "noParameterAssign": "off", + "noRestrictedGlobals": { "level": "error", "options": {} }, + "noVar": "error", + "noYodaExpression": "off", + "useBlockStatements": "error", + "useCollapsedElseIf": "off", + "useConsistentBuiltinInstantiation": "error", + "useConst": "warn", + "useDefaultSwitchClause": "off", + "useNumericLiterals": "error", + "useShorthandAssign": "off", + "useSingleVarDeclarator": "off", + "useTemplate": "off" + }, + "suspicious": { + "noAsyncPromiseExecutor": "error", + "noCatchAssign": "error", + "noClassAssign": "error", + "noCompareNegZero": "error", + "noControlCharactersInRegex": "off", + "noDebugger": "error", + "noDoubleEquals": "off", + "noDuplicateCase": "error", + "noDuplicateClassMembers": "error", + "noDuplicateObjectKeys": "error", + "noDuplicateParameters": "error", + "noEmptyBlockStatements": "off", + "noFallthroughSwitchClause": "error", + "noFunctionAssign": "error", + "noGlobalAssign": "error", + "noImportAssign": "error", + "noLabelVar": "error", + "noMisleadingCharacterClass": "error", + "noPrototypeBuiltins": "off", + "noRedeclare": "error", + "noSelfCompare": "error", + "noShadowRestrictedNames": "error", + "noSparseArray": "error", + "noUnsafeNegation": "error", + "useAwait": "error", + "useGetterReturn": "error", + "useValidTypeof": "error" + } + }, + "ignore": [ + "**/.transpile-cache/", + "**/venv/", + "**/static-transpile/", + "**/static-libs/", + "**/static-collected/", + "**/testing/", + "**/manifest.json", + "**/sw-template.js" + ] + }, + "javascript": { + "formatter": { + "jsxQuoteStyle": "double", + "quoteProperties": "asNeeded", + "trailingCommas": "none", + "semicolons": "asNeeded", + "indentWidth": 4, + "arrowParentheses": "asNeeded", + "bracketSameLine": true, + "quoteStyle": "double", + "attributePosition": "auto", + "bracketSpacing": false + } + } +} diff --git a/frontend/index.html b/demo/index.html similarity index 98% rename from frontend/index.html rename to demo/index.html index 7e710ec..0944228 100644 --- a/frontend/index.html +++ b/demo/index.html @@ -3,7 +3,7 @@ - pandoc-wasm playground + wasm-pandoc playground