Upload playground bundle to Cloudflare R2 (#7472)

* Configure Rclone remote

* refactor generate_cmijs script

* rewrite bundle uploading

* fix workflow

* add prefix

* fix comments

* update actions

* Sync available version list

* fix tar command

* add a note

* use semver explicitly

Author: cometkim

.github/workflows/ci.yml

@@ -396,11 +396,23 @@ jobs:
         if: matrix.build_playground
         run: yarn workspace playground test
 
+      - name: Setup Rclone
+        if: ${{ matrix.build_playground && startsWith(github.ref, 'refs/tags/v') }}
+        uses: cometkim/rclone-actions/setup-rclone@89de8311dd0ca0b43143329d3daeb1041852ad9f
+
+      - name: Configure Rclone remote
+        if: ${{ matrix.build_playground && startsWith(github.ref, 'refs/tags/v') }}
+        uses: cometkim/rclone-actions/configure-remote/s3-provider@89de8311dd0ca0b43143329d3daeb1041852ad9f
+        with:
+          name: rescript
+          provider: Cloudflare
+          endpoint: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com
+          access-key-id: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
+          secret-access-key: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
+          acl: private
+
       - name: Upload playground compiler to CDN
         if: ${{ matrix.build_playground && startsWith(github.ref, 'refs/tags/v') }}
-        env:
-          KEYCDN_USER: ${{ secrets.KEYCDN_USER }}
-          KEYCDN_PASSWORD: ${{ secrets.KEYCDN_PASSWORD }}
         run: yarn workspace playground upload-bundle
 
       - name: "Upload artifacts: binaries"

Makefile

@@ -69,14 +69,14 @@ artifacts: lib
 # Builds the core playground bundle (without the relevant cmijs files for the runtime)
 playground:
 	dune build --profile browser
-	cp ./_build/default/compiler/jsoo/jsoo_playground_main.bc.js packages/playground/compiler.js
+	cp -f ./_build/default/compiler/jsoo/jsoo_playground_main.bc.js packages/playground/compiler.js
 
 # Creates all the relevant core and third party cmij files to side-load together with the playground bundle
 playground-cmijs: artifacts
 	yarn workspace playground build
 
 # Builds the playground, runs some e2e tests and releases the playground to the
-# CDN (requires KEYCDN_USER and KEYCDN_PASSWORD set in the env variables)
+# Cloudflare R2 (requires Rclone `rescript:` remote)
 playground-release: playground playground-cmijs
 	yarn workspace playground test
 	yarn workspace playground upload-bundle

package.json

@@ -84,9 +84,11 @@
   "devDependencies": {
     "@biomejs/biome": "1.9.4",
     "@types/node": "^20.14.9",
+    "@types/semver": "^7.7.0",
     "@yarnpkg/types": "^4.0.1",
     "mocha": "10.8.2",
     "nyc": "15.0.0",
+    "semver": "^7.7.2",
     "typescript": "5.8.2"
   },
   "workspaces": [

packages/playground/.gitignore

@@ -32,3 +32,5 @@ yarn.lock
 
 /packages/
 /compiler.js
+
+.tmp/

packages/playground/package.json

@@ -5,8 +5,8 @@
   "scripts": {
     "clean": "rescript clean",
     "test": "node ./playground_test.cjs",
-    "build": "rescript clean && rescript build && node ./scripts/generate_cmijs.mjs && rollup -c",
-    "upload-bundle": "./scripts/upload_bundle.sh",
+    "build": "rescript clean && rescript build && node scripts/generate_cmijs.mjs && rollup -c",
+    "upload-bundle": "node scripts/upload_bundle.mjs",
     "revalidate": "./scripts/website_update_playground.sh"
   },
   "dependencies": {

packages/playground/scripts/common.mjs

@@ -0,0 +1,33 @@
+// @ts-check
+
+import * as child_process from "node:child_process";
+import * as path from "node:path";
+
+export const compilerRootDir = path.join(
+  import.meta.dirname,
+  "..",
+  "..",
+  "..",
+);
+
+// The playground-bundling root dir
+export const playgroundDir = path.join(import.meta.dirname, "..");
+
+// Final target output directory where all the cmijs will be stored
+export const playgroundPackagesDir = path.join(playgroundDir, "packages");
+
+/**
+ * @param {string} cmd
+ * @param {child_process.ExecSyncOptions} [opts={}]
+ */
+export function exec(cmd, opts = {}) {
+  console.log(`>>>>>> running command: ${cmd}`);
+  const result = child_process.execSync(cmd, {
+    cwd: playgroundDir,
+    stdio: "inherit",
+    ...opts,
+    encoding: "utf8",
+  });
+  console.log("<<<<<<");
+  return result || "";
+}

packages/playground/scripts/generate_cmijs.mjs

@@ -13,92 +13,54 @@
  * playground bundle.
  */
 
-import * as child_process from "node:child_process";
 import * as fs from "node:fs";
 import * as path from "node:path";
 
 import resConfig from "../rescript.json" with { type: "json" };
+import {
+  exec,
+  compilerRootDir,
+  playgroundPackagesDir,
+} from "./common.mjs";
 
-const RESCRIPT_COMPILER_ROOT_DIR = path.join(
-  import.meta.dirname,
-  "..",
-  "..",
-  "..",
-);
+exec("yarn rescript clean");
+exec("yarn rescript");
 
-// The playground-bundling root dir
-const PLAYGROUND_DIR = path.join(import.meta.dirname, "..");
-
-// Final target output directory where all the cmijs will be stored
-const PACKAGES_DIR = path.join(PLAYGROUND_DIR, "packages");
+// We need to build the compiler's builtin modules as a separate cmij.
+// Otherwise we can't use them for compilation within the playground.
+buildCmij(compilerRootDir, "compiler-builtins");
 
-// Making sure this directory exists, since it's not checked in to git
-fs.mkdirSync(PACKAGES_DIR, { recursive: true });
+const packages = resConfig["bs-dependencies"];
+for (const pkgName of packages) {
+  buildCmij(
+    path.join(compilerRootDir, "node_modules", pkgName),
+    pkgName,
+  );
+}
 
 /**
- * @param {string} cmd
+ * @param {string} pkgDir
+ * @param {string} pkgName
  */
-function e(cmd) {
-  console.log(`>>>>>> running command: ${cmd}`);
-  child_process.execSync(cmd, {
-    cwd: PLAYGROUND_DIR,
-    encoding: "utf8",
-    stdio: [0, 1, 2],
-  });
-  console.log("<<<<<<");
-}
-
-e("yarn rescript clean");
-e("yarn rescript");
-
-const packages = resConfig["bs-dependencies"];
-
-// We need to build the compiler's builtin modules as a separate cmij.
-// Otherwise we can't use them for compilation within the playground.
-function buildCompilerCmij() {
-  const rescriptLibOcamlFolder = path.join(
-    RESCRIPT_COMPILER_ROOT_DIR,
+function buildCmij(pkgDir, pkgName) {
+  const libOcamlFolder = path.join(
+    pkgDir,
     "lib",
     "ocaml",
   );
 
-  const outputFolder = path.join(PACKAGES_DIR, "compiler-builtins");
+  const outputFolder = path.join(playgroundPackagesDir, pkgName);
   fs.mkdirSync(outputFolder, { recursive: true });
 
   const cmijFile = path.join(outputFolder, "cmij.js");
-
-  e(
-    `find ${rescriptLibOcamlFolder} -name "*.cmi" -or -name "*.cmj" | xargs -n1 basename | xargs js_of_ocaml build-fs -o ${cmijFile} -I ${rescriptLibOcamlFolder}`,
-  );
+  const inputFiles = fs.readdirSync(libOcamlFolder).filter(isCmij).join(" ");
+  exec(`js_of_ocaml build-fs -o ${cmijFile} -I ${libOcamlFolder} ${inputFiles}`);
 }
 
-function buildThirdPartyCmijs() {
-  for (const pkg of packages) {
-    const libOcamlFolder = path.join(
-      RESCRIPT_COMPILER_ROOT_DIR,
-      "node_modules",
-      pkg,
-      "lib",
-      "ocaml",
-    );
-    const libEs6Folder = path.join(
-      RESCRIPT_COMPILER_ROOT_DIR,
-      "node_modules",
-      pkg,
-      "lib",
-      "es6",
-    );
-    const outputFolder = path.join(PACKAGES_DIR, pkg);
-    fs.mkdirSync(outputFolder, { recursive: true });
-
-    const cmijFile = path.join(outputFolder, "cmij.js");
-
-    e(`find ${libEs6Folder} -name '*.js' -exec cp {} ${outputFolder} \\;`);
-    e(
-      `find ${libOcamlFolder} -name "*.cmi" -or -name "*.cmj" | xargs -n1 basename | xargs js_of_ocaml build-fs -o ${cmijFile} -I ${libOcamlFolder}`,
-    );
-  }
+/**
+ * @param {string} basename 
+ * @return {boolean}
+ */
+function isCmij(basename) {
+  return /\.cm(i|j)$/.test(basename);
 }
-
-buildCompilerCmij();
-buildThirdPartyCmijs();

packages/playground/scripts/upload_bundle.mjs

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+
+// @ts-check
+
+// This script will publish the compiler.js bundle / packages cmij.js files to our Cloudclare R2.
+// The target folder on R2 bucket will be the compiler.js' version number.
+// This script requires `rclone` and `zstd` to be installed.
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as readline from "node:readline/promises";
+import { createRequire } from "node:module";
+import semver from "semver";
+
+import {
+  exec,
+  playgroundDir,
+  playgroundPackagesDir,
+} from "./common.mjs";
+
+const require = createRequire(import.meta.url);
+// @ts-ignore
+const { rescript_compiler } = require('../compiler.js');
+
+/**
+ * @type {number}
+ */
+const version = rescript_compiler.make().rescript.version;
+const tag = "v" + version;
+
+console.log("Uploading playground artifacts for %s", tag);
+if (!process.env.CI) {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+  const answer = await rl.question("Do you want to proceed? [y/N]: ");
+  rl.close();
+
+  if (answer.toLowerCase() !== "y") {
+    console.log("Cancelled");
+    process.exit(1);
+  }
+}
+
+const rcloneOpts = (process.env.CI
+  ? [
+    "--stats 5",
+    "--checkers 5000",
+    "--transfers 8",
+    "--buffer-size 128M",
+    "--s3-chunk-size 128M",
+    "--s3-upload-concurrency 8",
+  ]
+  : [
+    "--progress",
+    "--checkers 5000",
+    "--transfers 16",
+    "--buffer-size 128M",
+    "--s3-chunk-size 128M",
+    "--s3-upload-concurrency 16",
+  ]).join(" ");
+
+const remote = process.env.RCLONE_REMOTE || "rescript";
+const bucket = "cdn-assets";
+
+// Create a temporary directory for bundling
+const tmpDir = path.join(playgroundDir, ".tmp");
+const artifactsDir = path.join(tmpDir, tag);
+const archivePath = path.join(tmpDir, `${tag}.tar.zst`);
+fs.rmSync(tmpDir, { recursive: true, force: true });
+fs.mkdirSync(artifactsDir, { recursive: true });
+
+console.log("Copying compiler.js");
+fs.copyFileSync(
+  path.join(playgroundDir, "compiler.js"),
+  path.join(artifactsDir, "compiler.js"),
+);
+
+console.log("Copying packages");
+fs.cpSync(playgroundPackagesDir, artifactsDir, { recursive: true });
+
+// Create tar.zst archive
+console.log("Creating archive...");
+exec(`tar \\
+  --use-compress-program="zstd -T0 --adapt --exclude-compressed" \\
+  -cf "${archivePath}" \\
+  -C "${artifactsDir}" .
+`);
+
+// TODO: This will no longer be needed after the KeyCDN to Cloudflare transition is complete.
+console.log(`Uploading v${version} artifacts...`);
+exec(`rclone sync ${rcloneOpts} --fast-list \\
+  "${artifactsDir}" \\
+  "${remote}:${bucket}/${tag}"
+`);
+
+console.log("Uploading archive...");
+exec(`rclone copyto ${rcloneOpts} \\
+  "${archivePath}" \\
+  "${remote}:${bucket}/playground-bundles/${tag}.tar.zst"
+`);
+
+console.log("Update versions.json");
+{
+  const bundles = exec(
+    `rclone lsf --include="*.tar.zst" rescript:cdn-assets/playground-bundles`,
+    { stdio: ['ignore', 'pipe', 'pipe'] },
+  ).trimEnd().split("\n");
+
+  /** @type {string[]} */
+  let playgroundVersions = [];
+  for (const bundle of bundles) {
+    playgroundVersions.push(bundle.replace(".tar.zst", ""));
+  }
+  playgroundVersions = semver.sort(playgroundVersions);
+
+  console.log('Versions: ', playgroundVersions);
+
+  const versionsPath = path.join(tmpDir, "versions.json");
+  fs.writeFileSync(
+    versionsPath,
+    JSON.stringify(playgroundVersions),
+  );
+  exec(`rclone copyto ${rcloneOpts} \\
+    "${versionsPath}" \\
+    "${remote}:${bucket}/playground-bundles/versions.json"
+  `);
+}