WIP: use arrow-wasm instead of custom rust library for tests

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/arrow-wasm"]
+	path = tests/arrow-wasm
+	url = https://github.com/kylebarron/arrow-wasm

package.json

@@ -34,13 +34,19 @@
   "devDependencies": {
     "@rollup/plugin-terser": "^0.4.3",
     "@rollup/plugin-typescript": "^11.1.2",
+    "@types/node": "^20.6.1",
     "@types/tape": "^4.13.2",
     "apache-arrow": "^13",
+    "arrow-wasm-arrow1": "./tests/arrow-wasm/pkg_arrow1/",
+    "arrow-wasm-arrow2": "./tests/arrow-wasm/pkg_arrow2/",
     "rollup": "^3.26.2",
     "rollup-plugin-dts": "^5.3.0",
     "rust-arrow-ffi": "./tests/rust-arrow-ffi/pkg/",
     "ts-node": "^10.9.1",
     "typescript": "^4.7.4",
     "vitest": "^0.33.0"
+  },
+  "volta": {
+    "node": "20.6.1"
   }
 }

tests/ffi.test.ts

@@ -2,17 +2,18 @@ import { readFileSync } from "fs";
 import { describe, it, expect } from "vitest";
 import * as arrow from "apache-arrow";
 import * as wasm from "rust-arrow-ffi";
+import * as arrowWasm from "arrow-wasm-arrow2";
 import { arrowTableToFFI, arraysEqual, loadIPCTableFromDisk } from "./utils";
 import { parseField, parseVector } from "../src";
 import { Type } from "../src/types";
 
-wasm.setPanicHook();
+arrowWasm.setPanicHook();
 
-// @ts-expect-error
-const WASM_MEMORY: WebAssembly.Memory = wasm.__wasm.memory;
+const WASM_MEMORY: WebAssembly.Memory = arrowWasm.wasmMemory()
 
 const TEST_TABLE = loadIPCTableFromDisk("tests/table.arrow");
-const FFI_TABLE = arrowTableToFFI(TEST_TABLE);
+const WASM_TABLE = arrowWasm.Table.fromIPCStream(arrow.tableToIPC(TEST_TABLE, "stream"))
+const FFI_TABLE = WASM_TABLE.toFFI();
 
 interface FixtureType {
   data: any;
@@ -240,12 +241,12 @@ describe("large_binary", (t) => {
 
     // Then read the large table
     const tableBuffer = readFileSync("tests/large_table.arrow");
-    let ffiTable = wasm.arrowIPCToFFI(tableBuffer);
+    let ffiTable = arrowWasm.Table.fromIPCStream(tableBuffer).intoFFI();
 
     // This is hard-coded based on the order in pyarrow_generate_data.py
     let columnIndex = 0;
 
-    const fieldPtr = ffiTable.schemaAddr(columnIndex);
+    const fieldPtr = ffiTable.schemaAddr();
     const field = parseField(WASM_MEMORY.buffer, fieldPtr);
 
     expect(field.name).toStrictEqual("large_binary");

tests/record-batch.test.ts

@@ -1,17 +1,17 @@
 import { describe, it, expect } from "vitest";
-import * as wasm from "rust-arrow-ffi";
+import * as arrowWasm from "arrow-wasm-arrow2";
 import {
   arraysEqual,
   loadIPCTableFromDisk,
   arrowTableToFFIRecordBatch,
 } from "./utils";
 import { parseRecordBatch } from "../src";
 import { readFileSync } from "fs";
+import assert from "assert";
 
-wasm.setPanicHook();
+arrowWasm.setPanicHook();
 
-// @ts-expect-error
-const WASM_MEMORY: WebAssembly.Memory = wasm.__wasm.memory;
+const WASM_MEMORY: WebAssembly.Memory = arrowWasm.wasmMemory();
 
 const TEST_TABLE = loadIPCTableFromDisk("tests/table.arrow");
 const FFI_RECORD_BATCH = arrowTableToFFIRecordBatch(TEST_TABLE);
@@ -20,8 +20,8 @@ describe("record batch", (t) => {
   function test(copy: boolean) {
     const newRecordBatch = parseRecordBatch(
       WASM_MEMORY.buffer,
-      FFI_RECORD_BATCH.array_addr(),
-      FFI_RECORD_BATCH.field_addr(),
+      FFI_RECORD_BATCH.arrayAddr(),
+      FFI_RECORD_BATCH.schemaAddr(),
       copy
     );
 
@@ -53,12 +53,16 @@ describe("record batch with large types", (t) => {
 
   function test(copy: boolean) {
     const tableBuffer = readFileSync("tests/large_table.arrow");
-    let ffiRecordBatch = wasm.arrowIPCToFFIRecordBatch(tableBuffer);
+    const wasmTable = arrowWasm.Table.fromIPCFile(tableBuffer);
+    assert(wasmTable.numBatches === 1, "Should have one batch");
+    const wasmRecordBatch = wasmTable.recordBatch(0);
+    assert(wasmRecordBatch, "wasm record batch should not be undefined");
+    let ffiRecordBatch = wasmRecordBatch.intoFFI();
 
     const newRecordBatch = parseRecordBatch(
       WASM_MEMORY.buffer,
-      ffiRecordBatch.array_addr(),
-      ffiRecordBatch.field_addr(),
+      ffiRecordBatch.arrayAddr(),
+      ffiRecordBatch.schemaAddr(),
       copy
     );
 

tests/utils.ts

@@ -1,22 +1,34 @@
 import { readFileSync } from "fs";
 import * as arrow from "apache-arrow";
-import * as wasm from "rust-arrow-ffi";
+import * as arrowWasm from "arrow-wasm-arrow2";
 
 export function loadIPCTableFromDisk(path: string): arrow.Table {
   const buffer = readFileSync(path);
   return arrow.tableFromIPC(buffer);
 }
 
 /** Put an Arrow Table in Wasm memory and expose it via FFI */
-export function arrowTableToFFI(table: arrow.Table): wasm.FFIArrowTable {
-  return wasm.arrowIPCToFFI(arrow.tableToIPC(table, "file"));
+export function arrowTableToFFI(table: arrow.Table): arrowWasm.FFITable {
+  return arrowWasm.Table.fromIPCStream(
+    arrow.tableToIPC(table, "stream")
+  ).intoFFI();
 }
 
 /** Put an Arrow Table in Wasm memory and expose it via an FFI RecordBatch */
 export function arrowTableToFFIRecordBatch(
   table: arrow.Table
-): wasm.FFIArrowRecordBatch {
-  return wasm.arrowIPCToFFIRecordBatch(arrow.tableToIPC(table, "file"));
+): arrowWasm.FFIRecordBatch {
+  const wasmTable = arrowWasm.Table.fromIPCStream(
+    arrow.tableToIPC(table, "stream")
+  );
+  if (wasmTable.numBatches !== 1) {
+    throw new Error(`expected only one batch, got ${wasmTable.numBatches}`);
+  }
+  const wasmRecordBatch = wasmTable.recordBatch(0);
+  if (!wasmRecordBatch) {
+    throw new Error("wasm record batch is undefined");
+  }
+  return wasmRecordBatch.intoFFI();
 }
 
 export function arraysEqual<T>(

yarn.lock

@@ -464,6 +464,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/node@npm:^20.6.1":
+  version: 20.6.2
+  resolution: "@types/node@npm:20.6.2"
+  checksum: 96fe5303872640a173f3fd43e289a451776ed5b8f0090094447c6790b43f23fb607eea8268af0829cef4d132e5afa0bfa4cd871aa7412e9042a414a698e9e971
+  languageName: node
+  linkType: hard
+
 "@types/pad-left@npm:2.1.1":
   version: 2.1.1
   resolution: "@types/pad-left@npm:2.1.1"
@@ -715,8 +722,11 @@ __metadata:
   dependencies:
     "@rollup/plugin-terser": ^0.4.3
     "@rollup/plugin-typescript": ^11.1.2
+    "@types/node": ^20.6.1
     "@types/tape": ^4.13.2
     apache-arrow: ^13
+    arrow-wasm-arrow1: ./tests/arrow-wasm/pkg_arrow1/
+    arrow-wasm-arrow2: ./tests/arrow-wasm/pkg_arrow2/
     rollup: ^3.26.2
     rollup-plugin-dts: ^5.3.0
     rust-arrow-ffi: ./tests/rust-arrow-ffi/pkg/
@@ -728,6 +738,20 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"arrow-wasm-arrow1@file:./tests/arrow-wasm/pkg_arrow1/::locator=arrow-js-ffi%40workspace%3A.":
+  version: 0.1.0
+  resolution: "arrow-wasm-arrow1@file:./tests/arrow-wasm/pkg_arrow1/#./tests/arrow-wasm/pkg_arrow1/::hash=98dbb1&locator=arrow-js-ffi%40workspace%3A."
+  checksum: 012bbb0849532fb9aa42889c058850692c5ff9fd512aa9c50f6999f8ff341b2f0b826c6a70a47c799b9e3a39db8dddb2275455ec00dfc1657b678dba94ff8744
+  languageName: node
+  linkType: hard
+
+"arrow-wasm-arrow2@file:./tests/arrow-wasm/pkg_arrow2/::locator=arrow-js-ffi%40workspace%3A.":
+  version: 0.1.0
+  resolution: "arrow-wasm-arrow2@file:./tests/arrow-wasm/pkg_arrow2/#./tests/arrow-wasm/pkg_arrow2/::hash=94e2c0&locator=arrow-js-ffi%40workspace%3A."
+  checksum: ef606ea606fc68373f9d023d473b16c7612bb4af03a005dec2e27d6b0a6ed2dc22b9f993fce63b0e4a1120468dc8720b4582809520857aa6eeebb38e75a44252
+  languageName: node
+  linkType: hard
+
 "assertion-error@npm:^1.1.0":
   version: 1.1.0
   resolution: "assertion-error@npm:1.1.0"