implemented web workers for performance

Author: abrman

package-lock.json

@@ -1,5 +1,6 @@
 {
   "name": "landxml",
+  "type": "module",
   "version": "0.5.2",
   "description": "Parse LandXML surfaces on the modern web.",
   "main": "dist/index.js",
@@ -12,8 +13,8 @@
     "url": "https://github.com/abrman/landxml"
   },
   "scripts": {
-    "dev": "vitest",
-    "test": "vitest run",
+    "dev": "vitest --config ./vitest.config.ts --slow-test-threshold=0",
+    "test": "vitest --config ./vitest.config.ts run",
     "build": "tsup src/index.ts --format cjs,esm --dts",
     "lint": "tsc",
     "ci": "npm run lint && npm run test && npm run build",
@@ -30,14 +31,20 @@
     "@changesets/cli": "^2.26.2",
     "@types/geojson": "^7946.0.13",
     "@types/proj4": "^2.5.5",
+    "@types/sax": "^1.2.7",
     "@types/xml2json": "^0.11.6",
+    "jsdom": "^24.0.0",
     "tsup": "^8.0.0",
     "typescript": "^5.2.2",
-    "vitest": "^0.34.6"
+    "vitest": "^1.6.0"
   },
   "dependencies": {
     "@gltf-transform/core": "^3.9.0",
+    "@vitest/web-worker": "^1.6.0",
+    "easy-web-worker": "^6.2.0",
     "proj4": "^2.9.2",
-    "xml-js": "^1.6.11"
+    "sax": "^1.3.0",
+    "xml-js": "^1.6.11",
+    "xml2json": "^0.12.0"
   }
 }

package.json

@@ -0,0 +1,19 @@
+import { describe, expect, it, vi } from "vitest";
+import EasyWebWorker, { createEasyWebWorker } from "easy-web-worker";
+
+// Doesn't support multi-threading
+const fakeCreateEasyWebWorker = (callback: (onMessage: any) => void) => ({
+  send: (payload: any) =>
+    new Promise(async (resolve, reject) => {
+      const onMessage = (callback: ({ payload, resolve }: { payload: any; resolve: any }) => void) => {
+        callback({ payload, resolve });
+      };
+      callback({ onMessage });
+    }),
+});
+
+vi.mock("easy-web-worker", () => {
+  return {
+    createEasyWebWorker: vi.fn().mockImplementation(fakeCreateEasyWebWorker),
+  };
+});

src/easyWebWorkerMock.ts

@@ -1,9 +1,10 @@
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import parseXML from "./private/parse-xml";
 import getGlb from "./private/get-glb";
 import getContours, { linesToPolyLines, contourElevations, contourLineOnFace } from "./private/get-contours";
 import toGeojsonContours from "./public/to-geojson-contours";
 import reprojectGeoJson from "./public/reproject-geojson";
+import fs from "fs";
 
 const example_single_surface_landxml_with_sourcedata_breakline = `<?xml version="1.0"?>
 <LandXML xmlns="http://www.landxml.org/schema/LandXML-1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.landxml.org/schema/LandXML-1.2 http://www.landxml.org/schema/LandXML-1.2/LandXML-1.2.xsd" date="2023-11-20" time="12:40:44" version="1.2" language="English" readOnly="false">
@@ -231,4 +232,20 @@ describe("Parse LandXMLs", () => {
       [0.4, 0.6, 0.8, 1, 1.2, 1.4, 1.6, 1.8, 2].map(twoDecimalNumberString)
     );
   });
+
+  it("Large file test", async () => {
+    const n = 5; // size of the landXML to process
+    const landXmlString = fs.readFileSync(`./src/test_assets/S${n}.xml`, { encoding: "utf-8" });
+    let rawContours = await toGeojsonContours(landXmlString, 1);
+    const projection =
+      "+proj=utm +zone=16 +ellps=GRS80 +towgs84=-0.9738,1.9453,0.5486,-1.3357e-07,-4.872e-08,-5.507e-08,0 +units=m +no_defs +type=crs"; // any
+
+    if (rawContours[0]) {
+      const geojson = reprojectGeoJson(rawContours[0].geojson, projection, "WGS84", true);
+      // fs.writeFileSync(`./src/test_assets/S${n}_result.json`, JSON.stringify(geojson));
+    }
+  });
+  {
+    timeout: -1;
+  }
 });

src/index.test.ts

@@ -1,5 +1,129 @@
 import type { Feature, FeatureCollection, LineString, Position } from "geojson";
 import type { ParsedSurface } from "./parse-xml";
+import { createEasyWebWorker } from "easy-web-worker";
+
+const contoursWorker = createEasyWebWorker<
+  {
+    triangles: [x: number, y: number, z: number][][];
+    elevation: number;
+  },
+  {
+    elevation: number;
+    polylines: [number, number][][];
+  }
+>(
+  ({ onMessage }) => {
+    const contourLineOnFace = (face: [x: number, y: number, z: number][], z: number) => {
+      let vertsAtElevation = 0;
+      let line: [x: number, z: number][] = [];
+      for (let i = 0; i < face.length; i++) {
+        let vertex1 = face[i] as [x: number, y: number, z: number];
+        let vertex2 = face[(i + 1) % face.length] as [x: number, y: number, z: number];
+        if (vertex1[2] === z) vertsAtElevation++;
+
+        if (
+          ((vertex1[2] <= z && vertex2[2] >= z) || (vertex1[2] >= z && vertex2[2] <= z)) &&
+          !Number.isNaN((z - vertex1[2]) / (vertex2[2] - vertex1[2]))
+        ) {
+          let t = (z - vertex1[2]) / (vertex2[2] - vertex1[2]);
+          line.push([vertex1[0] + t * (vertex2[0] - vertex1[0]), vertex1[1] + t * (vertex2[1] - vertex1[1])]);
+        }
+      }
+
+      // If an edge is going to be detected by two triangles, prioritize the triangle with 3rd vertex at lower elevation
+      if (vertsAtElevation >= 2 && face.map((f) => f[2]).reduce((a, b) => a + b) > z * face.length) return undefined;
+
+      // Prevent zero length lines
+      if (
+        line.length === 2 &&
+        (line[0] as any)[0] === (line[1] as any)[0] &&
+        (line[0] as any)[1] === (line[1] as any)[1]
+      )
+        return undefined;
+
+      if (line.length > 2) {
+        line = [...new Set(line.map((v) => JSON.stringify(v)))].map((s) => JSON.parse(s));
+      }
+      return line.length > 0 ? (line as [[x: number, z: number], [x: number, z: number]]) : undefined;
+    };
+
+    const linesToPolyLines = (lineSegments: [[number, number], [number, number]][]) => {
+      if (!Array.isArray(lineSegments) || lineSegments?.length === 0) {
+        return [];
+        // throw new Error("Invalid input: Please provide a non-empty array of line segments.");
+      }
+
+      const segmentsMapIndexes: { [coordinateKey: string]: number[] } = {};
+      const polylines: [number, number][][] = [];
+      const parsedSegmentIndexes: number[] = [];
+
+      const lineSegmentStrings = lineSegments.map((v) => v.map((c) => c.join(","))) as [string, string][];
+
+      lineSegmentStrings.forEach(([start, end], i) => {
+        segmentsMapIndexes[start] = segmentsMapIndexes[start] ? [...(segmentsMapIndexes[start] || []), i] : [i];
+        segmentsMapIndexes[end] = segmentsMapIndexes[end] ? [...(segmentsMapIndexes[end] || []), i] : [i];
+      });
+
+      for (let i = 0; i < lineSegmentStrings.length; i++) {
+        if (parsedSegmentIndexes.includes(i)) continue;
+
+        parsedSegmentIndexes.push(i);
+
+        let [start, end]: (string | null)[] = lineSegmentStrings[i] as [string, string];
+        let polyline = [start, end];
+
+        while (start && segmentsMapIndexes[start]) {
+          const nextLineIndex: number | undefined = segmentsMapIndexes[start]?.find(
+            (lineIndex) => !parsedSegmentIndexes.includes(lineIndex)
+          );
+          if (nextLineIndex) {
+            parsedSegmentIndexes.push(nextLineIndex);
+            const nextLineSegment = lineSegmentStrings[nextLineIndex] as [string, string];
+            const nextLineSegmentPointIndex: number = nextLineSegment[0] === start ? 1 : 0;
+            const newPoint = nextLineSegment[nextLineSegmentPointIndex] as string;
+            polyline.unshift(newPoint);
+            start = newPoint;
+          } else {
+            start = null;
+          }
+        }
+
+        while (end && segmentsMapIndexes[end]) {
+          const nextLineIndex: number | undefined = segmentsMapIndexes[end]?.find(
+            (lineIndex) => !parsedSegmentIndexes.includes(lineIndex)
+          );
+          if (nextLineIndex) {
+            parsedSegmentIndexes.push(nextLineIndex);
+            const nextLineSegment = lineSegmentStrings[nextLineIndex] as [string, string];
+            const nextLineSegmentPointIndex: number = nextLineSegment[0] === end ? 1 : 0;
+            const newPoint = nextLineSegment[nextLineSegmentPointIndex] as string;
+            polyline.push(newPoint);
+            end = newPoint;
+          } else {
+            end = null;
+          }
+        }
+        polylines.push(polyline.map((coord) => coord.split(",").map((v) => parseFloat(v)) as [number, number]));
+      }
+      return polylines;
+    };
+
+    onMessage((message) => {
+      const { triangles, elevation } = message.payload;
+      const linesAtElevationE = triangles.reduce((prev, curr) => {
+        const line = contourLineOnFace(curr, elevation);
+        if (line) prev.push(line);
+        return prev;
+      }, [] as [[x: number, z: number], [x: number, z: number]][]);
+
+      message.resolve({
+        elevation,
+        polylines: linesToPolyLines(linesAtElevationE),
+      });
+    });
+  },
+  { maxWorkers: 10 }
+);
 
 const contourLineOnFace = (face: [x: number, y: number, z: number][], z: number) => {
   let vertsAtElevation = 0;
@@ -32,8 +156,9 @@ const contourLineOnFace = (face: [x: number, y: number, z: number][], z: number)
 };
 
 const linesToPolyLines = (lineSegments: [[number, number], [number, number]][]) => {
-  if (!Array.isArray(lineSegments) || lineSegments.length === 0) {
-    throw new Error("Invalid input: Please provide a non-empty array of line segments.");
+  if (!Array.isArray(lineSegments) || lineSegments?.length === 0) {
+    return [];
+    // throw new Error("Invalid input: Please provide a non-empty array of line segments.");
   }
 
   const segmentsMapIndexes: { [coordinateKey: string]: number[] } = {};
@@ -150,23 +275,10 @@ const getContours = async (data: ParsedSurface, interval: number = 2) => {
 
   const elevations = contourElevations(minElevation, maxElevation, interval);
 
-  const elevationPolylines = elevations.map((e) => {
-    const linesAtElevationE = triangles.reduce((prev, curr) => {
-      const line = contourLineOnFace(curr, e);
-      if (line) prev.push(line);
-      return prev;
-    }, [] as [[x: number, z: number], [x: number, z: number]][]);
-
-    const polylinesAtElevationE = linesToPolyLines(linesAtElevationE);
-    if (e === 442) {
-      console.log("linesAtElevationE", JSON.stringify(linesAtElevationE));
-      console.log("polylinesAtElevationE", JSON.stringify(polylinesAtElevationE));
-    }
-    return {
-      elevation: e,
-      polylines: polylinesAtElevationE,
-    };
-  });
+  const elevationPolylines: {
+    elevation: number;
+    polylines: [number, number][][];
+  }[] = await Promise.all(elevations.map((elevation) => (contoursWorker.send as any)({ triangles, elevation })));
 
   return constructGeojson(elevationPolylines);
 };

src/private/get-contours.ts

@@ -0,0 +1,101 @@
+interface LandXML {
+  LandXML: {
+    attr: {
+      xmlns: string;
+      "xmlns:xsi": string;
+      "xsi:schemaLocation": string;
+      date: string;
+      time: string;
+      version: string;
+      language: string;
+      readOnly: string;
+    };
+    Units:
+      | {
+          Imperial: {
+            attr: {
+              areaUnit: string;
+              linearUnit: string;
+              volumeUnit: string;
+              temperatureUnit: string;
+              pressureUnit: string;
+              diameterUnit: string;
+              angularUnit: string;
+              directionUnit: string;
+            };
+          };
+        }
+      | {
+          Metric: {
+            attr: {
+              areaUnit: string;
+              linearUnit: string;
+              volumeUnit: string;
+              temperatureUnit: string;
+              pressureUnit: string;
+              diameterUnit: string;
+              angularUnit: string;
+              directionUnit: string;
+            };
+          };
+        };
+    Project: {
+      attr: {
+        name: string;
+      };
+    };
+    CoordinateSystem: {
+      attr: {
+        desc: string;
+        ogcWktCode: string;
+      };
+    };
+    Application: {
+      attr: {
+        name: string;
+        desc: string;
+        manufacturer: string;
+        version: string;
+        timeStamp: string;
+      };
+    };
+    Surfaces: {
+      Surface: Surface | Surface[];
+    };
+  };
+}
+
+interface Surface {
+  attr: {
+    name: string;
+    desc: string;
+  };
+  SourceData?: any;
+  Definition: {
+    attr: {
+      surfType: string;
+    };
+    Pnts: {
+      P: SurfacePoint[];
+    };
+    Faces: {
+      F: SurfaceFace[];
+    };
+  };
+}
+
+type SurfacePoint = {
+  attr: {
+    id: string;
+  };
+  content: string;
+};
+
+type SurfaceFace =
+  | {
+      attr: {
+        i?: string;
+      };
+      content: string;
+    }
+  | string;