refactor, use Sequency for streams

Author: elisherer

.github/workflows/javascript-json-transform-test.yml

@@ -0,0 +1,33 @@
+name: JavaScript json-transform Test
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - javascript/json-transform/**
+
+# cancel previous tests if new commit is pushed to PR branch
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: npm
+          cache-dependency-path: ./javascript/json-transform/package-lock.json
+
+      - name: Install dependencies
+        working-directory: ./javascript/json-transform
+        run: npm ci
+
+      - name: Run tests
+        working-directory: ./javascript/json-transform
+        run: npm test

javascript/json-transform-core/package-lock.json

@@ -40,10 +40,10 @@
   },
   "homepage": "https://github.com/nlighten-oss/json-transform#readme",
   "peerDependencies": {
-    "@nlighten/json-schema-utils": "^1.0.0"
+    "@nlighten/json-schema-utils": "^1.0.2"
   },
   "devDependencies": {
-    "@nlighten/json-schema-utils": "^1.0.0",
+    "@nlighten/json-schema-utils": "^1.0.2",
     "microbundle": "^0.15.1",
     "prettier": "3.1.1",
     "typescript": "^5.3.3",

javascript/json-transform-core/package.json

@@ -40,12 +40,12 @@
   },
   "homepage": "https://github.com/nlighten-oss/json-transform#readme",
   "peerDependencies": {
-    "@nlighten/json-schema-utils": "^1.0.0",
+    "@nlighten/json-schema-utils": "^1.0.2",
     "jsonpath": "^1.1.1",
     "uuid": "^10.0.0"
   },
   "devDependencies": {
-    "@nlighten/json-schema-utils": "^1.0.0",
+    "@nlighten/json-schema-utils": "^1.0.2",
     "@types/jsonpath": "^0.2.4",
     "@types/uuid": "^10.0.0",
     "jsonpath": "^1.1.1",
@@ -54,5 +54,9 @@
     "typescript": "^5.3.3",
     "uuid": "^10.0.0",
     "vitest": "1.2.2"
+  },
+  "dependencies": {
+    "bignumber.js": "^9.1.2",
+    "sequency": "^0.20.0"
   }
 }

javascript/json-transform/package-lock.json

@@ -0,0 +1,67 @@
+import FunctionContext from "./functions/common/FunctionContext";
+import Sequence, { asSequence, emptySequence } from "sequency";
+
+class JsonElementStreamer {
+  private readonly context: FunctionContext;
+  private readonly transformed: boolean;
+  private readonly value?: any[];
+  private readonly _stream?: Sequence<any>;
+
+  private constructor(context: FunctionContext, value: any[] | Sequence<any>, transformed: boolean) {
+    this.context = context;
+    this.transformed = transformed;
+    if (Array.isArray(value)) {
+      this.value = value;
+      this._stream = undefined;
+    } else {
+      this.value = undefined;
+      this._stream = value;
+    }
+  }
+
+  public knownAsEmpty() {
+    return this.value && this.value.length == 0;
+  }
+
+  public stream(skip: number = 0, limit: number = -1) {
+    if (this._stream != null) {
+      const skipped = skip > 0 ? this._stream.drop(skip) : this._stream;
+      return limit > -1 ? skipped.take(limit) : skipped;
+    }
+    if (this.value == null) {
+      return emptySequence();
+    }
+    let valueStream = asSequence(this.value);
+    if (skip > 0) {
+      valueStream = valueStream.drop(skip);
+    }
+    if (limit > -1) {
+      valueStream = valueStream.take(limit);
+    }
+    if (!this.transformed) {
+      valueStream = valueStream.map(el => this.context.transform(el));
+    }
+    return valueStream;
+  }
+
+  public static fromJsonArray(context: FunctionContext, value: any[], transformed: boolean) {
+    return new JsonElementStreamer(context, value, transformed);
+  }
+
+  public static fromTransformedStream(context: FunctionContext, stream: Sequence<any>) {
+    return new JsonElementStreamer(context, stream, true);
+  }
+
+  public toJsonArray() {
+    if (this.value) {
+      return this.value;
+    }
+    const ja: any[] = [];
+    if (this._stream) {
+      this._stream.forEach(item => ja.push(item));
+    }
+    return ja;
+  }
+}
+
+export default JsonElementStreamer;

javascript/json-transform/package.json

@@ -1,5 +1,7 @@
 import { v4 as uuidv4 } from 'uuid';
 import DocumentContext from "./DocumentContext";
+import {BigDecimal} from "./functions/common/FunctionHelpers";
+import {areSimilar} from "@nlighten/json-schema-utils";
 
 const JSONPATH_ROOT = "$",
   JSONPATH_ROOT_ESC = "\\$",
@@ -29,15 +31,17 @@ const numberCompare = (a: number, b: number) => {
   return a < b ? -1 : (a === b ? 0 : 1);
 }
 
+const numberType = (a: any) => typeof a === 'number' || typeof a === 'bigint' || a instanceof BigDecimal;
+
 const compareTo = (a: any, b: any) => {
   if (Array.isArray(a) && Array.isArray(b)) {
     return numberCompare(a.length, b.length);
   } else if (a && b && typeof a === 'object' && typeof b === 'object') {
     return numberCompare(Object.keys(a).length, Object.keys(b).length);
   } else if (typeof a === 'string' && typeof b === 'string') {
     return a.localeCompare(b);
-  } else if (typeof a === 'number' && typeof b === 'number') {
-    return numberCompare(a, b);
+  } else if (numberType(a) && numberType(b)) {
+    return BigDecimal(a).comparedTo(BigDecimal(b));
   } else if (typeof a === 'boolean' && typeof b === 'boolean') {
     return a === b ? 0 : (a ? 1 : -1);
   } else if (isNullOrUndefined(a) && !isNullOrUndefined(b)) {
@@ -143,29 +147,42 @@ const lenientJsonParse = (input: string) => {
 
 const BIGINT_ZERO = BigInt(0);
 const isTruthy = (value: any, javascriptStyle?: boolean) => {
-  if (Array.isArray(value)) {
-    return value.length > 0;
-  } else if (value && typeof value === 'object') {
-    return Object.keys(value).length > 0;
-  }
   if (typeof value === 'boolean') {
     return value;
   } else if (typeof value === 'number') {
     return value != 0;
   } else if (typeof value === 'bigint') {
     return value !== BIGINT_ZERO;
+  } else if (value instanceof BigDecimal) {
+    return !value.isZero();
   } else if (typeof value === 'string') {
-    return javascriptStyle ? value !== '' : value === 'true';
+    return javascriptStyle ? Boolean(value) : value.toLowerCase() === 'true';
+  } else if (Array.isArray(value)) {
+    return value.length > 0;
+  } else if (value && typeof value === 'object') {
+    return Object.keys(value).length > 0;
   }
   return !isNullOrUndefined(value);
 }
 
+const isEqual = (value: any, other: any): boolean => {
+  if (value === other) {
+    return true;
+  }
+  if (numberType(value) && numberType(other)) {
+    return BigDecimal(value).eq(BigDecimal(other));
+  }
+  return areSimilar(value, other);
+}
+
+
 export {
   isNullOrUndefined,
   createPayloadResolver,
   getAsString,
   compareTo,
   getDocumentContext,
   lenientJsonParse,
-  isTruthy
+  isTruthy,
+  isEqual
 };

javascript/json-transform/src/JsonElementStreamer.ts

@@ -3,6 +3,7 @@ import {JsonTransformerFunction} from "./JsonTransformerFunction";
 import {ParameterResolver} from "./ParameterResolver";
 import {createPayloadResolver, isNullOrUndefined} from "./JsonHelpers";
 import transformerFunctions, { TransformerFunctions } from "./transformerFunctions";
+import JsonElementStreamer from "./JsonElementStreamer";
 
 class JsonTransformer implements Transformer {
 
@@ -32,13 +33,16 @@ class JsonTransformer implements Transformer {
 
   fromJsonPrimitive(definition: any, resolver: ParameterResolver, allowReturningStreams: boolean) : any {
     if (typeof definition !== 'string') {
-      return definition;
+      return definition ?? null;
     }
     try {
       // test for inline function (e.g. $$function:...)
       const match = this.transformerFunctions.matchInline(definition, resolver, this.JSON_TRANSFORMER);
       if (match != null) {
-        // TODO: add streams support
+        const matchResult = match.getResult();
+        if (matchResult instanceof JsonElementStreamer) {
+          return allowReturningStreams ? matchResult : matchResult.toJsonArray();
+        }
         return match.getResult();
       }
       // jsonpath / context
@@ -52,7 +56,10 @@ class JsonTransformer implements Transformer {
   fromJsonObject(definition: any, resolver: ParameterResolver, allowReturningStreams: boolean) : any {
     const match = this.transformerFunctions.matchObject(definition, resolver, this.JSON_TRANSFORMER);
     if (match != null) {
-      // TODO: add streams support
+      const res = match.getResult();
+      if (res instanceof JsonElementStreamer) {
+        return allowReturningStreams ? res : res.toJsonArray();
+      }
       return match.getResult();
     }
 

javascript/json-transform/src/JsonHelpers.ts

@@ -108,7 +108,8 @@ describe("JsonTransformer", () => {
     });
   });
 
-  test("InputExtractorSpreadDontRemoveByNull", () => {
+  // skipped since it doesn't work the same in javascript (null are being treated as values)
+  test.skip("InputExtractorSpreadDontRemoveByNull", () => {
     assertTransformation({
       "a": "A",
       "b": "B"