Add Lectern Client to monorepo

README.md

@@ -36,11 +36,11 @@ The modules in the monorepo are organized into three categories:
    * __libraries/__ - Interal modules shared between other apps, libraries, and packages.
    * __packages/__ - Packages published to [NPM](https://npmjs.com) meant to be imported into other TypeScript applications.
 
-| Component                           | Type        | Package Name                   | Path                  | Published Location                                                       | Description                                                                                   |
-| ----------------------------------- | ----------- | ------------------------------ | --------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------- |
-| [Lectern Server](apps/server/README.md)       | Application | server                         | apps/server/          | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application.                                                               |
-| [Lectern Client](https://github.com/overture-stack/js-lectern-client)  | Package     | @overture-stack/js-lectern-client |      | [NPM](https://www.npmjs.com/package/@overturebio-stack/lectern-client)   | TypeScript Client to interact with Lectern Server and perform data validation.                           |
-| [common](libraries/common/README.md) | Library     | common                     | libraries/common/ | N/A                                                                      | Non-specific but commonly reusable utilities. Includes shared Error classes. |
+| Component                                    | Type        | Package Name                   | Path                  | Published Location                                                       | Description                                                                                                                                                                                         |
+| -------------------------------------------- | ----------- | ------------------------------ | --------------------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [Lectern Server](apps/server/README.md)      | Application | server                         | apps/server/          | [GHCR](https://github.com/overture-stack/lectern/pkgs/container/lectern) | Lectern Server web application.                                                                                                                                                                     |
+| [Lectern Client](packages/client/README.md)  | Package     | @overture-stack/lectern-client | packages/client       | [NPM](https://www.npmjs.com/package/@overturebio-stack/lectern-client)   | TypeScript Client to interact with Lectern Server and perform data validation.                                                                                                                      |
+| [common](libraries/common/README.md)         | Library     | common                         | libraries/common/     | N/A                                                                      | Non-specific but commonly reusable utilities. Includes shared Error classes.                                                                                                                        |
 | [dictionary](libraries/dictionary/README.md) | Library     | dictionary                     | libraries/dictionary/ | N/A                                                                      | Dictionary meta-schema definition, includes TS types, and Zod schemas. This also exports all utilities for getting the diff of two dictionaries, and for validating data records with a Dictionary. |
 
 ## Developer Instructions

libraries/common/package.json

@@ -5,9 +5,12 @@
 	"main": "dist/index.js",
 	"scripts": {
 		"build": "pnpm build:clean && tsc",
-		"build:clean": "rm -rf dist/ && mkdir dist"
+		"build:clean": "rimraf dist/ && mkdir dist"
 	},
 	"keywords": [],
 	"author": "Ontario Institute for Cancer Research",
-	"license": "AGPL-3.0"
+	"license": "AGPL-3.0",
+	"devDependencies": {
+		"rimraf": "^3.0.2"
+	}
 }

libraries/common/tsconfig.json

@@ -1,6 +1,6 @@
 {
 	"compilerOptions": {
-		"target": "ES5",
+		"target": "ESNext",
 		"lib": ["ESNext"],
 		"module": "CommonJS",
 		"moduleResolution": "node",

libraries/dictionary/package.json

@@ -5,7 +5,7 @@
 	"main": "dist/index.js",
 	"scripts": {
 		"build": "pnpm build:clean && tsc",
-		"build:clean": "rm -rf dist/ && mkdir dist",
+		"build:clean": "rimraf dist/ && mkdir dist",
 		"test": "nyc mocha"
 	},
 	"keywords": [],
@@ -18,6 +18,7 @@
 		"zod": "^3.21.4"
 	},
 	"devDependencies": {
-		"@types/lodash": "^4.14.195"
+		"@types/lodash": "^4.14.195",
+		"rimraf": "^3.0.2"
 	}
 }

libraries/dictionary/tsconfig.json

@@ -1,6 +1,6 @@
 {
 	"compilerOptions": {
-		"target": "ES5",
+		"target": "ESNext",
 		"lib": ["ESNext"],
 		"module": "CommonJS",
 		"moduleResolution": "node",

packages/client/.mocharc.json

@@ -0,0 +1,5 @@
+{
+	"extension": ["ts"],
+	"require": "ts-node/register",
+	"spec": "test/**/*.spec.ts"
+}

packages/client/README.md

@@ -0,0 +1,12 @@
+# Lectern TypeScript Client
+
+## Features:
+- Runs different restrictions validations: regex, range, scripts, required fields, type checks, etc.
+- Transforms the data from string to their proper type.
+- Report validation errors.
+- Fetch dictionaries from the configured lectern service. 
+- Provide typed definitions for the dictionary object.
+- Analyze dictionary versions diff.
+
+## Usage examples:
+- icgc-argo/argo-clinical [https://github.com/icgc-argo/argo-clinical]

packages/client/package.json

@@ -0,0 +1,52 @@
+{
+	"name": "@overture-stack/lectern-client",
+	"version": "1.5.0",
+	"files": [
+		"dist/"
+	],
+	"main": "./dist/index.js",
+	"typings": "./dist/index.d.ts",
+	"description": "TypeScript client to interact with Lectern servers and perform data validation versus Lectern dictionaries.",
+	"scripts": {
+		"build": "rimraf dist && tsc",
+		"test": "nyc mocha --exit --timeout 5000 -r ts-node/register test/**.spec.ts",
+		"lint": "tslint -c tslint.json -e node_modules -p tsconfig.json"
+	},
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/overture-stack/lectern.git"
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"license": "AGPL-3.0",
+	"devDependencies": {
+		"@types/chai": "^4.2.16",
+		"@types/deep-freeze": "^0.1.2",
+		"@types/lodash": "^4.14.195",
+		"@types/mocha": "^8.2.2",
+		"@types/node": "^12.0.10",
+		"@types/node-fetch": "^2.5.10",
+		"chai": "^4.3.4",
+		"husky": "^6.0.0",
+		"mocha": "^8.3.2",
+		"prettier": "^2.2.1",
+		"pretty-quick": "^3.1.0",
+		"rimraf": "^3.0.2",
+		"ts-node": "^9.1.1",
+		"tslint": "^6.1.3",
+		"typedoc": "^0.17.7",
+		"typescript": "^5.1.6"
+	},
+	"dependencies": {
+		"cd": "^0.3.3",
+		"common": "workspace:^",
+		"deep-freeze": "^0.0.1",
+		"lodash": "^4.17.21",
+		"node-fetch": "^2.6.1",
+		"node-worker-threads-pool": "^1.4.3",
+		"promise-tools": "^2.1.0",
+		"winston": "^3.3.3"
+	},
+	"author": "Ontario Institute for Cancer Research"
+}

packages/client/src/change-analyzer.ts

@@ -0,0 +1,254 @@
+/*
+ * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved
+ *
+ * This program and the accompanying materials are made available under the terms of
+ * the GNU Affero General Public License v3.0. You should have received a copy of the
+ * GNU Affero General Public License along with this program.
+ *  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import { restClient } from './schema-rest-client';
+import {
+	SchemasDictionaryDiffs,
+	FieldChanges,
+	FieldDiff,
+	Change,
+	ChangeAnalysis,
+	ChangeTypeName,
+	RestrictionChanges,
+	FieldDefinition,
+} from './schema-entities';
+
+const isFieldChange = (obj: any): obj is Change => {
+	return obj.type !== undefined;
+};
+
+const isNestedChange = (obj: any): obj is { [field: string]: FieldChanges } => {
+	return obj.type === undefined;
+};
+
+const isRestrictionChange = (obj: any): obj is { [field: string]: FieldChanges } => {
+	return obj.type === undefined;
+};
+
+export const fetchDiffAndAnalyze = async (serviceUrl: string, name: string, fromVersion: string, toVersion: string) => {
+	const changes = await restClient.fetchDiff(serviceUrl, name, fromVersion, toVersion);
+	return analyzeChanges(changes);
+};
+
+export const analyzeChanges = (schemasDiff: SchemasDictionaryDiffs): ChangeAnalysis => {
+	const analysis: ChangeAnalysis = {
+		fields: {
+			addedFields: [],
+			renamedFields: [],
+			deletedFields: [],
+		},
+		isArrayDesignationChanges: [],
+		restrictionsChanges: {
+			codeList: {
+				created: [],
+				deleted: [],
+				updated: [],
+			},
+			regex: {
+				updated: [],
+				created: [],
+				deleted: [],
+			},
+			required: {
+				updated: [],
+				created: [],
+				deleted: [],
+			},
+			script: {
+				updated: [],
+				created: [],
+				deleted: [],
+			},
+			range: {
+				updated: [],
+				created: [],
+				deleted: [],
+			},
+		},
+		metaChanges: {
+			core: {
+				changedToCore: [],
+				changedFromCore: [],
+			},
+		},
+		valueTypeChanges: [],
+	};
+
+	for (const field of Object.keys(schemasDiff)) {
+		const fieldChange: FieldDiff = schemasDiff[field];
+		if (fieldChange) {
+			const fieldDiff = fieldChange.diff;
+			// if we have type at first level then it's a field add/delete
+			if (isFieldChange(fieldDiff)) {
+				categorizeFieldChanges(analysis, field, fieldDiff);
+			}
+
+			if (isNestedChange(fieldDiff)) {
+				if (fieldDiff.meta) {
+					categorizeMetaChagnes(analysis, field, fieldDiff.meta);
+				}
+
+				if (fieldDiff.restrictions) {
+					categorizeRestrictionChanges(analysis, field, fieldDiff.restrictions, fieldChange.after);
+				}
+
+				if (fieldDiff.isArray) {
+					categorizeFieldArrayDesignationChange(analysis, field, fieldDiff.isArray);
+				}
+
+				if (fieldDiff.valueType) {
+					categorizerValueTypeChange(analysis, field, fieldDiff.valueType);
+				}
+			}
+		}
+	}
+
+	return analysis;
+};
+
+const categorizeFieldArrayDesignationChange = (
+	analysis: ChangeAnalysis,
+	field: string,
+	changes: { [field: string]: FieldChanges } | Change,
+) => {
+	// changing isArray designation is a relevant change for all cases except if it is created and set to false
+	if (!(changes.type === 'created' && changes.data === false)) {
+		analysis.isArrayDesignationChanges.push(field);
+	}
+};
+
+const categorizerValueTypeChange = (
+	analysis: ChangeAnalysis,
+	field: string,
+	changes: { [field: string]: FieldChanges } | Change,
+) => {
+	analysis.valueTypeChanges.push(field);
+};
+
+const categorizeRestrictionChanges = (
+	analysis: ChangeAnalysis,
+	field: string,
+	restrictionsChange: { [field: string]: FieldChanges } | Change,
+	fieldDefinitionAfter?: FieldDefinition,
+) => {
+	const restrictionsToCheck = ['regex', 'script', 'required', 'codeList', 'range'];
+
+	// additions or deletions of a restriction object as whole (i.e. contains 1 or many restrictions within the 'data')
+	if (restrictionsChange.type) {
+		const createOrAddChange = restrictionsChange as Change;
+		const restrictionsData = createOrAddChange.data as any;
+
+		for (const k of restrictionsToCheck) {
+			if (restrictionsData[k]) {
+				analysis.restrictionsChanges[k as keyof RestrictionChanges][restrictionsChange.type as ChangeTypeName].push({
+					field: field,
+					definition: restrictionsData[k],
+				} as any);
+			}
+		}
+		return;
+	}
+
+	// in case 'restrictions' key was already there but we modified its contents
+	const restrictionUpdate = restrictionsChange as { [field: string]: FieldChanges };
+	for (const k of restrictionsToCheck) {
+		if (restrictionUpdate[k]) {
+			const change = restrictionUpdate[k] as Change;
+			// we need the '|| change'  in case of nested attributes like ranges
+			/*
+      "diff": {
+        "restrictions": {
+          "range": {
+            "exclusiveMin": {
+              "type": "deleted",
+              "data": 0
+            },
+            "max": {
+              "type": "updated",
+              "data": 200000
+            },
+            "min": {
+              "type": "created",
+              "data": 0
+            }
+          }
+        }
+      }
+      */
+			if (k == 'range' && !change.type) {
+				// if the change is nested (type is at min max level) then the boundries were updated only : ex:
+				/*
+         change = {
+           "max" : {
+             type: "updated"
+             data: "..."
+           },
+           "exclusiveMin": {
+             type: "deleted"
+             data ..
+           }
+         }
+        */
+				const def: any = {};
+				if (Object.keys(change).some((k) => k == 'max' || k == 'min' || k == 'exclusiveMin' || k == 'exclusiveMax')) {
+					analysis.restrictionsChanges[k]['updated'].push({
+						field: field,
+						// we push the whole range definition since it doesnt make sense to just
+						// push one boundary.
+						definition: fieldDefinitionAfter?.restrictions?.range,
+					});
+				}
+				return;
+			}
+			const definition = change.data || change;
+			analysis.restrictionsChanges[k as keyof RestrictionChanges][change.type as ChangeTypeName].push({
+				field: field,
+				definition,
+			} as any);
+		}
+	}
+};
+
+const categorizeFieldChanges = (analysis: ChangeAnalysis, field: string, changes: Change) => {
+	const changeType = changes.type;
+	if (changeType == 'created') {
+		analysis.fields.addedFields.push({
+			name: field,
+			definition: changes.data,
+		});
+	} else if (changeType == 'deleted') {
+		analysis.fields.deletedFields.push(field);
+	}
+};
+
+const categorizeMetaChagnes = (
+	analysis: ChangeAnalysis,
+	field: string,
+	metaChanges: { [field: string]: FieldChanges } | Change,
+) => {
+	// **** meta changes - core ***
+	if (metaChanges?.data?.core === true) {
+		const changeType = metaChanges.type;
+		if (changeType === 'created' || changeType === 'updated') {
+			analysis.metaChanges?.core.changedToCore.push(field);
+		} else if (changeType === 'deleted') {
+			analysis.metaChanges?.core.changedFromCore.push(field);
+		}
+	}
+};