[skip ci] usage examples and docs

Author: dem1fo

README.md

@@ -20,42 +20,8 @@ This repository is designed to be used with file-based data programmatically; fo
 
 ## Usage
 
-```ts
-const REGION = 'NWS';
-const CONFIG_PATH = join(__dirname, './config.toml');
-
-const INPUT_PATH = join(__dirname, 'files', 'input_data.csv');
-const OUTPUT_PATH = join(__dirname, 'output', 'output.csv');
-const VALIDATION_ERRORS_PATH = join(__dirname, 'output', 'validation_errors.csv');
-
-// load configuration file
-const configLoadResult = loadConfig(CONFIG_PATH, REGION);
-if (!configLoadResult.success) throw new Error('unable to load configuration file.');
-
-const config = configLoadResult.config;
-
-// validate the input file against all configured validation rules.
-const preprocessResult = await preprocessFile({
-  config: config,
-  inputFilePath: INPUT_PATH,
-  errorFileOutputPath: VALIDATION_ERRORS_PATH,
-  limit: 2,
-});
-
-if (!preprocessResult.isValid)
-  throw new Error('Validation errors found in input file, review error file output.');
-
-// validate the input file against all configured validation rules.
-const processFileResult = await processFile({
-  config: config,
-  inputFilePath: INPUT_PATH,
-  outputPath: OUTPUT_PATH,
-  hasherFactory: makeHasher,
-  limit: 2,
-});
-// print the result, save the result, etc.
-console.dir(processFileResult, { depth: 3 });
-```
+1. If using file-based data, see [this example](./examples/file_based_usage.ts)
+2. If using programmatic data (i.e. like an array in memory), see [this example](./examples/programmatic_usage.ts)
 
 ## 🧪 Data Processing Pipeline (file-based data)
 

examples/example_algorithm/_generic_hasher.ts

@@ -0,0 +1,49 @@
+/*
+ * This file is part of Building Blocks CommonID Tool
+ * Copyright (c) 2024 WFP
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { joinFieldsForHash, cleanValueList, extractAlgoColumnsFromObject, BaseHasher } from 'common-identifier-algorithm-shared';
+import type { Config, Validator, makeHasherFunction } from 'common-identifier-algorithm-shared';
+
+class GenericHasher extends BaseHasher {
+    constructor(config: Config.Options["algorithm"]) {
+        super(config);
+    }
+
+    private composeHashSource = (extractedObj: Config.AlgorithmColumns) => {
+        let staticValues = extractedObj.static;
+        let concatenated = joinFieldsForHash(cleanValueList(staticValues));
+        return concatenated;
+    }
+
+    generateHashForObject(obj: Validator.InputData["row"]) {
+        const extractedObj = extractAlgoColumnsFromObject(this.config.columns, obj);
+        const toBeHashed = this.composeHashSource(extractedObj);
+        return {
+            hashed_id: toBeHashed.length > 0 ? this.generateHashForValue(toBeHashed) : "",
+        }
+    }
+}
+
+export const REGION = "ANY";
+export const makeHasher: makeHasherFunction = (config: Config.Options["algorithm"]) => {
+    switch (config.hash.strategy.toLowerCase()) {
+        case 'sha256':
+            return new GenericHasher(config);
+        default:
+            throw new Error(`Unknown hash strategy in config: '${config.hash.strategy}'`);
+    }
+}

examples/example_algorithm/config.toml

@@ -0,0 +1,56 @@
+[meta]
+region="ANY"
+version="1.0.0"
+signature="a5da7472f72da37a149ca37668c308ff"
+
+[source]
+columns = [
+    { name = "ID",       alias = "id"   },
+    { name = "Column 2", alias = "col2" },
+    { name = "Column 3", alias = "col3" },
+]
+
+[validations]
+
+"*" = [ { op = "max_field_length", value = 200 } ]
+
+id = [
+    { op = "field_type", value = "string" },
+    { op = "regex_match", value = '(\d{11})', message="must be 11 numeric digits" }
+]
+
+[algorithm]
+
+[algorithm.columns]
+process = []
+static = [
+    "id",
+]
+reference = []
+
+[algorithm.hash]
+strategy = "SHA256"
+
+[algorithm.salt]
+source = "STRING"
+value = "{{ some_random_salt_value }}"
+
+[destination]
+columns = [
+    { name = "Common Identifier", alias = "hashed_id" },
+]
+postfix = "-OUTPUT-{{yyyy-MM-dd--HH-mm-ss}}"
+
+[destination_map]
+columns = [
+    { name = "ID",       alias = "id" },
+    { name = "Common Identifier", alias = "hashed_id" }
+]
+postfix = "-MAPPING-{{yyyy-MM-dd--HH-mm-ss}}"
+
+[destination_errors]
+columns = [
+    { name = "Errors",            alias = "errors" },
+    { name = "ID",                alias = "id" },
+]
+postfix = "-ERRORS-{{yyyy-MM-dd--HH-mm-ss}}"

examples/example_algorithm/input_10.csv

@@ -0,0 +1,11 @@
+ID,Column 2,Column 3
+43294300000,bar0,baz0
+38591500000,bar1,baz1
+17386300000,bar2,baz2
+54598700000,bar3,baz3
+56552200000,bar4,baz4
+59893200000,bar5,baz5
+98099300000,bar6,baz6
+43920900000,bar7,baz7
+83307400000,bar8,baz8
+89685800000,bar9,baz9

examples/file_based_usage.ts

@@ -0,0 +1,37 @@
+// REPLACE ALL REFERENCES TO "_generic_hasher" WITH THE DESIRED ALGORITHM IN THE ALGORITHMS DIRECTORY.
+
+import { loadConfig, preprocessFile, processFile } from 'common-identifier-algorithm-shared';
+import { makeHasher, REGION } from './example_algorithm/_generic_hasher';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const CONFIG_PATH = join(__dirname, 'example_algorithm', 'config.toml');
+const INPUT_PATH = join(__dirname, 'example_algorithm', 'input_10.csv');
+const OUTPUT_PATH = join(__dirname, 'output', 'output_10.csv');
+const VALIDATION_ERRORS_PATH = join(__dirname, 'output', 'validation_errors.csv');
+
+// load configuration from file
+const configLoadResult = loadConfig(CONFIG_PATH, REGION);
+if (!configLoadResult.success) throw new Error(`ERROR: Unable to load configuration file >> ${configLoadResult.error}`);
+
+// validate the input file against all configured validation rules.
+const preprocessResult = await preprocessFile({
+  config: configLoadResult.config,
+  inputFilePath: INPUT_PATH,
+  errorFileOutputPath: VALIDATION_ERRORS_PATH,
+});
+
+if (!preprocessResult.isValid)
+  throw new Error('Validation errors found in input file, review error file output.');
+
+// validate the input file against all configured validation rules.
+const processFileResult = await processFile({
+  config: configLoadResult.config,
+  inputFilePath: INPUT_PATH,
+  outputPath: OUTPUT_PATH,
+  hasherFactory: makeHasher,
+});
+// print the result, save the result, etc.
+console.dir(processFileResult, { depth: 3 });

examples/programmatic_usage.ts

@@ -0,0 +1,87 @@
+// REPLACE ALL REFERENCES TO "_generic_hasher" WITH THE DESIRED ALGORITHM IN THE ALGORITHMS DIRECTORY.
+
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { generateHashesForDocument, validateDocument, type CidDocument, type Config } from '../src/index';
+import { SUPPORTED_VALIDATORS } from '../src/validation/Validation';
+import { makeHasher } from './example_algorithm/_generic_hasher';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+/*
+  Construct a config object instructing the algorithm HOW to process the data being passed
+  in. This contains rules related to source schema, target schemas, validation rules, and
+  algorithm specifications. Normally, this will be read from file using loadConfig()
+
+  see ../docs/configuration-files.md for more detail on the relevant config fields.
+*/
+const config: Config.Options = {
+  meta: {
+    region: "UNKONWN", // this must match the shortCode of the algorithm being used
+    version: "",
+    signature: ""
+  },
+  // the schema information for the source data
+  source: {
+    columns: [
+      { name: "ID", alias: "id" },
+      { name: "Column 2", alias: "col2" },
+      { name: "Column 3", alias: "col3" },
+    ]
+  },
+  // [OPTIONAL] validation rules per column: see ../docs/validators.md
+  validations: {
+    id: [
+      { op: SUPPORTED_VALIDATORS.FIELD_TYPE, value: 'string' },
+      { op: SUPPORTED_VALIDATORS.MAX_FIELD_LENGTH, value: 11 }
+      // { op: SUPPORTED_VALIDATORS.LINKED_FIELD, target: "col2" }
+    ]
+  },
+  algorithm: {
+    hash: { strategy: "SHA256" },
+    salt: { source: "STRING", value: "testSalt" },
+    columns: {
+      process: [],
+      reference: [],
+      static: [ "id" ]
+    },
+  },
+  // schema for main output file, skipping for brevity
+  destination: { columns: [] },
+  // schema for mapping output file, skipping for brevity
+  destination_map: { columns: [] },
+  // schema for error files, skipping for brevity
+  destination_errors: { columns: [] }
+}
+
+/*
+  Construct a `document` containing the data to process.
+*/
+const doc: CidDocument = {
+  name: "Input Data",
+  data: [
+    { "id": "43294300000", "col2": "bar0", "col3": "baz0" },
+    { "id": "38591500000", "col2": "bar1", "col3": "baz1" },
+    { "id": "17386300000", "col2": "bar2", "col3": "baz2" },
+  ]
+}
+
+function main() {
+  // validate the input data against all configured validation rules.
+  const validationResult = validateDocument(config, doc, false);
+  if (!validationResult.ok) {
+    console.dir(validationResult.results, { depth: 5 });
+    throw new Error("Data contains validation errors, check input");
+  }
+
+  // initialise the selected algorithm
+  const hasher = makeHasher(config.algorithm);
+  // run the algorithm against the input data
+  const result = generateHashesForDocument(hasher, doc);
+
+  // print the results, save the results, up to you.
+  console.dir(result, { depth: 5 });
+}
+
+main();