[config] refactor config.options into core and file-based configuration types

 - permits omission of optional fields when using the library programmatically
- forces more constraints on config validation when using the UI

Author: dem1fo

examples/example_algorithm/_generic_hasher.ts

@@ -18,7 +18,7 @@ import { joinFieldsForHash, cleanValueList, extractAlgoColumnsFromObject, BaseHa
 import type { Config, Validator, makeHasherFunction } from 'common-identifier-algorithm-shared';
 
 class GenericHasher extends BaseHasher {
-    constructor(config: Config.Options["algorithm"]) {
+    constructor(config: Config.CoreConfiguration["algorithm"]) {
         super(config);
     }
 
@@ -38,7 +38,7 @@ class GenericHasher extends BaseHasher {
 }
 
 export const REGION = "ANY";
-export const makeHasher: makeHasherFunction = (config: Config.Options["algorithm"]) => {
+export const makeHasher: makeHasherFunction = (config: Config.CoreConfiguration["algorithm"]) => {
     switch (config.hash.strategy.toLowerCase()) {
         case 'sha256':
             return new GenericHasher(config);

examples/programmatic_usage.ts

@@ -1,5 +1,5 @@
 // REPLACE ALL REFERENCES TO "_generic_hasher" WITH THE DESIRED ALGORITHM IN THE ALGORITHMS DIRECTORY.
-import { generateHashesForDocument, validateDocument, type CidDocument, type Config } from '../src/index';
+import { generateHashesForDocument, validateConfigCore, validateDocument, type CidDocument, type Config } from '../src/index';
 import { SUPPORTED_VALIDATORS } from '../src/validation/Validation';
 import { makeHasher } from './example_algorithm/_generic_hasher';
 
@@ -10,10 +10,10 @@ import { makeHasher } from './example_algorithm/_generic_hasher';
 
   see ../docs/configuration-files.md for more detail on the relevant config fields.
 */
-const config: Config.Options = {
+const config: Config.CoreConfiguration = {
   meta: {
     // this must match the shortCode of the algorithm being used
-    region: "UNKONWN"
+    region: "UNKNOWN"
   },
   // the schema information for the source data
   source: {
@@ -28,7 +28,6 @@ const config: Config.Options = {
     id: [
       { op: SUPPORTED_VALIDATORS.FIELD_TYPE, value: 'string' },
       { op: SUPPORTED_VALIDATORS.MAX_FIELD_LENGTH, value: 11 }
-      // { op: SUPPORTED_VALIDATORS.LINKED_FIELD, target: "col2" }
     ]
   },
   algorithm: {
@@ -39,13 +38,7 @@ const config: Config.Options = {
       reference: [],
       static: [ "id" ]
     },
-  },
-  // schema for main output file, skipping for brevity
-  destination: { columns: [] },
-  // schema for mapping output file, skipping for brevity
-  destination_map: { columns: [] },
-  // schema for error files, skipping for brevity
-  destination_errors: { columns: [] }
+  }
 }
 
 /*
@@ -61,8 +54,14 @@ const doc: CidDocument = {
 }
 
 function main() {
+  const configValidationResult = validateConfigCore(config, "UNKNOWN");
+  if (!!configValidationResult) {
+    console.log(`ERROR: ${configValidationResult}`);
+    throw new Error("Runtime validation of config failed, check config input");
+  }
+
   // validate the input data against all configured validation rules.
-  const validationResult = validateDocument(config, doc, false);
+  const validationResult = validateDocument({ config: config, decoded: doc, isMapping: false });
   if (!validationResult.ok) {
     console.dir(validationResult.results, { depth: 5 });
     throw new Error("Data contains validation errors, check input");

scratch.js

@@ -0,0 +1,13 @@
+"use strict";
+// Common Identifier Application
+// Copyright (C) 2024 World Food Programme
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Config = void 0;
+var Config;
+(function (Config) {
+    ;
+})(Config || (exports.Config = Config = {}));

src/config/Config.ts

@@ -30,8 +30,8 @@ export namespace Config {
     static: string[];
     reference: string[];
   }
-  export type StringBasedSalt = { source: 'STRING'; value: string };
-  export type FileBasedSalt = {
+  type StringBasedSalt = { source: 'STRING'; value: string };
+  type FileBasedSalt = {
     source: 'FILE';
     validator_regex?: string;
     value: {
@@ -40,18 +40,8 @@ export namespace Config {
       linux?: string;
     };
   };
-  export interface Options {
-    isBackup?: boolean;
-    meta: {
-      region: string;
-      version?: string;
-      signature?: string;
-    };
-    messages?: {
-      terms_and_conditions: string;
-      error_in_config: string;
-      error_in_salt: string;
-    };
+  export interface CoreConfiguration {
+    meta: { region: string }
     source: ColumnMap;
     validations?: { [key: string]: ValidationRule[] };
     algorithm: {
@@ -61,6 +51,21 @@ export namespace Config {
       };
       salt: StringBasedSalt | FileBasedSalt;
     };
+  }
+  export interface FileConfiguration extends CoreConfiguration {
+    isBackup?: boolean;
+    meta: {
+      region: string;
+      version: string;
+      signature: string;
+    }
+    // messages relevant for UI only so defining as optional here.
+    // TODO: messages are not relevant for file-based usage without the UI, factor them out to a UIConfig type.
+    messages?: {
+      terms_and_conditions: string;
+      error_in_config: string;
+      error_in_salt: string;
+    };
     destination: ColumnMap;
     destination_map: ColumnMap;
     destination_errors: ColumnMap;

src/config/configStore.ts

@@ -36,35 +36,38 @@ function ensureAppDirectoryExists(appDir: string) {
 // This function does not do any validations on the data
 // (that should have happened after loading the config)
 // NOTE: the config is saved as JSON (TOML serialization can be weird)
-function saveConfig(configData: Config.Options, outputPath: string) {
+function saveConfig(configData: Config.FileConfiguration, outputPath: string) {
   // update the config hash on import to account for the
   const outputData = JSON.stringify(configData, null, '    ');
   fs.writeFileSync(outputPath, outputData, CONFIG_FILE_ENCODING);
   log('Written config data to ', outputPath);
 }
 
 interface ConfigStorePaths {
-  configFilePath: string;
-  appConfigFilePath: string;
-  backupConfigFilePath: string;
-  region: string;
+  config: string;
+  appConfig: string;
+  backupConfig: string;
 }
 
 export class ConfigStore {
-  data?: Config.Options = undefined;
+  data?: Config.FileConfiguration = undefined;
   validationResult = {};
   hasConfigLoaded: boolean = false;
   isValid: boolean = false;
   isUsingBackupConfig: boolean = false;
   lastUpdated: Date;
   loadError: string | undefined;
+  usingUI: boolean;
 
   appConfig: AppConfigData = DEFAULT_APP_CONFIG;
-  configPaths: ConfigStorePaths;
+  filePaths: ConfigStorePaths;
+  region: string;
 
-  constructor(configPaths: ConfigStorePaths) {
+  constructor(filePaths: ConfigStorePaths, region: string, usingUI: boolean = false) {
     this.lastUpdated = new Date();
-    this.configPaths = configPaths;
+    this.filePaths = filePaths;
+    this.region = region;
+    this.usingUI = usingUI;
   }
 
   getConfig() {
@@ -73,22 +76,22 @@ export class ConfigStore {
 
   // Returns the region of the store
   getRegion() {
-    return this.configPaths.region;
+    return this.region;
   }
 
   // Returns the path of the user config file
   getConfigFilePath() {
-    return this.configPaths.configFilePath;
+    return this.filePaths.config;
   }
 
   // Returns the path of the backup config file
   getBackupConfigFilePath() {
-    return this.configPaths.backupConfigFilePath;
+    return this.filePaths.backupConfig;
   }
 
   // Returns the path of the application config file
   getAppConfigFilePath() {
-    return this.configPaths.appConfigFilePath;
+    return this.filePaths.appConfig;
   }
 
   // Returns true if the current config is a backup configuration
@@ -213,14 +216,14 @@ export class ConfigStore {
   }
 
   // use a backup config and store its 'backupness'
-  useBackupConfig(configData: Config.Options) {
+  useBackupConfig(configData: Config.FileConfiguration) {
     this.isUsingBackupConfig = true;
     this.lastUpdated = new Date();
     this._useConfig(configData);
   }
 
   // use an actual user-provided config and store its 'user-providedness'
-  useUserConfig(configData: Config.Options, lastUpdateDate: Date) {
+  useUserConfig(configData: Config.FileConfiguration, lastUpdateDate: Date) {
     this.isUsingBackupConfig = false;
     this.lastUpdated = lastUpdateDate;
     this._useConfig(configData);
@@ -234,7 +237,7 @@ export class ConfigStore {
   }
 
   // use an already validated config as the app config
-  _useConfig(configData: Config.Options) {
+  _useConfig(configData: Config.FileConfiguration) {
     this.data = configData;
     this.hasConfigLoaded = true;
     this.isValid = true;
@@ -248,7 +251,7 @@ export class ConfigStore {
   }
 
   // Overwrites the existing configuration file with the new data
-  saveNewConfigData(configData: Config.Options) {
+  saveNewConfigData(configData: Config.FileConfiguration) {
     // before saving ensure that we can save the configuration
     ensureAppDirectoryExists(dirname(this.getConfigFilePath()));
     // TODO: maybe do a rename w/ timestamp for backup
@@ -292,7 +295,7 @@ export class ConfigStore {
   }
 }
 
-export function makeConfigStore(storeConfig: ConfigStorePaths) {
-  if (!storeConfig) throw new Error(`ConfigStore params MUST be provided.`);
-  return new ConfigStore(storeConfig);
+export function makeConfigStore({filePaths, region, usingUI}: {filePaths: ConfigStorePaths, region: string, usingUI: boolean}) {
+  if (!filePaths || !region) throw new Error(`ConfigStore params MUST be provided.`);
+  return new ConfigStore(filePaths, region, usingUI);
 }

src/config/generateConfigHash.ts

@@ -25,16 +25,20 @@ type RecursivePartial<T> = { [P in keyof T]?: RecursivePartial<T[P]> };
 
 // Takes a config, removes the "signature" and salt keys from it, generates
 // a stable JSON representation and hashes it using the provided algorithm
-export function generateConfigHash(config: Config.Options, hashType = DEFAULT_HASH_TYPE) {
+export function generateConfigHash<T extends Config.CoreConfiguration>(config: T, hashType = DEFAULT_HASH_TYPE) {
   // create a nested copy of the object
-  const configCopy = { ...(JSON.parse(JSON.stringify(config)) as RecursivePartial<Config.Options>) };
+  const configCopy = { ...(JSON.parse(JSON.stringify(config)) as RecursivePartial<T>) };
 
   // remove the "signature" key
-  delete configCopy.meta!.signature;
+  if (configCopy.meta && "signature" in configCopy.meta) {
+    delete configCopy.meta.signature;
+  }
 
   // remove the "messages" key
   // TODO: messages should go in a separate locales file to future proof translations
-  delete configCopy.messages;
+  if ("messages" in configCopy) {
+    delete configCopy.messages;
+  }
 
   // remove the "algorithm.salt" part as it may have injected keys
   // TODO: this enables messing with the salt file path pre-injection without signature validations, but is required for compatibility w/ the injection workflow

src/config/index.ts

@@ -1,5 +1,5 @@
 export { loadConfig } from './loadConfig';
-export { validateConfig } from './validateConfig';
+export { validateConfigCore, validateConfigFile } from './validateConfig';
 export { generateConfigHash } from './generateConfigHash';
 export { makeConfigStore, ConfigStore } from './configStore';
 export { appDataLocation, attemptToReadTOMLData } from './utils';

src/config/loadConfig.ts

@@ -16,7 +16,7 @@
 
 import fs from 'node:fs';
 
-import { validateConfig } from './validateConfig';
+import { validateConfigFile } from './validateConfig';
 import { loadSaltFile } from './loadSaltFile';
 import { generateConfigHash } from './generateConfigHash';
 
@@ -29,18 +29,9 @@ const log = Debug('CID:loadConfig');
 export const CONFIG_FILE_ENCODING: fs.EncodingOption = 'utf-8';
 
 type LoadConfigResult =
-  | { success: true; lastUpdated: Date; config: Config.Options }
-  | {
-      success: false;
-      error: string;
-      isSaltFileError: false;
-    }
-  | {
-      success: false;
-      error: string;
-      isSaltFileError: true;
-      config: Config.Options;
-    };
+  | { success: true; lastUpdated: Date; config: Config.FileConfiguration; }
+  | { success: false; error: string; isSaltFileError: false; }
+  | { success: false; error: string; isSaltFileError: true; config: Config.FileConfiguration; };
 
 
 // Main entry point for loading a config file.
@@ -49,11 +40,11 @@ type LoadConfigResult =
 // - { success: false, error: "string" } if there are errors
 // - { success: false, isSaltFileError: true, error: "string"}
 //     if there is something wrong with the salt file
-export function loadConfig(configPath: string, region: string): LoadConfigResult {
+export function loadConfig(configPath: string, region: string, usingUI: boolean=false): LoadConfigResult {
   log('Loading config from', configPath);
 
   // attempt to read the file
-  const configData = attemptToReadTOMLData<Config.Options>(configPath, CONFIG_FILE_ENCODING);
+  const configData = attemptToReadTOMLData<Config.FileConfiguration>(configPath, CONFIG_FILE_ENCODING);
 
   // if cannot be read, we have an error
   if (!configData) {
@@ -68,7 +59,7 @@ export function loadConfig(configPath: string, region: string): LoadConfigResult
   const lastUpdateDate = new Date(fs.statSync(configPath).mtime);
 
   // validate the config
-  const validationResult = validateConfig(configData, region);
+  const validationResult = validateConfigFile(configData, region, usingUI);
 
   // if the config is not valid return false
   if (validationResult) {
@@ -132,7 +123,7 @@ export function loadConfig(configPath: string, region: string): LoadConfigResult
   }
 
   // replace the "FILE" with "STRING" amd embed the salt data
-  configData.algorithm.salt = configData.algorithm.salt as unknown as Config.StringBasedSalt;
+  configData.algorithm.salt = configData.algorithm.salt as unknown as Config.CoreConfiguration["algorithm"]["salt"];
   configData.algorithm.salt.source = 'STRING';
   configData.algorithm.salt.value = saltData;
 

src/config/loadSaltFile.ts

@@ -30,7 +30,7 @@ const DEFAULT_VALIDATOR_REGEXP: RegExp =
 
 // Attempts to load and clean up the salt file data
 export function loadSaltFile(
-  saltFilePath: Config.Options['algorithm']['salt']['value'],
+  saltFilePath: Config.FileConfiguration['algorithm']['salt']['value'],
   validatorRegexp = DEFAULT_VALIDATOR_REGEXP,
 ) {
   // resolve the salt file path from the config & platform

src/config/utils.ts

@@ -54,7 +54,7 @@ export function attemptToReadTOMLData<T>(filePath: string, encoding: fs.Encoding
 
 // takes into consideration the platform and the type of value provided by the config
 // to return an actual, absolute salt file path
-export function getSaltFilePath(saltValueConfig: Config.Options['algorithm']['salt']['value']) {
+export function getSaltFilePath(saltValueConfig: Config.CoreConfiguration['algorithm']['salt']['value']) {
   // if the value is a string always use it
   if (typeof saltValueConfig === 'string') return saltValueConfig;