Skip to content

Commit 7a64962

Browse files
jasnyjasny
committed
Load custom schemas.
Added `yaml-options` for customization.
1 parent c436a90 commit 7a64962

File tree

14 files changed

+135
-22
lines changed

14 files changed

+135
-22
lines changed

‎.gitignore

+5-3
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,10 @@ lerna-debug.log*
3838
!.vscode/launch.json
3939
!.vscode/extensions.json
4040

41-
# Runtime data
42-
/storage
41+
# Project data
42+
/scenarios
43+
/schemas
4344

4445
# Local configuration
45-
/src/config/local
46+
src/config/local.json
47+
src/config/*.local.json

‎scenarios/.gitkeep

Whitespace-only changes.

‎schemas/.gitkeep

Whitespace-only changes.

‎src/app.module.ts

+10-1
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,18 @@ import { ApiKeyModule } from './apikey/apikey.module';
88
import { ConfigModule } from './common/config/config.module';
99
import { AuthModule } from './common/auth/auth.module';
1010
import { EventEmitterModule } from '@nestjs/event-emitter';
11+
import { AjvModule } from '@/common/ajv/ajv.module';
1112

1213
@Module({
13-
imports: [ConfigModule, AuthModule, ScenarioModule, ProcessModule, ApiKeyModule, EventEmitterModule.forRoot()],
14+
imports: [
15+
ConfigModule,
16+
AuthModule,
17+
ScenarioModule,
18+
ProcessModule,
19+
ApiKeyModule,
20+
EventEmitterModule.forRoot(),
21+
AjvModule,
22+
],
1423
controllers: [AppController],
1524
providers: [AppService],
1625
})

‎src/app.service.ts

+9-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { Injectable, OnModuleInit } from '@nestjs/common';
22
import { ConfigService } from './common/config/config.service';
3+
import Ajv from 'ajv';
34

45
@Injectable()
56
export class AppService implements OnModuleInit {
@@ -8,9 +9,13 @@ export class AppService implements OnModuleInit {
89
version: string;
910
description: string;
1011
env: string;
12+
schemas: string[];
1113
};
1214

13-
constructor(private config: ConfigService) {}
15+
constructor(
16+
private readonly config: ConfigService,
17+
private readonly ajv: Ajv,
18+
) {}
1419

1520
onModuleInit() {
1621
this.initInfo();
@@ -25,6 +30,9 @@ export class AppService implements OnModuleInit {
2530
version: packageInfo.version,
2631
description: packageInfo.description,
2732
env: this.config.get('env'),
33+
schemas: Object.values(this.ajv.schemas)
34+
.map((env) => (env as any)?.schema?.$id as string | undefined)
35+
.filter((id) => !!id && !id.match(/^https:\/\/json-schema\.org\/.+\/meta\//)),
2836
};
2937
}
3038
}

‎src/common/ajv/ajv.module.ts

+68-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,78 @@
1-
import { Module } from '@nestjs/common';
2-
import Ajv from 'ajv/dist/2020';
1+
import { Logger, Module } from '@nestjs/common';
2+
import Ajv from 'ajv';
3+
import Ajv2020 from 'ajv/dist/2020';
4+
import { actionSchema, actorSchema, fnSchema, scenarioSchema, schemaSchema } from '@letsflow/core/schemas/v1.0.0';
5+
import fs from 'fs/promises';
6+
import { ConfigModule } from '@/common/config/config.module';
7+
import { ConfigService } from '@/common/config/config.service';
8+
import { yaml } from '@letsflow/core';
9+
import { normalize, Schema } from '@letsflow/core/scenario';
10+
import yamlOptions from '@/common/yaml-options';
311

412
@Module({
13+
imports: [ConfigModule],
514
providers: [
615
{
716
provide: Ajv,
8-
useValue: new Ajv(),
17+
useFactory: async (config: ConfigService) => {
18+
const logger = new Logger('Ajv');
19+
20+
const ajv = new Ajv2020({ allErrors: true });
21+
ajv.addKeyword('$anchor'); // Workaround for https://github.com/ajv-validator/ajv/issues/1854
22+
ajv.addSchema([scenarioSchema, actionSchema, actorSchema, fnSchema, schemaSchema]);
23+
24+
const schemas = await loadSchemas(config.get('schema.path'), logger);
25+
ajv.addSchema(schemas);
26+
27+
return ajv;
28+
},
29+
inject: [ConfigService],
930
},
1031
],
1132
exports: [Ajv],
1233
})
1334
export class AjvModule {}
35+
36+
async function loadSchemas(dir: string, logger: Logger): Promise<Schema[]> {
37+
const files = await fs.readdir(dir);
38+
const schemaContent = await Promise.all(
39+
files
40+
.filter((file) => file.endsWith('.json') || file.endsWith('.yaml') || file.endsWith('.yml'))
41+
.map((file) => fs.readFile(`${dir}/${file}`, 'utf8').then((content) => [file, content] as const)),
42+
);
43+
44+
return schemaContent
45+
.map(([file, content]): [string, Schema | undefined] => {
46+
try {
47+
const schema = file.endsWith('.json') ? JSON.parse(content) : yaml.parse(content, yamlOptions);
48+
return [file, schema];
49+
} catch (e) {
50+
logger.warn(`Failed to parse schema ${file}`);
51+
return [file, undefined];
52+
}
53+
})
54+
.filter(([file, schema]) => {
55+
if (typeof schema === 'undefined') {
56+
return false;
57+
}
58+
59+
if (!schema || !schema.$id) {
60+
logger.warn(`Skipping schema '${file}': Missing '$id' property`);
61+
return false;
62+
}
63+
64+
if (schema.$schema && typeof schema.$schema !== 'string') {
65+
logger.warn(`Skipping schema '${file}': '$schema' must be a string`);
66+
return false;
67+
}
68+
69+
if (schema.$schema && schema.$schema !== schemaSchema.id && schema.$schema !== schemaSchema.$schema) {
70+
logger.warn(`Skipping schema '${file}': Unsupported schema ${schema.$schema}`);
71+
return false;
72+
}
73+
74+
logger.debug(`Loaded schema ${schema.$id}`);
75+
return true;
76+
})
77+
.map(([, schema]) => normalize(schema, { $schema: schemaSchema.$id }));
78+
}

‎src/common/config/config.service.ts

+8-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import convict from 'convict';
33
import schema from '../../config/schema';
44
import * as fs from 'node:fs';
55
import * as path from 'node:path';
6+
import * as process from 'node:process';
67

78
type SchemaOf<T extends convict.Schema<any>> = T extends convict.Schema<infer R> ? R : any;
89
type Schema = SchemaOf<typeof schema>;
@@ -14,6 +15,7 @@ type PathValue<K extends Path> = K extends null | undefined
1415
: never;
1516

1617
const CONFIG_PATH = path.normalize(__dirname + '/../../config');
18+
const LOCAL_CONFIG_PATH = process.env.CONFIG_PATH;
1719

1820
@Injectable()
1921
export class ConfigService implements OnModuleInit, OnModuleDestroy {
@@ -40,7 +42,12 @@ export class ConfigService implements OnModuleInit, OnModuleDestroy {
4042
const config = convict(schema);
4143
const env = config.get('env');
4244

43-
const configFiles = [`${env}.json`, `local/settings.json`, `local/${env}.json`]
45+
const configFiles = [
46+
`${env}.json`,
47+
`local.json`,
48+
`${env}.local.json`,
49+
...(LOCAL_CONFIG_PATH ? [`${LOCAL_CONFIG_PATH}/settings.json`, `${LOCAL_CONFIG_PATH}/${env}.json`] : []),
50+
]
4451
.map((file) => path.join(CONFIG_PATH, file))
4552
.filter(fs.existsSync);
4653

‎src/common/yaml-options.ts

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
const yamlOptions = {};
2+
3+
export default yamlOptions;

‎src/config/schema.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ export default {
3434
env: 'SCENARIO_STORAGE',
3535
},
3636
path: {
37-
default: './storage/scenarios',
37+
default: './scenarios',
3838
env: 'SCENARIO_PATH',
3939
},
4040
readOnly: {
@@ -56,6 +56,12 @@ export default {
5656
env: 'PROCESS_ADDITIONAL_SUMMERY_FIELDS',
5757
},
5858
},
59+
schema: {
60+
path: {
61+
default: './schemas',
62+
env: 'SCHEMA_PATH',
63+
},
64+
},
5965
jwt: {
6066
issuer: {
6167
default: '',

‎src/middleware/yamlBodyParser.middleware.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { Injectable, NestMiddleware } from '@nestjs/common';
22
import { NextFunction, Request, Response } from 'express';
33
import { yaml } from '@letsflow/core';
4+
import yamlOptions from '@/common/yaml-options';
45

56
@Injectable()
67
export class YamlBodyParserMiddleware implements NestMiddleware {
@@ -19,7 +20,7 @@ export class YamlBodyParserMiddleware implements NestMiddleware {
1920

2021
req.on('end', () => {
2122
try {
22-
req.body = yaml.parse(data);
23+
req.body = yaml.parse(data, yamlOptions);
2324
next();
2425
} catch (error) {
2526
next(error);

‎src/process/process.controller.spec.ts

+3-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ import { Response } from 'express';
66
import { StartInstructions } from './process.dto';
77
import { HttpStatus } from '@nestjs/common';
88
import { AuthService } from '@/common/auth/auth.service';
9-
import Ajv from 'ajv/dist/2020';
9+
import Ajv from 'ajv';
10+
import Ajv2020 from 'ajv/dist/2020';
1011
import { ScenarioService } from '@/scenario/scenario.service';
1112
import { Account } from '@/common/auth';
1213
import { ApiKey } from '@/apikey';
@@ -44,7 +45,7 @@ describe('ProcessController', () => {
4445
},
4546
{
4647
provide: Ajv,
47-
useValue: new Ajv(),
48+
useValue: new Ajv2020(),
4849
},
4950
ValidationService,
5051
{

‎src/process/validation/validation.service.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import Ajv from 'ajv/dist/2020';
1+
import Ajv from 'ajv';
22
import { Injectable } from '@nestjs/common';
33
import { ScenarioService } from '@/scenario/scenario.service';
44
import { StartInstructions } from '../process.dto';

‎src/scenario/scenario-fs/scenario-fs.service.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import * as fs from 'node:fs/promises';
66
import { watch } from 'node:fs';
77
import { uuid, yaml } from '@letsflow/core';
88
import { ScenarioService } from '../scenario.service';
9+
import yamlOptions from '@/common/yaml-options';
910

1011
@Injectable()
1112
export class ScenarioFsService extends ScenarioService {
@@ -51,7 +52,7 @@ export class ScenarioFsService extends ScenarioService {
5152
private async loadScenario(file: string, scenarioYaml?: string) {
5253
scenarioYaml ??= await fs.readFile(`${this.path}/${file}`, 'utf8');
5354

54-
const scenario = yaml.parse(scenarioYaml);
55+
const scenario = yaml.parse(scenarioYaml, yamlOptions);
5556
const normalized = normalize(scenario);
5657
const id = uuid(normalized);
5758

‎yarn.lock

+17-7
Original file line numberDiff line numberDiff line change
@@ -1197,17 +1197,18 @@ __metadata:
11971197

11981198
"@letsflow/core@file:../core::locator=%40letsflow%2Fengine%40workspace%3A.":
11991199
version: 0.0.0
1200-
resolution: "@letsflow/core@file:../core#../core::hash=c9608c&locator=%40letsflow%2Fengine%40workspace%3A."
1200+
resolution: "@letsflow/core@file:../core#../core::hash=46c4d6&locator=%40letsflow%2Fengine%40workspace%3A."
12011201
dependencies:
1202-
"@letsflow/jmespath": ../jmespath/
1202+
"@letsflow/jmespath": ^1.1.4-jasny.1
12031203
"@noble/hashes": ^1.4.0
12041204
ajv: ^8.16.0
12051205
fast-json-stable-stringify: ^2.1.0
12061206
get-value: ^3.0.1
1207+
mustache: ^4.2.0
12071208
set-value: ^4.1.0
12081209
uuid: ^10.0.0
12091210
yaml: ^2.4.5
1210-
checksum: dbaf900233303acbda86eb8e535b88ab5efe68dbd8bc2f38ee2b8d00311a0c6312915adf8df02d4d81850c7aaf7c6ce3be22f9307e2108d6c46e7866869e4d03
1211+
checksum: c52ec5e08fbadc8438a2a2222c25f66cf4f76020126c7de07c043da96908bcbcde3c8b042687746af2d750b9f6af255771a1c95f68d0c022dc9d2a410ea1a378
12111212
languageName: node
12121213
linkType: hard
12131214

@@ -1261,10 +1262,10 @@ __metadata:
12611262
languageName: unknown
12621263
linkType: soft
12631264

1264-
"@letsflow/jmespath@file:../jmespath/::locator=%40letsflow%2Fcore%40file%3A..%2Fcore%23..%2Fcore%3A%3Ahash%3Dc9608c%26locator%3D%2540letsflow%252Fengine%2540workspace%253A.":
1265-
version: 1.1.3
1266-
resolution: "@letsflow/jmespath@file:../jmespath/#../jmespath/::hash=02e8a5&locator=%40letsflow%2Fcore%40file%3A..%2Fcore%23..%2Fcore%3A%3Ahash%3Dc9608c%26locator%3D%2540letsflow%252Fengine%2540workspace%253A."
1267-
checksum: 1ea9ec4de4ac4adce021915445d73a50658cc46977c1425f841ba65530d4aeac03f365828b9c2b421223505b12a48dd3b2eec506079949b85a2205ff58d3d505
1265+
"@letsflow/jmespath@npm:^1.1.4-jasny.1":
1266+
version: 1.1.4-jasny.1
1267+
resolution: "@letsflow/jmespath@npm:1.1.4-jasny.1"
1268+
checksum: 8e7b215ae124601485ed4cab28070b846f7dda3b6a9952e01a532e4ce4694f6534b0a160aff95715eee719dac55b5e5200c70a764e64b908a0b9cf53c2c530f4
12681269
languageName: node
12691270
linkType: hard
12701271

@@ -6231,6 +6232,15 @@ __metadata:
62316232
languageName: node
62326233
linkType: hard
62336234

6235+
"mustache@npm:^4.2.0":
6236+
version: 4.2.0
6237+
resolution: "mustache@npm:4.2.0"
6238+
bin:
6239+
mustache: bin/mustache
6240+
checksum: 928fcb63e3aa44a562bfe9b59ba202cccbe40a46da50be6f0dd831b495be1dd7e38ca4657f0ecab2c1a89dc7bccba0885eab7ee7c1b215830da765758c7e0506
6241+
languageName: node
6242+
linkType: hard
6243+
62346244
"mute-stream@npm:0.0.8":
62356245
version: 0.0.8
62366246
resolution: "mute-stream@npm:0.0.8"

0 commit comments

Comments
 (0)