diff --git a/.env.schema b/.env.schema
index 5f76b7b..d3b784d 100644
--- a/.env.schema
+++ b/.env.schema
@@ -11,3 +11,4 @@ LECTERN_URL=
LOG_LEVEL=
PORT=3030
UPLOAD_LIMIT=''
+PLURALIZE_SCHEMAS_ENABLED=
diff --git a/README.md b/README.md
index 9b183da..aac15b5 100644
--- a/README.md
+++ b/README.md
@@ -72,21 +72,22 @@ Create a `.env` file based on `.env.schema` located on the root folder and set t
The Environment Variables used for this application are listed in the table bellow
-| Name | Description | Default |
-| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------- |
-| `AUDIT_ENABLED` | Ensures that any modifications to the submitted data are logged, providing a way to identify who made changes and when they were made. | true |
-| `DB_HOST` | Database Hostname | |
-| `DB_NAME` | Database Name | |
-| `DB_PASSWORD` | Database Password | |
-| `DB_PORT` | Database Port | |
-| `DB_USER` | Database User | |
-| `ID_CUSTOM_ALPHABET` | Custom Alphabet for local ID generation | '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' |
-| `ID_CUSTOM_SIZE` | Custom size of ID for local ID generation | 21 |
-| `ID_USELOCAL` | Generate ID locally | true |
-| `LECTERN_URL` | Schema Service (Lectern) URL | |
-| `LOG_LEVEL` | Log Level | 'info' |
-| `PORT` | Server Port. | 3030 |
-| `UPLOAD_LIMIT` | Limit upload file size in string or number.
Supported units and abbreviations are as follows and are case-insensitive:
- b for bytes
- kb for kilobytes
- mb for megabytes
- gb for gigabytes
- tb for terabytes
- pb for petabytes
Any other text is considered as byte | '10mb' |
+| Name | Description | Default |
+| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------- |
+| `AUDIT_ENABLED` | Ensures that any modifications to the submitted data are logged, providing a way to identify who made changes and when they were made. | true |
+| `DB_HOST` | Database Hostname | |
+| `DB_NAME` | Database Name | |
+| `DB_PASSWORD` | Database Password | |
+| `DB_PORT` | Database Port | |
+| `DB_USER` | Database User | |
+| `ID_CUSTOM_ALPHABET` | Custom Alphabet for local ID generation | '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' |
+| `ID_CUSTOM_SIZE` | Custom size of ID for local ID generation | 21 |
+| `ID_USELOCAL` | Generate ID locally | true |
+| `LECTERN_URL` | Schema Service (Lectern) URL | |
+| `LOG_LEVEL` | Log Level | 'info' |
+| `PLURALIZE_SCHEMAS_ENABLED` | This feature automatically convert schema names to their plural forms when handling compound documents. Pluralization assumes the words are in English | true |
+| `PORT` | Server Port. | 3030 |
+| `UPLOAD_LIMIT` | Limit upload file size in string or number.
Supported units and abbreviations are as follows and are case-insensitive:
- b for bytes
- kb for kilobytes
- mb for megabytes
- gb for gigabytes
- tb for terabytes
- pb for petabytes
Any other text is considered as byte | '10mb' |
## Script commands (Workspace)
diff --git a/apps/server/src/config/server.ts b/apps/server/src/config/server.ts
index 90ceb08..cb8f1b8 100644
--- a/apps/server/src/config/server.ts
+++ b/apps/server/src/config/server.ts
@@ -39,6 +39,9 @@ export const defaultAppConfig: AppConfig = {
audit: {
enabled: getBoolean(process.env.AUDIT_ENABLED, true),
},
+ recordHierarchy: {
+ pluralizeSchemasName: getBoolean(process.env.PLURALIZE_SCHEMAS_ENABLED, true),
+ },
},
idService: {
useLocal: getBoolean(process.env.ID_USELOCAL, true),
diff --git a/apps/server/swagger/data-api.yml b/apps/server/swagger/data-api.yml
index 2ead29b..a3f6dc9 100644
--- a/apps/server/swagger/data-api.yml
+++ b/apps/server/swagger/data-api.yml
@@ -4,14 +4,9 @@
tags:
- Data
parameters:
- - name: categoryId
- in: path
- required: true
- schema:
- type: string
- description: ID of the category
+ - $ref: '#/components/parameters/path/CategoryId'
- name: entityName
- description: Array of strings to filter by entity names
+ description: Array of strings to filter by entity names. Incompatible with `compound` view
in: query
required: false
schema:
@@ -19,18 +14,9 @@
type: array
items:
type: string
- - name: page
- in: query
- required: false
- schema:
- type: integer
- description: Optional query parameter to specify the page number of the results. Default value is 1
- - name: pageSize
- in: query
- required: false
- schema:
- type: integer
- description: Optional query parameter to specify the number of results per page. Default value is 20
+ - $ref: '#/components/parameters/query/Page'
+ - $ref: '#/components/parameters/query/PageSize'
+ - $ref: '#/components/parameters/query/View'
responses:
200:
description: Submitted Data
@@ -49,26 +35,48 @@
503:
$ref: '#/components/responses/ServiceUnavailableError'
-/data/category/{categoryId}/organization/{organization}:
+/data/category/{categoryId}/id/{systemId}:
get:
- summary: Retrieve Submitted Data for a specific Category and Organization
+ summary: Retrieve Submitted Data Record for a System ID
tags:
- Data
parameters:
- - name: categoryId
+ - $ref: '#/components/parameters/path/CategoryId'
+ - name: systemId
in: path
required: true
schema:
type: string
- description: ID of the category
- - name: organization
- in: path
- required: true
- schema:
- type: string
- description: Organization name
+ description: ID of the record
+ - $ref: '#/components/parameters/query/View'
+ responses:
+ 200:
+ description: Submitted Data
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/SubmittedDataRecord'
+ 400:
+ $ref: '#/components/responses/BadRequest'
+ 401:
+ $ref: '#/components/responses/UnauthorizedError'
+ 404:
+ $ref: '#/components/responses/NotFound'
+ 500:
+ $ref: '#/components/responses/ServerError'
+ 503:
+ $ref: '#/components/responses/ServiceUnavailableError'
+
+/data/category/{categoryId}/organization/{organization}:
+ get:
+ summary: Retrieve Submitted Data for a specific Category and Organization
+ tags:
+ - Data
+ parameters:
+ - $ref: '#/components/parameters/path/CategoryId'
+ - $ref: '#/components/parameters/path/Organization'
- name: entityName
- description: Array of strings to filter by entity names
+ description: Array of strings to filter by entity names. Incompatible with `compound` view
in: query
required: false
schema:
@@ -76,18 +84,9 @@
type: array
items:
type: string
- - name: page
- in: query
- required: false
- schema:
- type: integer
- description: Optional query parameter to specify the page number of the results. Default value is 1
- - name: pageSize
- in: query
- required: false
- schema:
- type: integer
- description: Optional query parameter to specify the number of results per page. Default value is 20
+ - $ref: '#/components/parameters/query/Page'
+ - $ref: '#/components/parameters/query/PageSize'
+ - $ref: '#/components/parameters/query/View'
responses:
200:
description: Submitted Data
@@ -112,18 +111,8 @@
tags:
- Data
parameters:
- - name: categoryId
- in: path
- required: true
- schema:
- type: string
- description: ID of the category
- - name: organization
- in: path
- required: true
- schema:
- type: string
- description: Organization name
+ - $ref: '#/components/parameters/path/CategoryId'
+ - $ref: '#/components/parameters/path/Organization'
- name: entityName
description: Array of strings to filter by entity names
in: query
@@ -133,18 +122,8 @@
type: array
items:
type: string
- - name: page
- in: query
- required: false
- schema:
- type: integer
- description: Optional query parameter to specify the page number of the results. Default value is 1
- - name: pageSize
- in: query
- required: false
- schema:
- type: integer
- description: Optional query parameter to specify the number of results per page. Default value is 20
+ - $ref: '#/components/parameters/query/Page'
+ - $ref: '#/components/parameters/query/PageSize'
requestBody:
description: Custom filter SQON Notation to provide a flexible system for combining filters in a JSON object format. Find more documentation on https://github.com/overture-stack/sqon-builder
required: true
diff --git a/apps/server/swagger/dictionary-api.yml b/apps/server/swagger/dictionary-api.yml
index 7461235..abd68fa 100644
--- a/apps/server/swagger/dictionary-api.yml
+++ b/apps/server/swagger/dictionary-api.yml
@@ -14,15 +14,19 @@
categoryName:
type: string
description: A user-defined classification to group and organize data based on shared characteristics or criteria
- required: true
dictionaryName:
type: string
description: A matching Dictionary Name defined on Dictionary Manager (Lectern)
- required: true
- version:
+ dictionaryVersion:
type: string
description: A matching Dictionary Version defined on Dictionary Manager (Lectern)
- required: true
+ defaultCentricEntity:
+ type: string
+ description: The default centric entity name
+ required:
+ - categoryName
+ - dictionaryName
+ - dictionaryVersion
responses:
200:
description: Dictionary info
diff --git a/apps/server/swagger/parameters.yml b/apps/server/swagger/parameters.yml
new file mode 100644
index 0000000..9980e89
--- /dev/null
+++ b/apps/server/swagger/parameters.yml
@@ -0,0 +1,40 @@
+components:
+ parameters:
+ path:
+ CategoryId:
+ name: categoryId
+ in: path
+ required: true
+ schema:
+ type: string
+ description: ID of the category to which the data belongs
+ Organization:
+ name: organization
+ in: path
+ required: true
+ schema:
+ type: string
+ description: Organization name
+ query:
+ Page:
+ name: page
+ in: query
+ required: false
+ schema:
+ type: integer
+ description: Optional query parameter to specify the page number of the results. Default value is 1
+ PageSize:
+ name: pageSize
+ in: query
+ required: false
+ schema:
+ type: integer
+ description: Optional query parameter to specify the number of results per page. Default value is 20
+ View:
+ name: view
+ in: query
+ required: false
+ schema:
+ type: string
+ enum: ['flat', 'compound']
+ description: Optional query parameter to define the data format. Choose 'flat' for a simple, single-level collection of records, or 'compound' for a nested, schema-centric structure. The default value is 'flat'
diff --git a/apps/server/swagger/schemas.yml b/apps/server/swagger/schemas.yml
index 9751537..f914da0 100644
--- a/apps/server/swagger/schemas.yml
+++ b/apps/server/swagger/schemas.yml
@@ -425,17 +425,6 @@ components:
type: string
description: ID of the Submission
- DeleteDataActiveSubmission:
- type: object
- properties:
- records:
- type: array
- items:
- $ref: '#/components/schemas/SubmittedDataRecord'
- submissionId:
- type: string
- description: ID of the Submission
-
GetSubmittedDataResult:
type: object
properties:
@@ -503,10 +492,8 @@ components:
type: object
properties:
data:
- type: array
- items:
- type: object
- description: Content of the record in JSON format
+ type: object
+ description: Content of the record in JSON format
entityName:
type: string
isValid:
diff --git a/apps/server/swagger/submission-api.yml b/apps/server/swagger/submission-api.yml
index 3e07f15..3cea0ee 100644
--- a/apps/server/swagger/submission-api.yml
+++ b/apps/server/swagger/submission-api.yml
@@ -111,10 +111,7 @@
tags:
- Submission
parameters:
- - name: categoryId
- in: path
- type: string
- required: true
+ - $ref: '#/components/parameters/path/CategoryId'
responses:
200:
description: Active Submissions
@@ -138,14 +135,8 @@
tags:
- Submission
parameters:
- - name: categoryId
- in: path
- type: string
- required: true
- - name: organization
- in: path
- type: string
- required: true
+ - $ref: '#/components/parameters/path/CategoryId'
+ - $ref: '#/components/parameters/path/Organization'
responses:
200:
description: Active Submission
@@ -169,10 +160,7 @@
tags:
- Submission
parameters:
- - name: categoryId
- in: path
- type: file
- required: true
+ - $ref: '#/components/parameters/path/CategoryId'
- name: submissionId
in: path
type: string
@@ -203,10 +191,7 @@
consumes:
- multipart/form-data
parameters:
- - name: categoryId
- in: path
- type: string
- required: true
+ - $ref: '#/components/parameters/path/CategoryId'
requestBody:
content:
multipart/form-data:
@@ -220,6 +205,9 @@
format: binary
organization:
type: string
+ required:
+ - files
+ - organization
responses:
200:
description: Submission accepted
@@ -242,12 +230,7 @@
consumes:
- multipart/form-data
parameters:
- - name: categoryId
- in: path
- required: true
- schema:
- type: string
- description: ID of the category
+ - $ref: '#/components/parameters/path/CategoryId'
requestBody:
content:
multipart/form-data:
@@ -261,6 +244,9 @@
format: binary
organization:
type: string
+ required:
+ - files
+ - organization
responses:
200:
description: Edit Data request accepted
@@ -283,12 +269,7 @@
tags:
- Submission
parameters:
- - name: categoryId
- in: path
- required: true
- schema:
- type: string
- description: ID of the category to which the data belongs
+ - $ref: '#/components/parameters/path/CategoryId'
- name: systemId
in: path
required: true
diff --git a/eslint.config.js b/eslint.config.js
index ec2fae6..1bca68a 100644
--- a/eslint.config.js
+++ b/eslint.config.js
@@ -53,6 +53,12 @@ export default tseslint.config(
],
},
],
+ '@typescript-eslint/consistent-type-assertions': [
+ 'warn',
+ {
+ assertionStyle: 'never',
+ },
+ ],
'simple-import-sort/exports': 'error',
'prettier/prettier': 'error',
},
diff --git a/packages/data-model/docs/README.md b/packages/data-model/docs/README.md
new file mode 100644
index 0000000..719eaa7
--- /dev/null
+++ b/packages/data-model/docs/README.md
@@ -0,0 +1,119 @@
+# Lyric Data Model
+
+This document provides an overview of the entities and relationships used to manage data submissions, it explains how each entity contributes to organizing, auditing, and validating data, ensuring that submissions adhere to the correct version and structure based on defined dictionary categories.
+
+
+## Entities
+*View full [Data Model](schema.dbml)*
+
+
+### - `dictionaries`
+Stores individual dictionaries, each containing data schema definitions as JSON objects.
+
+Key components of the dictionary table include:
+
+- `name`: A descriptive name that identifies the dictionary schema. This allows for easy categorization and retrieval of specific schemas by name.
+
+- `version`: A version identifier that tracks changes and updates to the dictionary schema over time. Versioning ensures that data submissions reference the correct schema version, allowing for backward compatibility and controlled evolution of the schema structure.
+
+- `dictionary`: A JSONB column that stores the full dictionary schema as a JSON object. This schema defines the fields, data types, relationships, and validation rules required for data submissions, providing a flexible and easily accessible format for schema definitions.
+
+> [!NOTE]
+> Creating a new dictionary requires [Lectern](https://github.com/overture-stack/lectern) as a Data Dictionary Management Service.
+
+
+### - `dictionary_categories`
+Organizes dictionaries into categories, each specifying a default centric entity to determine data hierarchy.
+
+Key fields in the dictionary_categories table include:
+
+- `active_dictionary_id`: This field references the current, active dictionary version associated with the category. By linking to a specific dictionary entry, this field enables version control, ensuring that all data submissions associated with this category adhere to the latest schema version defined by the active dictionary.
+
+- `name`: The name of the category, which is a unique and descriptive identifier used to label and retrieve the category. This helps users quickly locate the appropriate category and its associated schema for different submission types or data groups.
+
+- `default_centric_entity`: Central entity that defines the primary structure for the data. The central entity compounds the primary entity of the dictionary, nesting its children entities in an array and associating a single parent entity. This hierarchical structure enables complex data relationships to be represented accurately for easy interpretation.
+
+
+### - `submissions`
+Stores data submissions, each associated with a dictionary category and dictionary version.
+
+Key fields in the submissions table include:
+
+- `data`: This field contains the actual submission data stored as a JSON object. The use of JSON allows for flexible and dynamic data structures, accommodating varying submission formats while preserving the integrity of the information.
+
+- `dictionary_category_id`: This field establishes a link to the corresponding dictionary category, which defines the schema against which the submission data will be validated. By referencing the dictionary category, the submission ensures compliance with the rules and structures outlined in the associated dictionary.
+
+- `dictionary_id`: This field links the submission directly to the specific dictionary version being utilized for validation. This connection ensures that the submission is aligned with the correct schema version, enabling consistent validation and data integrity.
+
+- `errors`: This field captures any validation errors encountered during the submission process. It allows for detailed tracking of issues, providing insights into why certain data may not conform to the expected schema.
+
+- `organization`: This field indicates the organization responsible for the submission. This contextual information is essential for data management and auditing purposes, allowing for the tracking of submissions by different entities.
+
+- `status`: This field represents the current state of the submission and can include values such as open, valid, invalid, closed, or committed. The status helps to monitor the submission's lifecycle, ensuring that users can easily identify which submissions are pending, validated, or finalized.
+
+#### What is a Submission?
+A Submission is a structured process used to prepare, validate, and commit data for final storage and usage. This process consists of two main steps to ensure data accuracy, completeness, and alignment with specified schema requirements.
+
+1. **Preparation and Validation:** In the initial step, data is organized and validated against a specific dictionary category and schema. During validation, the data is checked to ensure it meets the defined structure and rules for its category. This validation helps catch errors or inconsistencies before data is finalized, allowing for corrections or adjustments if necessary.
+
+2. **Commitment as Submitted Data:** Once validated, the data can then be committed as Submitted Data. This final step solidifies the data submission, converting it into a permanent record that is ready for future retrieval, auditing, or analysis. At this stage, the data is fully compliant with the schema version defined by the referenced dictionary, ensuring consistency across all submitted records.
+
+To view the State diagram for submission status [click here](./stateDiagramSubmissionStatus.md)
+
+To view Submission commit workflow [click here](./submissionCommit.md)
+
+
+### - `submitted_data`
+Stores individual data entries within a submission, capturing their validation status and relationships to specific schemas.
+
+`submitted_data` represents the finalized, validated records from the submission process. Once data passes the initial preparation and validation stages, it is committed as `submitted_data`, becoming a stable and structured record that is ready for storage and future reference.
+
+Key fields in the submitted_data table include:
+
+- `data`: This field contains the submission data stored as a JSON object. Using JSON allows for flexibility in the data structure, accommodating various formats and types of information while preserving the integrity of the submission.
+
+- `entity_name`: This field defines the specific name of the entity associated with the submitted data. It provides context for understanding the type of data being submitted and allows for easier categorization and retrieval based on the entity represented.
+
+- `organization`: This field indicates the organization responsible for the submission. Including this information is crucial for effective data management, as each organization may store distinct and unrelated data.
+
+- `system_id`: This unique identifier distinguishes each entry within the `submitted_data` table. The `system_id` ensures that there are no duplicates, providing a reliable reference point for each record and facilitating easy retrieval and tracking of submissions.
+
+- `is_valid`: This boolean field indicates whether the submitted data is considered valid based on the schema rules defined in the corresponding dictionary. Tracking the validity status is crucial for ensuring data quality and compliance.
+
+- `last_valid_schema_id` and `original_schema_id`: These fields provide references to the schema versions that were used to validate the submission, supporting effective version control and historical tracking of changes.
+
+- **Timestamps and User Metadata**: Fields such as `created_at`, `created_by`, `updated_at`, and `updated_by` capture important metadata about when and by whom the record was created or modified. This information is essential for maintaining a comprehensive audit trail.
+
+
+
+### - audit_submitted_data
+This table records historical changes to submitted data, providing an audit trail.
+
+Key fields in the audit_submitted_data table include:
+
+- `action`: This field records the type of action performed on the submitted data, represented as an enumerated type (e.g., UPDATE or DELETE)
+
+- `dictionary_category_id`: This field links the audit entry to the corresponding dictionary category associated with the submitted data. It helps contextualize the action within the framework of the defined schema.
+
+- `data_diff`: This field stores a JSON object that captures the differences between the previous and current states of the submitted data. By providing a detailed view of what was changed, this field is essential for tracking modifications and understanding the impact of those changes.
+
+- `entity_name`: This field specifies the name of the entity related to the submitted data, providing additional context for the action logged in the audit entry.
+
+- `last_valid_schema_id`: This field references the schema version that was last validated before the action was taken, ensuring that users can trace back to the applicable schema for the submission.
+
+- `new_data_is_valid`: This boolean field indicates whether the newly submitted data is considered valid according to the schema rules. This status is important for ensuring that only compliant data is retained.
+
+- `old_data_is_valid`: Similar to the previous field, this boolean indicates the validity of the data before the action was taken, allowing for a comparison of data quality over time.
+
+- `organization`: This field identifies the organization responsible for the submission, providing context for the audit entry and aiding in data governance.
+
+- `original_schema_id`: This field references the original schema version used during the initial submission of the data, allowing for a historical perspective on schema changes over time.
+
+- `submission_id`: This field links the audit record to the specific submission, creating a direct connection to the submission responsible for this action.
+
+- `system_id`: A unique identifier for the submitted data, ensuring that each record in the audit trail can be traced back to its corresponding submission.
+
+- `created_at`: This timestamp records when the audit entry was created, providing a chronological context for the logged action.
+
+- `created_by`: This field indicates who performed the action, enhancing accountability by identifying the user responsible for the change.
+
diff --git a/packages/data-model/docs/schema.dbml b/packages/data-model/docs/schema.dbml
index 56be802..8fb595f 100644
--- a/packages/data-model/docs/schema.dbml
+++ b/packages/data-model/docs/schema.dbml
@@ -39,7 +39,8 @@ table dictionaries {
table dictionary_categories {
id serial [pk, not null, increment]
- active_dictionary_id integer
+ active_dictionary_id integer [not null]
+ default_centric_entity varchar
name varchar [not null, unique]
created_at timestamp [default: `now()`]
created_by varchar
diff --git a/packages/data-model/migrations/0009_add_default_centric_entity.sql b/packages/data-model/migrations/0009_add_default_centric_entity.sql
new file mode 100644
index 0000000..ac894ac
--- /dev/null
+++ b/packages/data-model/migrations/0009_add_default_centric_entity.sql
@@ -0,0 +1,2 @@
+ALTER TABLE "dictionary_categories" ALTER COLUMN "active_dictionary_id" SET NOT NULL;--> statement-breakpoint
+ALTER TABLE "dictionary_categories" ADD COLUMN "default_centric_entity" varchar;
diff --git a/packages/data-model/migrations/meta/0009_snapshot.json b/packages/data-model/migrations/meta/0009_snapshot.json
new file mode 100644
index 0000000..eee89db
--- /dev/null
+++ b/packages/data-model/migrations/meta/0009_snapshot.json
@@ -0,0 +1,545 @@
+{
+ "id": "9d7d98fa-e067-4891-a2c1-65a19d6430f3",
+ "prevId": "b83f6dc6-52bf-412a-a8ac-63947ed92eec",
+ "version": "5",
+ "dialect": "pg",
+ "tables": {
+ "audit_submitted_data": {
+ "name": "audit_submitted_data",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "action": {
+ "name": "action",
+ "type": "audit_action",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "dictionary_category_id": {
+ "name": "dictionary_category_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "data_diff": {
+ "name": "data_diff",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "entity_name": {
+ "name": "entity_name",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "last_valid_schema_id": {
+ "name": "last_valid_schema_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "new_data_is_valid": {
+ "name": "new_data_is_valid",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "old_data_is_valid": {
+ "name": "old_data_is_valid",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "organization": {
+ "name": "organization",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "original_schema_id": {
+ "name": "original_schema_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "submission_id": {
+ "name": "submission_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "system_id": {
+ "name": "system_id",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "audit_submitted_data_dictionary_category_id_dictionary_categories_id_fk": {
+ "name": "audit_submitted_data_dictionary_category_id_dictionary_categories_id_fk",
+ "tableFrom": "audit_submitted_data",
+ "tableTo": "dictionary_categories",
+ "columnsFrom": [
+ "dictionary_category_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "audit_submitted_data_last_valid_schema_id_dictionaries_id_fk": {
+ "name": "audit_submitted_data_last_valid_schema_id_dictionaries_id_fk",
+ "tableFrom": "audit_submitted_data",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "last_valid_schema_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "audit_submitted_data_original_schema_id_dictionaries_id_fk": {
+ "name": "audit_submitted_data_original_schema_id_dictionaries_id_fk",
+ "tableFrom": "audit_submitted_data",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "original_schema_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "audit_submitted_data_submission_id_submissions_id_fk": {
+ "name": "audit_submitted_data_submission_id_submissions_id_fk",
+ "tableFrom": "audit_submitted_data",
+ "tableTo": "submissions",
+ "columnsFrom": [
+ "submission_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {}
+ },
+ "dictionaries": {
+ "name": "dictionaries",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "dictionary": {
+ "name": "dictionary",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "name": {
+ "name": "name",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "version": {
+ "name": "version",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {}
+ },
+ "dictionary_categories": {
+ "name": "dictionary_categories",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "active_dictionary_id": {
+ "name": "active_dictionary_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "default_centric_entity": {
+ "name": "default_centric_entity",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "name": {
+ "name": "name",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_by": {
+ "name": "updated_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "dictionary_categories_name_unique": {
+ "name": "dictionary_categories_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ }
+ },
+ "submissions": {
+ "name": "submissions",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "data": {
+ "name": "data",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "dictionary_category_id": {
+ "name": "dictionary_category_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "dictionary_id": {
+ "name": "dictionary_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "errors": {
+ "name": "errors",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "organization": {
+ "name": "organization",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status": {
+ "name": "status",
+ "type": "submission_status",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "updated_by": {
+ "name": "updated_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "submissions_dictionary_category_id_dictionary_categories_id_fk": {
+ "name": "submissions_dictionary_category_id_dictionary_categories_id_fk",
+ "tableFrom": "submissions",
+ "tableTo": "dictionary_categories",
+ "columnsFrom": [
+ "dictionary_category_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "submissions_dictionary_id_dictionaries_id_fk": {
+ "name": "submissions_dictionary_id_dictionaries_id_fk",
+ "tableFrom": "submissions",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "dictionary_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {}
+ },
+ "submitted_data": {
+ "name": "submitted_data",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "serial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "data": {
+ "name": "data",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "dictionary_category_id": {
+ "name": "dictionary_category_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "entity_name": {
+ "name": "entity_name",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_valid": {
+ "name": "is_valid",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "last_valid_schema_id": {
+ "name": "last_valid_schema_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "organization": {
+ "name": "organization",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "original_schema_id": {
+ "name": "original_schema_id",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "system_id": {
+ "name": "system_id",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "created_by": {
+ "name": "created_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "now()"
+ },
+ "updated_by": {
+ "name": "updated_by",
+ "type": "varchar",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "organization_index": {
+ "name": "organization_index",
+ "columns": [
+ "organization"
+ ],
+ "isUnique": false
+ }
+ },
+ "foreignKeys": {
+ "submitted_data_dictionary_category_id_dictionary_categories_id_fk": {
+ "name": "submitted_data_dictionary_category_id_dictionary_categories_id_fk",
+ "tableFrom": "submitted_data",
+ "tableTo": "dictionary_categories",
+ "columnsFrom": [
+ "dictionary_category_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "submitted_data_last_valid_schema_id_dictionaries_id_fk": {
+ "name": "submitted_data_last_valid_schema_id_dictionaries_id_fk",
+ "tableFrom": "submitted_data",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "last_valid_schema_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ },
+ "submitted_data_original_schema_id_dictionaries_id_fk": {
+ "name": "submitted_data_original_schema_id_dictionaries_id_fk",
+ "tableFrom": "submitted_data",
+ "tableTo": "dictionaries",
+ "columnsFrom": [
+ "original_schema_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "no action",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "submitted_data_system_id_unique": {
+ "name": "submitted_data_system_id_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "system_id"
+ ]
+ }
+ }
+ }
+ },
+ "enums": {
+ "audit_action": {
+ "name": "audit_action",
+ "values": {
+ "UPDATE": "UPDATE",
+ "DELETE": "DELETE"
+ }
+ },
+ "submission_status": {
+ "name": "submission_status",
+ "values": {
+ "OPEN": "OPEN",
+ "VALID": "VALID",
+ "INVALID": "INVALID",
+ "CLOSED": "CLOSED",
+ "COMMITTED": "COMMITTED"
+ }
+ }
+ },
+ "schemas": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
diff --git a/packages/data-model/migrations/meta/_journal.json b/packages/data-model/migrations/meta/_journal.json
index 7380079..54143ff 100644
--- a/packages/data-model/migrations/meta/_journal.json
+++ b/packages/data-model/migrations/meta/_journal.json
@@ -64,6 +64,13 @@
"when": 1727965304851,
"tag": "0008_set_not_null_last_valid_schema_id",
"breakpoints": true
+ },
+ {
+ "idx": 9,
+ "version": "5",
+ "when": 1730128901514,
+ "tag": "0009_add_default_centric_entity",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/packages/data-model/src/models/dictionary_categories.ts b/packages/data-model/src/models/dictionary_categories.ts
index fa93e66..43c1957 100644
--- a/packages/data-model/src/models/dictionary_categories.ts
+++ b/packages/data-model/src/models/dictionary_categories.ts
@@ -7,7 +7,8 @@ import { submittedData } from './submitted_data.js';
export const dictionaryCategories = pgTable('dictionary_categories', {
id: serial('id').primaryKey(),
- activeDictionaryId: integer('active_dictionary_id'),
+ activeDictionaryId: integer('active_dictionary_id').notNull(),
+ defaultCentricEntity: varchar('default_centric_entity'),
name: varchar('name').unique().notNull(),
createdAt: timestamp('created_at').defaultNow(),
createdBy: varchar('created_by'),
diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json
index 822c603..6ef60bf 100644
--- a/packages/data-provider/package.json
+++ b/packages/data-provider/package.json
@@ -47,6 +47,7 @@
"multer": "1.4.5-lts.1",
"nanoid": "^5.0.7",
"pg": "^8.12.0",
+ "plur": "^5.1.0",
"winston": "^3.13.1",
"zod": "^3.23.8"
},
diff --git a/packages/data-provider/src/config/config.ts b/packages/data-provider/src/config/config.ts
index 34fb4d5..1a7e196 100644
--- a/packages/data-provider/src/config/config.ts
+++ b/packages/data-provider/src/config/config.ts
@@ -16,8 +16,13 @@ export type DbConfig = {
password: string;
};
+export type RecordHierarchyConfig = {
+ pluralizeSchemasName: boolean;
+};
+
export type FeaturesConfig = {
audit?: AuditConfig;
+ recordHierarchy: RecordHierarchyConfig;
};
export type SchemaServiceConfig = {
diff --git a/packages/data-provider/src/controllers/auditController.ts b/packages/data-provider/src/controllers/auditController.ts
index ad8ed02..091ec9a 100644
--- a/packages/data-provider/src/controllers/auditController.ts
+++ b/packages/data-provider/src/controllers/auditController.ts
@@ -18,8 +18,8 @@ const controller = (dependencies: BaseDependencies) => {
const organization = req.params.organization;
// pagination parameters
- const page = parseInt(req.query.page as string) || defaultPage;
- const pageSize = parseInt(req.query.pageSize as string) || defaultPageSize;
+ const page = parseInt(String(req.query.page)) || defaultPage;
+ const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize;
// optional query parameters
const { entityName, eventType, startDate, endDate, systemId } = req.query;
diff --git a/packages/data-provider/src/controllers/dictionaryController.ts b/packages/data-provider/src/controllers/dictionaryController.ts
index c6be15d..aeb9cf9 100644
--- a/packages/data-provider/src/controllers/dictionaryController.ts
+++ b/packages/data-provider/src/controllers/dictionaryController.ts
@@ -13,18 +13,20 @@ const controller = (dependencies: BaseDependencies) => {
try {
const categoryName = req.body.categoryName;
const dictionaryName = req.body.dictionaryName;
- const dictionaryVersion = req.body.version;
+ const dictionaryVersion = req.body.dictionaryVersion;
+ const defaultCentricEntity = req.body.defaultCentricEntity;
logger.info(
LOG_MODULE,
`Register Dictionary Request categoryName '${categoryName}' name '${dictionaryName}' version '${dictionaryVersion}'`,
);
- const { dictionary, category } = await dictionaryService.register(
+ const { dictionary, category } = await dictionaryService.register({
categoryName,
dictionaryName,
dictionaryVersion,
- );
+ defaultCentricEntity,
+ });
logger.info(LOG_MODULE, `Register Dictionary completed!`);
diff --git a/packages/data-provider/src/controllers/submissionController.ts b/packages/data-provider/src/controllers/submissionController.ts
index 993d67c..ac7641b 100644
--- a/packages/data-provider/src/controllers/submissionController.ts
+++ b/packages/data-provider/src/controllers/submissionController.ts
@@ -1,8 +1,8 @@
import { isEmpty } from 'lodash-es';
import { BaseDependencies } from '../config/config.js';
-import submissionService from '../services/submissionService.js';
-import submittedDataService from '../services/submittedDataService.js';
+import submissionService from '../services/submission/submission.js';
+import submittedDataService from '../services/submittedData/submmittedData.js';
import { BadRequest, NotFound } from '../utils/errors.js';
import { hasTsvExtension, processFiles } from '../utils/fileUtils.js';
import { validateRequest } from '../utils/requestValidation.js';
@@ -121,7 +121,7 @@ const controller = (dependencies: BaseDependencies) => {
editSubmittedData: validateRequest(dataEditRequestSchema, async (req, res, next) => {
try {
const categoryId = Number(req.params.categoryId);
- const files = req.files as Express.Multer.File[];
+ const files = Array.isArray(req.files) ? req.files : [];
const organization = req.body.organization;
logger.info(LOG_MODULE, `Request Edit Submitted Data`);
@@ -233,7 +233,7 @@ const controller = (dependencies: BaseDependencies) => {
upload: validateRequest(uploadSubmissionRequestSchema, async (req, res, next) => {
try {
const categoryId = Number(req.params.categoryId);
- const files = req.files as Express.Multer.File[];
+ const files = Array.isArray(req.files) ? req.files : [];
const organization = req.body.organization;
// TODO: get userName from auth
diff --git a/packages/data-provider/src/controllers/submittedDataController.ts b/packages/data-provider/src/controllers/submittedDataController.ts
index ad2b8c4..83f4a50 100644
--- a/packages/data-provider/src/controllers/submittedDataController.ts
+++ b/packages/data-provider/src/controllers/submittedDataController.ts
@@ -1,7 +1,8 @@
import * as _ from 'lodash-es';
+import { convertToViewType } from '..//utils/submittedDataUtils.js';
import { BaseDependencies } from '../config/config.js';
-import submittedDataService from '../services/submittedDataService.js';
+import submittedDataService from '../services/submittedData/submmittedData.js';
import { parseSQON } from '../utils/convertSqonToQuery.js';
import { NotFound } from '../utils/errors.js';
import { asArray } from '../utils/formatUtils.js';
@@ -9,9 +10,10 @@ import { validateRequest } from '../utils/requestValidation.js';
import {
dataGetByCategoryRequestSchema,
dataGetByOrganizationRequestSchema,
- dataGetByQueryRequestschema,
+ dataGetByQueryRequestSchema,
+ dataGetBySystemIdRequestSchema,
} from '../utils/schemas.js';
-import { SubmittedDataPaginatedResponse } from '../utils/types.js';
+import { SubmittedDataPaginatedResponse, VIEW_TYPE } from '../utils/types.js';
const controller = (dependencies: BaseDependencies) => {
const service = submittedDataService(dependencies);
@@ -19,29 +21,33 @@ const controller = (dependencies: BaseDependencies) => {
const LOG_MODULE = 'SUBMITTED_DATA_CONTROLLER';
const defaultPage = 1;
const defaultPageSize = 20;
+ const defaultView = VIEW_TYPE.Values.flat;
+
return {
getSubmittedDataByCategory: validateRequest(dataGetByCategoryRequestSchema, async (req, res, next) => {
try {
const categoryId = Number(req.params.categoryId);
// query params
- const entityName = asArray(req.query.entityName);
- const page = parseInt(req.query.page as string) || defaultPage;
- const pageSize = parseInt(req.query.pageSize as string) || defaultPageSize;
+ const entityName = asArray(req.query.entityName || []);
+ const page = parseInt(String(req.query.page)) || defaultPage;
+ const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize;
+ const view = convertToViewType(req.query.view) || defaultView;
logger.info(
LOG_MODULE,
`Request Submitted Data on categoryId '${categoryId}'`,
`pagination params: page '${page}' pageSize '${pageSize}'`,
+ `view '${view}'`,
);
const submittedDataResult = await service.getSubmittedDataByCategory(
categoryId,
{ page, pageSize },
- { entityName },
+ { entityName, view },
);
- if (_.isEmpty(submittedDataResult.data)) {
+ if (_.isEmpty(submittedDataResult.result)) {
throw new NotFound('No Submitted Data found');
}
@@ -52,7 +58,7 @@ const controller = (dependencies: BaseDependencies) => {
totalPages: Math.ceil(submittedDataResult.metadata.totalRecords / pageSize),
totalRecords: submittedDataResult.metadata.totalRecords,
},
- records: submittedDataResult.data,
+ records: submittedDataResult.result,
};
return res.status(200).send(response);
@@ -67,14 +73,16 @@ const controller = (dependencies: BaseDependencies) => {
const organization = req.params.organization;
// query parameters
- const entityName = asArray(req.query.entityName);
- const page = parseInt(req.query.page as string) || defaultPage;
- const pageSize = parseInt(req.query.pageSize as string) || defaultPageSize;
+ const entityName = asArray(req.query.entityName || []);
+ const page = parseInt(String(req.query.page)) || defaultPage;
+ const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize;
+ const view = convertToViewType(String(req.query.view)) || defaultView;
logger.info(
LOG_MODULE,
`Request Submitted Data on categoryId '${categoryId}' and organization '${organization}'`,
`pagination params: page '${page}' pageSize '${pageSize}'`,
+ `view '${view}'`,
);
const submittedDataResult = await service.getSubmittedDataByOrganization(
@@ -84,7 +92,7 @@ const controller = (dependencies: BaseDependencies) => {
page,
pageSize,
},
- { entityName },
+ { entityName, view },
);
if (submittedDataResult.metadata.errorMessage) {
@@ -98,7 +106,7 @@ const controller = (dependencies: BaseDependencies) => {
totalPages: Math.ceil(submittedDataResult.metadata.totalRecords / pageSize),
totalRecords: submittedDataResult.metadata.totalRecords,
},
- records: submittedDataResult.data,
+ records: submittedDataResult.result,
};
return res.status(200).send(responsePaginated);
@@ -107,16 +115,16 @@ const controller = (dependencies: BaseDependencies) => {
}
}),
- getSubmittedDataByQuery: validateRequest(dataGetByQueryRequestschema, async (req, res, next) => {
+ getSubmittedDataByQuery: validateRequest(dataGetByQueryRequestSchema, async (req, res, next) => {
try {
const categoryId = Number(req.params.categoryId);
const organization = req.params.organization;
const sqon = parseSQON(req.body);
// query parameters
- const entityName = asArray(req.query.entityName);
- const page = parseInt(req.query.page as string) || defaultPage;
- const pageSize = parseInt(req.query.pageSize as string) || defaultPageSize;
+ const entityName = asArray(req.query.entityName || []);
+ const page = parseInt(String(req.query.page)) || defaultPage;
+ const pageSize = parseInt(String(req.query.pageSize)) || defaultPageSize;
logger.info(
LOG_MODULE,
@@ -134,7 +142,7 @@ const controller = (dependencies: BaseDependencies) => {
page,
pageSize,
},
- { sqon, entityName },
+ { sqon, entityName, view: VIEW_TYPE.Values.flat },
);
if (submittedDataResult.metadata.errorMessage) {
@@ -148,7 +156,7 @@ const controller = (dependencies: BaseDependencies) => {
totalPages: Math.ceil(submittedDataResult.metadata.totalRecords / pageSize),
totalRecords: submittedDataResult.metadata.totalRecords,
},
- records: submittedDataResult.data,
+ records: submittedDataResult.result,
};
return res.status(200).send(responsePaginated);
@@ -156,6 +164,33 @@ const controller = (dependencies: BaseDependencies) => {
next(error);
}
}),
+ getSubmittedDataBySystemId: validateRequest(dataGetBySystemIdRequestSchema, async (req, res, next) => {
+ try {
+ const categoryId = Number(req.params.categoryId);
+ const systemId = req.params.systemId;
+ const view = convertToViewType(String(req.query.view)) || defaultView;
+
+ logger.info(
+ LOG_MODULE,
+ 'Request Submitted Data',
+ `categoryId '${categoryId}'`,
+ `systemId '${systemId}'`,
+ `params: view '${view}'`,
+ );
+
+ const submittedDataResult = await service.getSubmittedDataBySystemId(categoryId, systemId, {
+ view,
+ });
+
+ if (submittedDataResult.metadata.errorMessage) {
+ throw new NotFound(submittedDataResult.metadata.errorMessage);
+ }
+
+ return res.status(200).send(submittedDataResult.result);
+ } catch (error) {
+ next(error);
+ }
+ }),
};
};
diff --git a/packages/data-provider/src/core/provider.ts b/packages/data-provider/src/core/provider.ts
index 5457d8e..dc0664f 100644
--- a/packages/data-provider/src/core/provider.ts
+++ b/packages/data-provider/src/core/provider.ts
@@ -1,6 +1,11 @@
import { AppConfig, BaseDependencies } from '../config/config.js';
import { connect } from '../config/db.js';
import { getLogger } from '../config/logger.js';
+import auditController from '../controllers/auditController.js';
+import categoryController from '../controllers/categoryController.js';
+import dictionaryController from '../controllers/dictionaryController.js';
+import submissionController from '../controllers/submissionController.js';
+import submittedDataController from '../controllers/submittedDataController.js';
import auditRouter from '../routers/auditRouter.js';
import categoryRouter from '../routers/categoryRouter.js';
import dictionaryRouter from '../routers/dictionaryRouter.js';
@@ -9,11 +14,18 @@ import submittedDataRouter from '../routers/submittedDataRouter.js';
import auditService from '../services/auditService.js';
import categoryService from '../services/categoryService.js';
import dictionaryService from '../services/dictionaryService.js';
-import submissionService from '../services/submissionService.js';
-import submittedDataService from '../services/submittedDataService.js';
+import submissionService from '../services/submission/submission.js';
+import submittedDataService from '../services/submittedData/submmittedData.js';
+import * as auditUtils from '../utils/auditUtils.js';
+import * as convertSqonToQueryUtils from '../utils/convertSqonToQuery.js';
+import * as dictionarySchemaRelationUtils from '../utils/dictionarySchemaRelations.js';
import * as dictionaryUtils from '../utils/dictionaryUtils.js';
+import * as errorUtils from '../utils/errors.js';
+import * as fileUtils from '../utils/fileUtils.js';
+import * as schemaUtils from '../utils/schemas.js';
import * as submissionUtils from '../utils/submissionUtils.js';
import * as submittedDataUtils from '../utils/submittedDataUtils.js';
+import * as typeUtils from '../utils/types.js';
/**
* The main provider of submission resources
@@ -23,9 +35,7 @@ import * as submittedDataUtils from '../utils/submittedDataUtils.js';
const provider = (configData: AppConfig) => {
const baseDeps: BaseDependencies = {
db: connect(configData.db),
- features: {
- audit: configData.features?.audit,
- },
+ features: configData.features,
idService: configData.idService,
limits: configData.limits,
logger: getLogger(configData.logger),
@@ -41,6 +51,13 @@ const provider = (configData: AppConfig) => {
submission: submissionRouter(baseDeps),
submittedData: submittedDataRouter(baseDeps),
},
+ controllers: {
+ audit: auditController(baseDeps),
+ category: categoryController(baseDeps),
+ dictionary: dictionaryController(baseDeps),
+ submission: submissionController(baseDeps),
+ submittedData: submittedDataController(baseDeps),
+ },
services: {
audit: auditService(baseDeps),
category: categoryService(baseDeps),
@@ -49,9 +66,16 @@ const provider = (configData: AppConfig) => {
submittedData: submittedDataService(baseDeps),
},
utils: {
+ audit: auditUtils,
+ convertSqonToQuery: convertSqonToQueryUtils,
+ dictionarySchemaRelations: dictionarySchemaRelationUtils,
dictionary: dictionaryUtils,
+ errors: errorUtils,
+ file: fileUtils,
+ schema: schemaUtils,
submission: submissionUtils,
submittedData: submittedDataUtils,
+ type: typeUtils,
},
};
};
diff --git a/packages/data-provider/src/repository/activeSubmissionRepository.ts b/packages/data-provider/src/repository/activeSubmissionRepository.ts
index da663af..be22f1e 100644
--- a/packages/data-provider/src/repository/activeSubmissionRepository.ts
+++ b/packages/data-provider/src/repository/activeSubmissionRepository.ts
@@ -22,7 +22,7 @@ const repository = (dependencies: BaseDependencies) => {
updatedBy: true,
};
- const getActiveSubmissionRelations = {
+ const getActiveSubmissionRelations: { [key: string]: { columns: BooleanTrueObject } } = {
dictionary: {
columns: {
name: true,
diff --git a/packages/data-provider/src/repository/submittedRepository.ts b/packages/data-provider/src/repository/submittedRepository.ts
index c8cdf1b..f079bbf 100644
--- a/packages/data-provider/src/repository/submittedRepository.ts
+++ b/packages/data-provider/src/repository/submittedRepository.ts
@@ -85,10 +85,10 @@ const repository = (dependencies: BaseDependencies) => {
/**
* Build a SQL object to search submitted data by entity Name
- * @param {string[] | undefined} entityNameArray
+ * @param {string[]} entityNameArray
* @returns {SQL | undefined}
*/
- const filterByEntityNameArray = (entityNameArray?: (string | undefined)[]): SQL | undefined => {
+ const filterByEntityNameArray = (entityNameArray?: string[]): SQL | undefined => {
if (Array.isArray(entityNameArray)) {
return or(
...entityNameArray
@@ -188,12 +188,14 @@ const repository = (dependencies: BaseDependencies) => {
* Find SubmittedData by category ID with pagination
* @param {number} categoryId Category ID
* @param {PaginationOptions} paginationOptions Pagination properties
+ * @param {object} filter Filter Options
+ * @param {string[] | undefined} filter.entityNames Array of entity names to filter
* @returns The SubmittedData found
*/
getSubmittedDataByCategoryIdPaginated: async (
categoryId: number,
paginationOptions: PaginationOptions,
- filter?: { entityNames?: (string | undefined)[] },
+ filter?: { entityNames?: string[] },
): Promise => {
const { page, pageSize } = paginationOptions;
@@ -217,15 +219,17 @@ const repository = (dependencies: BaseDependencies) => {
* Find SubmittedData by category ID and Organization with pagination
* @param {number} categoryId Category ID
* @param {string} organization Organization Name
- * @param {SQL} filter Optional filter
* @param {PaginationOptions} paginationOptions Pagination properties
+ * @param {object} filter Filter Options
+ * @param {SQL | undefined} filter.sql SQL command to filter
+ * @param {string[] | undefined} filter.entityNames Array of entity names to filter
* @returns The SubmittedData found
*/
getSubmittedDataByCategoryIdAndOrganizationPaginated: async (
categoryId: number,
organization: string,
paginationOptions: PaginationOptions,
- filter?: { sql?: SQL; entityNames?: (string | undefined)[] },
+ filter?: { sql?: SQL; entityNames?: string[] },
): Promise => {
const { page, pageSize } = paginationOptions;
@@ -260,13 +264,13 @@ const repository = (dependencies: BaseDependencies) => {
* @param {string} organization Organization Name
* @param {object} filter Filter Options
* @param {SQL | undefined} filter.sql SQL command to filter
- * @param {(string | undefined)[] | undefined} filter.entityNames Array of entity names to filter
+ * @param {string[] | undefined} filter.entityNames Array of entity names to filter
* @returns Total number of recourds
*/
getTotalRecordsByCategoryIdAndOrganization: async (
categoryId: number,
organization: string,
- filter?: { sql?: SQL; entityNames?: (string | undefined)[] },
+ filter?: { sql?: SQL; entityNames?: string[] },
): Promise => {
const filterEntityNameSql = filterByEntityNameArray(filter?.entityNames);
@@ -298,12 +302,12 @@ const repository = (dependencies: BaseDependencies) => {
* @param {number} categoryId Category ID
* @param {object} filter Filter options
* @param {SQL | undefined} filter.sql SQL command
- * @param {(string | undefined)[] | undefined} filter.entityNames Array of entity names to filter
+ * @param {string[] | undefined} filter.entityNames Array of entity names to filter
* @returns Total number of recourds
*/
getTotalRecordsByCategoryId: async (
categoryId: number,
- filter?: { sql?: SQL; entityNames?: (string | undefined)[] },
+ filter?: { sql?: SQL; entityNames?: string[] },
): Promise => {
const filterEntityNameSql = filterByEntityNameArray(filter?.entityNames);
try {
diff --git a/packages/data-provider/src/routers/submittedDataRouter.ts b/packages/data-provider/src/routers/submittedDataRouter.ts
index a8dcf5a..24d8bd8 100644
--- a/packages/data-provider/src/routers/submittedDataRouter.ts
+++ b/packages/data-provider/src/routers/submittedDataRouter.ts
@@ -1,4 +1,4 @@
-import { Router, urlencoded } from 'express';
+import { json, Router, urlencoded } from 'express';
import { BaseDependencies } from '../config/config.js';
import submittedDataController from '../controllers/submittedDataController.js';
@@ -7,6 +7,7 @@ import { auth } from '../middleware/auth.js';
const router = (dependencies: BaseDependencies): Router => {
const router = Router();
router.use(urlencoded({ extended: false }));
+ router.use(json());
router.get('/category/:categoryId', auth, submittedDataController(dependencies).getSubmittedDataByCategory);
@@ -22,6 +23,12 @@ const router = (dependencies: BaseDependencies): Router => {
submittedDataController(dependencies).getSubmittedDataByQuery,
);
+ router.get(
+ '/category/:categoryId/id/:systemId',
+ auth,
+ submittedDataController(dependencies).getSubmittedDataBySystemId,
+ );
+
return router;
};
diff --git a/packages/data-provider/src/services/dictionaryService.ts b/packages/data-provider/src/services/dictionaryService.ts
index e8c01b1..a5584e4 100644
--- a/packages/data-provider/src/services/dictionaryService.ts
+++ b/packages/data-provider/src/services/dictionaryService.ts
@@ -17,6 +17,7 @@ const dictionaryService = (dependencies: BaseDependencies) => {
* @param dictionaryName The name of the dictionary to create
* @param version The version of the dictionary to create
* @param schemas The Schema of the dictionary
+ * @param defaultCentricEntity The Centric schema of the dictionary
* @returns The new dictionary created or the existing one
*/
const createDictionaryIfDoesNotExist = async (
@@ -67,21 +68,32 @@ const dictionaryService = (dependencies: BaseDependencies) => {
}
};
- const register = async (
- categoryName: string,
- dictionaryName: string,
- version: string,
- ): Promise<{ dictionary: Dictionary; category: Category }> => {
+ const register = async ({
+ categoryName,
+ dictionaryName,
+ dictionaryVersion,
+ defaultCentricEntity,
+ }: {
+ categoryName: string;
+ dictionaryName: string;
+ dictionaryVersion: string;
+ defaultCentricEntity?: string;
+ }): Promise<{ dictionary: Dictionary; category: Category }> => {
logger.debug(
LOG_MODULE,
- `Register new dictionary categoryName '${categoryName}' dictionaryName '${dictionaryName}' version '${version}'`,
+ `Register new dictionary categoryName '${categoryName}' dictionaryName '${dictionaryName}' dictionaryVersion '${dictionaryVersion}'`,
);
const categoryRepo = categoryRepository(dependencies);
- const dictionary = await fetchDictionaryByVersion(dictionaryName, version);
+ const dictionary = await fetchDictionaryByVersion(dictionaryName, dictionaryVersion);
- const savedDictionary = await createDictionaryIfDoesNotExist(dictionaryName, version, dictionary.schemas);
+ if (defaultCentricEntity && !dictionary.schemas.some((schema) => schema.name === defaultCentricEntity)) {
+ logger.error(LOG_MODULE, `Entity '${defaultCentricEntity}' does not exist in this dictionary`);
+ throw new Error(`Entity '${defaultCentricEntity}' does not exist in this dictionary`);
+ }
+
+ const savedDictionary = await createDictionaryIfDoesNotExist(dictionaryName, dictionaryVersion, dictionary.schemas);
// Check if Category exist
const foundCategory = await categoryRepo.getCategoryByName(categoryName);
@@ -93,7 +105,10 @@ const dictionaryService = (dependencies: BaseDependencies) => {
return { dictionary: savedDictionary, category: foundCategory };
} else if (foundCategory && foundCategory.activeDictionaryId !== savedDictionary.id) {
// Update the dictionary on existing Category
- const updatedCategory = await categoryRepo.update(foundCategory.id, { activeDictionaryId: savedDictionary.id });
+ const updatedCategory = await categoryRepo.update(foundCategory.id, {
+ activeDictionaryId: savedDictionary.id,
+ defaultCentricEntity,
+ });
logger.info(
LOG_MODULE,
@@ -106,6 +121,7 @@ const dictionaryService = (dependencies: BaseDependencies) => {
const newCategory: NewCategory = {
name: categoryName,
activeDictionaryId: savedDictionary.id,
+ defaultCentricEntity,
};
const savedCategory = await categoryRepo.save(newCategory);
diff --git a/packages/data-provider/src/services/submissionService.ts b/packages/data-provider/src/services/submission/processor.ts
similarity index 60%
rename from packages/data-provider/src/services/submissionService.ts
rename to packages/data-provider/src/services/submission/processor.ts
index 9ad92b9..97f18a5 100644
--- a/packages/data-provider/src/services/submissionService.ts
+++ b/packages/data-provider/src/services/submission/processor.ts
@@ -6,7 +6,6 @@ import {
DictionaryValidationRecordErrorDetails,
} from '@overture-stack/lectern-client';
import {
- type NewSubmission,
Submission,
SubmissionData,
type SubmissionDeleteData,
@@ -15,19 +14,15 @@ import {
SubmittedData,
} from '@overture-stack/lyric-data-model';
-import { BaseDependencies } from '../config/config.js';
-import systemIdGenerator from '../external/systemIdGenerator.js';
-import submissionRepository from '../repository/activeSubmissionRepository.js';
-import categoryRepository from '../repository/categoryRepository.js';
-import dictionaryRepository from '../repository/dictionaryRepository.js';
-import submittedRepository from '../repository/submittedRepository.js';
-import { getDictionarySchemaRelations, type SchemaChildNode } from '../utils/dictionarySchemaRelations.js';
-import { BadRequest, InternalServerError, StatusConflict } from '../utils/errors.js';
-import { tsvToJson } from '../utils/fileUtils.js';
+import { BaseDependencies } from '../../config/config.js';
+import submissionRepository from '../../repository/activeSubmissionRepository.js';
+import categoryRepository from '../../repository/categoryRepository.js';
+import dictionaryRepository from '../../repository/dictionaryRepository.js';
+import submittedRepository from '../../repository/submittedRepository.js';
+import { getDictionarySchemaRelations, type SchemaChildNode } from '../../utils/dictionarySchemaRelations.js';
+import { BadRequest } from '../../utils/errors.js';
+import { tsvToJson } from '../../utils/fileUtils.js';
import {
- canTransitionToClosed,
- checkEntityFieldNames,
- checkFileNames,
extractSchemaDataFromMergedDataRecords,
filterDeletesFromUpdates,
filterRelationsForPrimaryIdUpdate,
@@ -38,13 +33,10 @@ import {
mergeDeleteRecords,
mergeInsertsRecords,
mergeUpdatesBySystemId,
- parseActiveSubmissionResponse,
- parseActiveSubmissionSummaryResponse,
- removeItemsFromSubmission,
segregateFieldChangeRecords,
submissionInsertDataFromFiles,
validateSchemas,
-} from '../utils/submissionUtils.js';
+} from '../../utils/submissionUtils.js';
import {
computeDataDiff,
groupByEntityName,
@@ -53,221 +45,13 @@ import {
hasErrorsByIndex,
mergeSubmittedDataAndDeduplicateById,
updateSubmittedDataArray,
-} from '../utils/submittedDataUtils.js';
-import {
- ActiveSubmissionSummaryResponse,
- CommitSubmissionParams,
- CommitSubmissionResult,
- CREATE_SUBMISSION_STATUS,
- CreateSubmissionResult,
- SUBMISSION_ACTION_TYPE,
- SUBMISSION_STATUS,
- type SubmissionActionType,
- ValidateFilesParams,
-} from '../utils/types.js';
-import submittedDataService from './submittedDataService.js';
-
-const service = (dependencies: BaseDependencies) => {
- const LOG_MODULE = 'SUBMISSION_SERVICE';
- const { logger } = dependencies;
-
- /**
- * Runs Schema validation asynchronously and moves the Active Submission to Submitted Data
- * @param {number} categoryId
- * @param {number} submissionId
- * @returns {Promise}
- */
- const commitSubmission = async (
- categoryId: number,
- submissionId: number,
- userName: string,
- ): Promise => {
- const { getSubmissionById } = submissionRepository(dependencies);
- const { getSubmittedDataByCategoryIdAndOrganization } = submittedRepository(dependencies);
- const { getActiveDictionaryByCategory } = categoryRepository(dependencies);
- const { generateIdentifier } = systemIdGenerator(dependencies);
-
- const submission = await getSubmissionById(submissionId);
- if (!submission) {
- throw new BadRequest(`Submission '${submissionId}' not found`);
- }
-
- if (submission.dictionaryCategoryId !== categoryId) {
- throw new BadRequest(`Category ID provided does not match the category for the Submission`);
- }
+} from '../../utils/submittedDataUtils.js';
+import { CommitSubmissionParams, SUBMISSION_STATUS, type ValidateFilesParams } from '../../utils/types.js';
+import searchDataRelations from '../submittedData/searchDataRelations.js';
- if (submission.status !== SUBMISSION_STATUS.VALID) {
- throw new StatusConflict('Submission does not have status VALID and cannot be committed');
- }
-
- const currentDictionary = await getActiveDictionaryByCategory(categoryId);
- if (_.isEmpty(currentDictionary)) {
- throw new BadRequest(`Dictionary in category '${categoryId}' not found`);
- }
-
- const submittedDataToValidate = await getSubmittedDataByCategoryIdAndOrganization(
- categoryId,
- submission?.organization,
- );
-
- const entitiesToProcess = new Set();
-
- submittedDataToValidate?.forEach((data) => entitiesToProcess.add(data.entityName));
-
- const insertsToValidate = submission.data?.inserts
- ? Object.entries(submission.data.inserts).flatMap(([entityName, submissionData]) => {
- entitiesToProcess.add(entityName);
-
- return submissionData.records.map((record) => ({
- data: record,
- dictionaryCategoryId: categoryId,
- entityName,
- isValid: false, // By default, New Submitted Data is created as invalid until validation proves otherwise
- organization: submission.organization,
- originalSchemaId: submission.dictionaryId,
- systemId: generateIdentifier(entityName, record),
- createdBy: userName,
- }));
- })
- : [];
-
- const deleteDataArray = submission.data?.deletes
- ? Object.entries(submission.data.deletes).flatMap(([entityName, submissionDeleteData]) => {
- entitiesToProcess.add(entityName);
- return submissionDeleteData;
- })
- : [];
-
- const updateDataArray =
- submission.data?.updates &&
- Object.entries(submission.data.updates).reduce>(
- (acc, [entityName, submissionUpdateData]) => {
- entitiesToProcess.add(entityName);
- submissionUpdateData.forEach((record) => {
- acc[record.systemId] = record;
- });
- return acc;
- },
- {},
- );
-
- // To Commit Active Submission we need to validate SubmittedData + Active Submission
- performCommitSubmissionAsync({
- dataToValidate: {
- inserts: insertsToValidate,
- submittedData: submittedDataToValidate,
- deletes: deleteDataArray,
- updates: updateDataArray,
- },
- submission,
- dictionary: currentDictionary,
- userName: userName,
- });
-
- return {
- status: CREATE_SUBMISSION_STATUS.PROCESSING,
- dictionary: {
- name: currentDictionary.name,
- version: currentDictionary.version,
- },
- processedEntities: Array.from(entitiesToProcess.values()),
- };
- };
-
- /**
- * Updates Submission status to CLOSED
- * This action is allowed only if current Submission Status as OPEN, VALID or INVALID
- * Returns the resulting Active Submission with its status
- * @param {number} submissionId
- * @param {string} userName
- * @returns {Promise}
- */
- const deleteActiveSubmissionById = async (
- submissionId: number,
- userName: string,
- ): Promise => {
- const { getSubmissionById, update } = submissionRepository(dependencies);
-
- const submission = await getSubmissionById(submissionId);
- if (!submission) {
- throw new BadRequest(`Submission '${submissionId}' not found`);
- }
-
- if (!canTransitionToClosed(submission.status)) {
- throw new StatusConflict('Only Submissions with statuses "OPEN", "VALID", "INVALID" can be deleted');
- }
-
- const updatedRecord = await update(submission.id, {
- status: SUBMISSION_STATUS.CLOSED,
- updatedBy: userName,
- });
-
- logger.info(LOG_MODULE, `Submission '${submissionId}' updated with new status '${SUBMISSION_STATUS.CLOSED}'`);
-
- return updatedRecord;
- };
-
- /**
- * Function to remove an entity from an Active Submission by given Submission ID
- * It validates resulting Active Submission running cross schema validation along with the existing Submitted Data
- * Returns the resulting Active Submission with its status
- * @param {number} submissionId
- * @param {string} entityName
- * @param {string} userName
- * @returns { Promise} Resulting Active Submittion
- */
- const deleteActiveSubmissionEntity = async (
- submissionId: number,
- userName: string,
- filter: {
- actionType: SubmissionActionType;
- entityName: string;
- index: number | null;
- },
- ): Promise => {
- const { getSubmissionById } = submissionRepository(dependencies);
-
- const submission = await getSubmissionById(submissionId);
- if (!submission) {
- throw new BadRequest(`Submission '${submissionId}' not found`);
- }
-
- if (
- SUBMISSION_ACTION_TYPE.Values.INSERTS.includes(filter.actionType) &&
- !_.has(submission.data.inserts, filter.entityName)
- ) {
- throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`);
- }
-
- if (
- SUBMISSION_ACTION_TYPE.Values.UPDATES.includes(filter.actionType) &&
- !_.has(submission.data.updates, filter.entityName)
- ) {
- throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`);
- }
-
- if (
- SUBMISSION_ACTION_TYPE.Values.DELETES.includes(filter.actionType) &&
- !_.has(submission.data.deletes, filter.entityName)
- ) {
- throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`);
- }
-
- // Remove entity from the Submission
- const updatedActiveSubmissionData = removeItemsFromSubmission(submission.data, {
- ...filter,
- });
-
- const updatedRecord = await performDataValidation({
- originalSubmission: submission,
- submissionData: updatedActiveSubmissionData,
- userName,
- });
-
- logger.info(LOG_MODULE, `Submission '${updatedRecord.id}' updated with new status '${updatedRecord.status}'`);
-
- return updatedRecord;
- };
+const processor = (dependencies: BaseDependencies) => {
+ const LOG_MODULE = 'SUBMISSION_PROCESSOR_SERVICE';
+ const { logger } = dependencies;
/**
* Finds and returns the dependent updates based on the provided submission update data.
@@ -294,7 +78,7 @@ const service = (dependencies: BaseDependencies) => {
submissionUpdateData: Record;
}): Promise<{ submissionUpdateData: SubmissionUpdateData; dependents: Record }[]> => {
const { getSubmittedDataFiltered } = submittedRepository(dependencies);
- const { searchDirectDependents } = submittedDataService(dependencies);
+ const { searchDirectDependents } = searchDataRelations(dependencies);
const dependentUpdates = Object.entries(submissionUpdateData).reduce<
Promise<{ submissionUpdateData: SubmissionUpdateData; dependents: Record }[]>
@@ -352,114 +136,6 @@ const service = (dependencies: BaseDependencies) => {
return dependentUpdates;
};
- /**
- * Get an active Submission by Category
- * @param {Object} params
- * @param {number} params.categoryId
- * @param {string} params.userName
- * @returns One Active Submission
- */
- const getActiveSubmissionsByCategory = async ({
- categoryId,
- userName,
- }: {
- categoryId: number;
- userName: string;
- }): Promise => {
- const { getActiveSubmissionsWithRelationsByCategory } = submissionRepository(dependencies);
-
- const submissions = await getActiveSubmissionsWithRelationsByCategory({ userName, categoryId });
- if (!submissions || submissions.length === 0) {
- return;
- }
-
- return submissions.map((response) => parseActiveSubmissionSummaryResponse(response));
- };
-
- /**
- * Get Active Submission by Submission ID
- * @param {number} submissionId A Submission ID
- * @returns One Active Submission
- */
- const getActiveSubmissionById = async (submissionId: number) => {
- const { getActiveSubmissionWithRelationsById } = submissionRepository(dependencies);
-
- const submission = await getActiveSubmissionWithRelationsById(submissionId);
- if (_.isEmpty(submission)) {
- return;
- }
-
- return parseActiveSubmissionResponse(submission);
- };
-
- /**
- * Get an active Submission by Organization
- * @param {Object} params
- * @param {number} params.categoryId
- * @param {string} params.userName
- * @param {string} params.organization
- * @returns One Active Submission
- */
- const getActiveSubmissionByOrganization = async ({
- categoryId,
- userName,
- organization,
- }: {
- categoryId: number;
- userName: string;
- organization: string;
- }): Promise => {
- const { getActiveSubmissionWithRelationsByOrganization } = submissionRepository(dependencies);
-
- const submission = await getActiveSubmissionWithRelationsByOrganization({ organization, userName, categoryId });
- if (_.isEmpty(submission)) {
- return;
- }
-
- return parseActiveSubmissionSummaryResponse(submission);
- };
-
- /**
- * Find the current Active Submission or Create an Open Active Submission with initial values and no schema data.
- * @param {object} params
- * @param {string} params.userName Owner of the Submission
- * @param {number} params.categoryId Category ID of the Submission
- * @param {string} params.organization Organization name
- * @returns {Submission} An Active Submission
- */
- const getOrCreateActiveSubmission = async (params: {
- userName: string;
- categoryId: number;
- organization: string;
- }): Promise => {
- const { categoryId, userName, organization } = params;
- const submissionRepo = submissionRepository(dependencies);
- const categoryRepo = categoryRepository(dependencies);
-
- const activeSubmission = await submissionRepo.getActiveSubmission({ categoryId, userName, organization });
- if (activeSubmission) {
- return activeSubmission;
- }
-
- const currentDictionary = await categoryRepo.getActiveDictionaryByCategory(categoryId);
-
- if (!currentDictionary) {
- throw new InternalServerError(`Dictionary in category '${categoryId}' not found`);
- }
-
- const newSubmissionInput: NewSubmission = {
- createdBy: userName,
- data: {},
- dictionaryCategoryId: categoryId,
- dictionaryId: currentDictionary.id,
- errors: {},
- organization: organization,
- status: SUBMISSION_STATUS.OPEN,
- };
-
- return submissionRepo.save(newSubmissionInput);
- };
-
/**
* This function iterates over records that are changing ID fields and fetches existing submitted data by `systemId`,
* then generates a record to be deleted and to be inserted.
@@ -622,7 +298,7 @@ const service = (dependencies: BaseDependencies) => {
dataSubmittedRepo.deleteBySystemId({
submissionId: submission.id,
systemId: item.systemId,
- diff: computeDataDiff(item.data as DataRecord, null),
+ diff: computeDataDiff(item.data, null),
userName,
});
});
@@ -710,197 +386,6 @@ const service = (dependencies: BaseDependencies) => {
});
};
- /**
- * Update Active Submission in database
- * @param {Object} input
- * @param {number} input.dictionaryId The Dictionary ID of the Submission
- * @param {SubmissionData} input.submissionData Data to be submitted grouped on inserts, updates and deletes
- * @param {number} input.idActiveSubmission ID of the Active Submission
- * @param {Record>} input.schemaErrors Array of schemaErrors
- * @param {string} input.userName User updating the active submission
- * @returns {Promise} An Active Submission updated
- */
- const updateActiveSubmission = async (input: {
- dictionaryId: number;
- submissionData: SubmissionData;
- idActiveSubmission: number;
- schemaErrors: Record>;
- userName: string;
- }): Promise => {
- const { dictionaryId, submissionData, idActiveSubmission, schemaErrors, userName } = input;
- const { update } = submissionRepository(dependencies);
- const newStatusSubmission =
- Object.keys(schemaErrors).length > 0 ? SUBMISSION_STATUS.INVALID : SUBMISSION_STATUS.VALID;
- // Update with new data
- const updatedActiveSubmission = await update(idActiveSubmission, {
- data: submissionData,
- status: newStatusSubmission,
- dictionaryId: dictionaryId,
- updatedBy: userName,
- errors: schemaErrors,
- });
-
- logger.info(
- LOG_MODULE,
- `Updated Active submission '${updatedActiveSubmission.id}' with status '${newStatusSubmission}' on category '${updatedActiveSubmission.dictionaryCategoryId}'`,
- );
- return updatedActiveSubmission;
- };
-
- /**
- * Construct a SubmissionUpdateData object per each file returning a Record type based on entityName
- * @param {Record} files
- * @param {SchemasDictionary} schemasDictionary,
- * @returns {Promise>}
- */
- const submissionUpdateDataFromFiles = async (
- files: Record,
- schemasDictionary: SchemasDictionary,
- ): Promise> => {
- const { getSubmittedDataBySystemId } = submittedRepository(dependencies);
- const results: Record = {};
-
- // Process files in parallel using Promise.all
- await Promise.all(
- Object.entries(files).map(async ([entityName, file]) => {
- const schema = schemasDictionary.schemas.find((schema) => schema.name === entityName);
- if (!schema) {
- throw new Error(`No schema found for : '${entityName}'`);
- }
- const parsedFileData = await tsvToJson(file.path, schema);
-
- // Process records concurrently using Promise.all
- const recordPromises = parsedFileData.records.map(async (record) => {
- const systemId = record['systemId']?.toString();
- const changeData = _.omit(record, 'systemId');
- if (!systemId) return;
-
- const foundSubmittedData = await getSubmittedDataBySystemId(systemId);
- if (foundSubmittedData?.data) {
- const diffData = computeDataDiff(foundSubmittedData.data, changeData);
- if (!_.isEmpty(diffData.old) && !_.isEmpty(diffData.new)) {
- // Initialize an array for each entityName
- if (!results[entityName]) {
- results[entityName] = [];
- }
-
- results[entityName].push({
- systemId: systemId,
- old: diffData.old,
- new: diffData.new,
- });
- }
- }
- });
-
- // Wait for all records of the current file to be processed
- await Promise.all(recordPromises);
- }),
- );
-
- return results;
- };
-
- /**
- * Validates and Creates the Entities Schemas of the Active Submission and stores it in the database
- * @param {object} params
- * @param {Express.Multer.File[]} params.files An array of files
- * @param {number} params.categoryId Category ID of the Submission
- * @param {string} params.organization Organization name
- * @param {string} params.userName User name creating the Submission
- * @returns The Active Submission created or Updated
- */
- const uploadSubmission = async ({
- files,
- categoryId,
- organization,
- userName,
- }: {
- files: Express.Multer.File[];
- categoryId: number;
- organization: string;
- userName: string;
- }): Promise => {
- logger.info(LOG_MODULE, `Processing '${files.length}' files on category id '${categoryId}'`);
- const { getActiveDictionaryByCategory } = categoryRepository(dependencies);
-
- if (files.length === 0) {
- return {
- status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
- description: 'No valid files for submission',
- batchErrors: [],
- inProcessEntities: [],
- };
- }
-
- const currentDictionary = await getActiveDictionaryByCategory(categoryId);
-
- if (_.isEmpty(currentDictionary)) {
- throw new BadRequest(`Dictionary in category '${categoryId}' not found`);
- }
-
- const schemasDictionary: SchemasDictionary = {
- name: currentDictionary.name,
- version: currentDictionary.version,
- schemas: currentDictionary.schemas,
- };
-
- // step 1 Validation. Validate entity type (filename matches dictionary entities, remove duplicates)
- const schemaNames: string[] = schemasDictionary.schemas.map((item) => item.name);
- const { validFileEntity, batchErrors: fileNamesErrors } = await checkFileNames(files, schemaNames);
-
- if (_.isEmpty(validFileEntity)) {
- logger.info(LOG_MODULE, `No valid files for submission`);
- }
-
- // step 2 Validation. Validate fieldNames (missing required fields based on schema)
- const { checkedEntities, fieldNameErrors } = await checkEntityFieldNames(schemasDictionary, validFileEntity);
-
- const batchErrors = [...fileNamesErrors, ...fieldNameErrors];
- const entitiesToProcess = Object.keys(checkedEntities);
-
- if (_.isEmpty(checkedEntities)) {
- logger.info(LOG_MODULE, 'Found errors on Submission files.', JSON.stringify(batchErrors));
- return {
- status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
- description: 'No valid entities in submission',
- batchErrors,
- inProcessEntities: entitiesToProcess,
- };
- }
-
- // Get Active Submission or Open a new one
- const activeSubmission = await getOrCreateActiveSubmission({ categoryId, userName, organization });
- const activeSubmissionId = activeSubmission.id;
-
- // Running Schema validation in the background do not need to wait
- // Result of validations will be stored in database
- validateFilesAsync(checkedEntities, {
- schemasDictionary,
- categoryId,
- organization,
- userName,
- });
-
- if (batchErrors.length === 0) {
- return {
- status: CREATE_SUBMISSION_STATUS.PROCESSING,
- description: 'Submission files are being processed',
- submissionId: activeSubmissionId,
- batchErrors,
- inProcessEntities: entitiesToProcess,
- };
- }
-
- return {
- status: CREATE_SUBMISSION_STATUS.PARTIAL_SUBMISSION,
- description: 'Some Submission files are being processed while others were unable to process',
- submissionId: activeSubmissionId,
- batchErrors,
- inProcessEntities: entitiesToProcess,
- };
- };
-
/**
* Void function to process and validate uploaded files on an Active Submission.
* Performs the schema data validation of data to be edited combined with all Submitted Data.
@@ -991,6 +476,97 @@ const service = (dependencies: BaseDependencies) => {
});
};
+ /**
+ * Construct a SubmissionUpdateData object per each file returning a Record type based on entityName
+ * @param {Record} files
+ * @param {SchemasDictionary} schemasDictionary,
+ * @returns {Promise>}
+ */
+ const submissionUpdateDataFromFiles = async (
+ files: Record,
+ schemasDictionary: SchemasDictionary,
+ ): Promise> => {
+ const { getSubmittedDataBySystemId } = submittedRepository(dependencies);
+ const results: Record = {};
+
+ // Process files in parallel using Promise.all
+ await Promise.all(
+ Object.entries(files).map(async ([entityName, file]) => {
+ const schema = schemasDictionary.schemas.find((schema) => schema.name === entityName);
+ if (!schema) {
+ throw new Error(`No schema found for : '${entityName}'`);
+ }
+ const parsedFileData = await tsvToJson(file.path, schema);
+
+ // Process records concurrently using Promise.all
+ const recordPromises = parsedFileData.records.map(async (record) => {
+ const systemId = record['systemId']?.toString();
+ const changeData = _.omit(record, 'systemId');
+ if (!systemId) return;
+
+ const foundSubmittedData = await getSubmittedDataBySystemId(systemId);
+ if (foundSubmittedData?.data) {
+ const diffData = computeDataDiff(foundSubmittedData.data, changeData);
+ if (!_.isEmpty(diffData.old) && !_.isEmpty(diffData.new)) {
+ // Initialize an array for each entityName
+ if (!results[entityName]) {
+ results[entityName] = [];
+ }
+
+ results[entityName].push({
+ systemId: systemId,
+ old: diffData.old,
+ new: diffData.new,
+ });
+ }
+ }
+ });
+
+ // Wait for all records of the current file to be processed
+ await Promise.all(recordPromises);
+ }),
+ );
+
+ return results;
+ };
+
+ /**
+ * Update Active Submission in database
+ * @param {Object} input
+ * @param {number} input.dictionaryId The Dictionary ID of the Submission
+ * @param {SubmissionData} input.submissionData Data to be submitted grouped on inserts, updates and deletes
+ * @param {number} input.idActiveSubmission ID of the Active Submission
+ * @param {Record>} input.schemaErrors Array of schemaErrors
+ * @param {string} input.userName User updating the active submission
+ * @returns {Promise} An Active Submission updated
+ */
+ const updateActiveSubmission = async (input: {
+ dictionaryId: number;
+ submissionData: SubmissionData;
+ idActiveSubmission: number;
+ schemaErrors: Record>;
+ userName: string;
+ }): Promise => {
+ const { dictionaryId, submissionData, idActiveSubmission, schemaErrors, userName } = input;
+ const { update } = submissionRepository(dependencies);
+ const newStatusSubmission =
+ Object.keys(schemaErrors).length > 0 ? SUBMISSION_STATUS.INVALID : SUBMISSION_STATUS.VALID;
+ // Update with new data
+ const updatedActiveSubmission = await update(idActiveSubmission, {
+ data: submissionData,
+ status: newStatusSubmission,
+ dictionaryId: dictionaryId,
+ updatedBy: userName,
+ errors: schemaErrors,
+ });
+
+ logger.info(
+ LOG_MODULE,
+ `Updated Active submission '${updatedActiveSubmission.id}' with status '${newStatusSubmission}' on category '${updatedActiveSubmission.dictionaryCategoryId}'`,
+ );
+ return updatedActiveSubmission;
+ };
+
/**
* Void function to process and validate uploaded files on an Active Submission.
* Performs the schema data validation combined with all Submitted Data.
@@ -1032,17 +608,12 @@ const service = (dependencies: BaseDependencies) => {
};
return {
- commitSubmission,
- deleteActiveSubmissionById,
- deleteActiveSubmissionEntity,
- getActiveSubmissionsByCategory,
- getActiveSubmissionById,
- getActiveSubmissionByOrganization,
- getOrCreateActiveSubmission,
- performDataValidation,
processEditFilesAsync,
- uploadSubmission,
+ performCommitSubmissionAsync,
+ performDataValidation,
+ updateActiveSubmission,
+ validateFilesAsync,
};
};
-export default service;
+export default processor;
diff --git a/packages/data-provider/src/services/submission/submission.ts b/packages/data-provider/src/services/submission/submission.ts
new file mode 100644
index 0000000..3e65486
--- /dev/null
+++ b/packages/data-provider/src/services/submission/submission.ts
@@ -0,0 +1,455 @@
+import * as _ from 'lodash-es';
+
+import { Dictionary as SchemasDictionary } from '@overture-stack/lectern-client';
+import { type NewSubmission, Submission, type SubmissionUpdateData } from '@overture-stack/lyric-data-model';
+
+import { BaseDependencies } from '../../config/config.js';
+import systemIdGenerator from '../../external/systemIdGenerator.js';
+import submissionRepository from '../../repository/activeSubmissionRepository.js';
+import categoryRepository from '../../repository/categoryRepository.js';
+import submittedRepository from '../../repository/submittedRepository.js';
+import { BadRequest, InternalServerError, StatusConflict } from '../../utils/errors.js';
+import {
+ canTransitionToClosed,
+ checkEntityFieldNames,
+ checkFileNames,
+ parseActiveSubmissionResponse,
+ parseActiveSubmissionSummaryResponse,
+ removeItemsFromSubmission,
+} from '../../utils/submissionUtils.js';
+import {
+ ActiveSubmissionSummaryResponse,
+ CommitSubmissionResult,
+ CREATE_SUBMISSION_STATUS,
+ type CreateSubmissionResult,
+ SUBMISSION_ACTION_TYPE,
+ SUBMISSION_STATUS,
+ type SubmissionActionType,
+} from '../../utils/types.js';
+import processor from './processor.js';
+
+const service = (dependencies: BaseDependencies) => {
+ const LOG_MODULE = 'SUBMISSION_SERVICE';
+ const { logger } = dependencies;
+ const { performCommitSubmissionAsync, performDataValidation } = processor(dependencies);
+
+ /**
+ * Runs Schema validation asynchronously and moves the Active Submission to Submitted Data
+ * @param {number} categoryId
+ * @param {number} submissionId
+ * @returns {Promise}
+ */
+ const commitSubmission = async (
+ categoryId: number,
+ submissionId: number,
+ userName: string,
+ ): Promise => {
+ const { getSubmissionById } = submissionRepository(dependencies);
+ const { getSubmittedDataByCategoryIdAndOrganization } = submittedRepository(dependencies);
+ const { getActiveDictionaryByCategory } = categoryRepository(dependencies);
+ const { generateIdentifier } = systemIdGenerator(dependencies);
+
+ const submission = await getSubmissionById(submissionId);
+ if (!submission) {
+ throw new BadRequest(`Submission '${submissionId}' not found`);
+ }
+
+ if (submission.dictionaryCategoryId !== categoryId) {
+ throw new BadRequest(`Category ID provided does not match the category for the Submission`);
+ }
+
+ if (submission.status !== SUBMISSION_STATUS.VALID) {
+ throw new StatusConflict('Submission does not have status VALID and cannot be committed');
+ }
+
+ const currentDictionary = await getActiveDictionaryByCategory(categoryId);
+ if (_.isEmpty(currentDictionary)) {
+ throw new BadRequest(`Dictionary in category '${categoryId}' not found`);
+ }
+
+ const submittedDataToValidate = await getSubmittedDataByCategoryIdAndOrganization(
+ categoryId,
+ submission?.organization,
+ );
+
+ const entitiesToProcess = new Set();
+
+ submittedDataToValidate?.forEach((data) => entitiesToProcess.add(data.entityName));
+
+ const insertsToValidate = submission.data?.inserts
+ ? Object.entries(submission.data.inserts).flatMap(([entityName, submissionData]) => {
+ entitiesToProcess.add(entityName);
+
+ return submissionData.records.map((record) => ({
+ data: record,
+ dictionaryCategoryId: categoryId,
+ entityName,
+ isValid: false, // By default, New Submitted Data is created as invalid until validation proves otherwise
+ organization: submission.organization,
+ originalSchemaId: submission.dictionaryId,
+ systemId: generateIdentifier(entityName, record),
+ createdBy: userName,
+ }));
+ })
+ : [];
+
+ const deleteDataArray = submission.data?.deletes
+ ? Object.entries(submission.data.deletes).flatMap(([entityName, submissionDeleteData]) => {
+ entitiesToProcess.add(entityName);
+ return submissionDeleteData;
+ })
+ : [];
+
+ const updateDataArray =
+ submission.data?.updates &&
+ Object.entries(submission.data.updates).reduce>(
+ (acc, [entityName, submissionUpdateData]) => {
+ entitiesToProcess.add(entityName);
+ submissionUpdateData.forEach((record) => {
+ acc[record.systemId] = record;
+ });
+ return acc;
+ },
+ {},
+ );
+
+ // To Commit Active Submission we need to validate SubmittedData + Active Submission
+ performCommitSubmissionAsync({
+ dataToValidate: {
+ inserts: insertsToValidate,
+ submittedData: submittedDataToValidate,
+ deletes: deleteDataArray,
+ updates: updateDataArray,
+ },
+ submission,
+ dictionary: currentDictionary,
+ userName: userName,
+ });
+
+ return {
+ status: CREATE_SUBMISSION_STATUS.PROCESSING,
+ dictionary: {
+ name: currentDictionary.name,
+ version: currentDictionary.version,
+ },
+ processedEntities: Array.from(entitiesToProcess.values()),
+ };
+ };
+
+ /**
+ * Updates Submission status to CLOSED
+ * This action is allowed only if current Submission Status as OPEN, VALID or INVALID
+ * Returns the resulting Active Submission with its status
+ * @param {number} submissionId
+ * @param {string} userName
+ * @returns {Promise}
+ */
+ const deleteActiveSubmissionById = async (
+ submissionId: number,
+ userName: string,
+ ): Promise => {
+ const { getSubmissionById, update } = submissionRepository(dependencies);
+
+ const submission = await getSubmissionById(submissionId);
+ if (!submission) {
+ throw new BadRequest(`Submission '${submissionId}' not found`);
+ }
+
+ if (!canTransitionToClosed(submission.status)) {
+ throw new StatusConflict('Only Submissions with statuses "OPEN", "VALID", "INVALID" can be deleted');
+ }
+
+ const updatedRecord = await update(submission.id, {
+ status: SUBMISSION_STATUS.CLOSED,
+ updatedBy: userName,
+ });
+
+ logger.info(LOG_MODULE, `Submission '${submissionId}' updated with new status '${SUBMISSION_STATUS.CLOSED}'`);
+
+ return updatedRecord;
+ };
+
+ /**
+ * Function to remove an entity from an Active Submission by given Submission ID
+ * It validates resulting Active Submission running cross schema validation along with the existing Submitted Data
+ * Returns the resulting Active Submission with its status
+ * @param {number} submissionId
+ * @param {string} entityName
+ * @param {string} userName
+ * @returns { Promise} Resulting Active Submittion
+ */
+ const deleteActiveSubmissionEntity = async (
+ submissionId: number,
+ userName: string,
+ filter: {
+ actionType: SubmissionActionType;
+ entityName: string;
+ index: number | null;
+ },
+ ): Promise => {
+ const { getSubmissionById } = submissionRepository(dependencies);
+
+ const submission = await getSubmissionById(submissionId);
+ if (!submission) {
+ throw new BadRequest(`Submission '${submissionId}' not found`);
+ }
+
+ if (
+ SUBMISSION_ACTION_TYPE.Values.INSERTS.includes(filter.actionType) &&
+ !_.has(submission.data.inserts, filter.entityName)
+ ) {
+ throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`);
+ }
+
+ if (
+ SUBMISSION_ACTION_TYPE.Values.UPDATES.includes(filter.actionType) &&
+ !_.has(submission.data.updates, filter.entityName)
+ ) {
+ throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`);
+ }
+
+ if (
+ SUBMISSION_ACTION_TYPE.Values.DELETES.includes(filter.actionType) &&
+ !_.has(submission.data.deletes, filter.entityName)
+ ) {
+ throw new BadRequest(`Entity '${filter.entityName}' not found on '${filter.actionType}' Submission`);
+ }
+
+ // Remove entity from the Submission
+ const updatedActiveSubmissionData = removeItemsFromSubmission(submission.data, {
+ ...filter,
+ });
+
+ const updatedRecord = await performDataValidation({
+ originalSubmission: submission,
+ submissionData: updatedActiveSubmissionData,
+ userName,
+ });
+
+ logger.info(LOG_MODULE, `Submission '${updatedRecord.id}' updated with new status '${updatedRecord.status}'`);
+
+ return updatedRecord;
+ };
+
+ /**
+ * Get an active Submission by Category
+ * @param {Object} params
+ * @param {number} params.categoryId
+ * @param {string} params.userName
+ * @returns One Active Submission
+ */
+ const getActiveSubmissionsByCategory = async ({
+ categoryId,
+ userName,
+ }: {
+ categoryId: number;
+ userName: string;
+ }): Promise => {
+ const { getActiveSubmissionsWithRelationsByCategory } = submissionRepository(dependencies);
+
+ const submissions = await getActiveSubmissionsWithRelationsByCategory({ userName, categoryId });
+ if (!submissions || submissions.length === 0) {
+ return;
+ }
+
+ return submissions.map((response) => parseActiveSubmissionSummaryResponse(response));
+ };
+
+ /**
+ * Get Active Submission by Submission ID
+ * @param {number} submissionId A Submission ID
+ * @returns One Active Submission
+ */
+ const getActiveSubmissionById = async (submissionId: number) => {
+ const { getActiveSubmissionWithRelationsById } = submissionRepository(dependencies);
+
+ const submission = await getActiveSubmissionWithRelationsById(submissionId);
+ if (_.isEmpty(submission)) {
+ return;
+ }
+
+ return parseActiveSubmissionResponse(submission);
+ };
+
+ /**
+ * Get an active Submission by Organization
+ * @param {Object} params
+ * @param {number} params.categoryId
+ * @param {string} params.userName
+ * @param {string} params.organization
+ * @returns One Active Submission
+ */
+ const getActiveSubmissionByOrganization = async ({
+ categoryId,
+ userName,
+ organization,
+ }: {
+ categoryId: number;
+ userName: string;
+ organization: string;
+ }): Promise => {
+ const { getActiveSubmissionWithRelationsByOrganization } = submissionRepository(dependencies);
+
+ const submission = await getActiveSubmissionWithRelationsByOrganization({ organization, userName, categoryId });
+ if (_.isEmpty(submission)) {
+ return;
+ }
+
+ return parseActiveSubmissionSummaryResponse(submission);
+ };
+
+ /**
+ * Find the current Active Submission or Create an Open Active Submission with initial values and no schema data.
+ * @param {object} params
+ * @param {string} params.userName Owner of the Submission
+ * @param {number} params.categoryId Category ID of the Submission
+ * @param {string} params.organization Organization name
+ * @returns {Submission} An Active Submission
+ */
+ const getOrCreateActiveSubmission = async (params: {
+ userName: string;
+ categoryId: number;
+ organization: string;
+ }): Promise => {
+ const { categoryId, userName, organization } = params;
+ const submissionRepo = submissionRepository(dependencies);
+ const categoryRepo = categoryRepository(dependencies);
+
+ const activeSubmission = await submissionRepo.getActiveSubmission({ categoryId, userName, organization });
+ if (activeSubmission) {
+ return activeSubmission;
+ }
+
+ const currentDictionary = await categoryRepo.getActiveDictionaryByCategory(categoryId);
+
+ if (!currentDictionary) {
+ throw new InternalServerError(`Dictionary in category '${categoryId}' not found`);
+ }
+
+ const newSubmissionInput: NewSubmission = {
+ createdBy: userName,
+ data: {},
+ dictionaryCategoryId: categoryId,
+ dictionaryId: currentDictionary.id,
+ errors: {},
+ organization: organization,
+ status: SUBMISSION_STATUS.OPEN,
+ };
+
+ return submissionRepo.save(newSubmissionInput);
+ };
+
+ /**
+ * Validates and Creates the Entities Schemas of the Active Submission and stores it in the database
+ * @param {object} params
+ * @param {Express.Multer.File[]} params.files An array of files
+ * @param {number} params.categoryId Category ID of the Submission
+ * @param {string} params.organization Organization name
+ * @param {string} params.userName User name creating the Submission
+ * @returns The Active Submission created or Updated
+ */
+ const uploadSubmission = async ({
+ files,
+ categoryId,
+ organization,
+ userName,
+ }: {
+ files: Express.Multer.File[];
+ categoryId: number;
+ organization: string;
+ userName: string;
+ }): Promise => {
+ logger.info(LOG_MODULE, `Processing '${files.length}' files on category id '${categoryId}'`);
+ const { getActiveDictionaryByCategory } = categoryRepository(dependencies);
+ const { validateFilesAsync } = processor(dependencies);
+
+ if (files.length === 0) {
+ return {
+ status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
+ description: 'No valid files for submission',
+ batchErrors: [],
+ inProcessEntities: [],
+ };
+ }
+
+ const currentDictionary = await getActiveDictionaryByCategory(categoryId);
+
+ if (_.isEmpty(currentDictionary)) {
+ throw new BadRequest(`Dictionary in category '${categoryId}' not found`);
+ }
+
+ const schemasDictionary: SchemasDictionary = {
+ name: currentDictionary.name,
+ version: currentDictionary.version,
+ schemas: currentDictionary.schemas,
+ };
+
+ // step 1 Validation. Validate entity type (filename matches dictionary entities, remove duplicates)
+ const schemaNames: string[] = schemasDictionary.schemas.map((item) => item.name);
+ const { validFileEntity, batchErrors: fileNamesErrors } = await checkFileNames(files, schemaNames);
+
+ if (_.isEmpty(validFileEntity)) {
+ logger.info(LOG_MODULE, `No valid files for submission`);
+ }
+
+ // step 2 Validation. Validate fieldNames (missing required fields based on schema)
+ const { checkedEntities, fieldNameErrors } = await checkEntityFieldNames(schemasDictionary, validFileEntity);
+
+ const batchErrors = [...fileNamesErrors, ...fieldNameErrors];
+ const entitiesToProcess = Object.keys(checkedEntities);
+
+ if (_.isEmpty(checkedEntities)) {
+ logger.info(LOG_MODULE, 'Found errors on Submission files.', JSON.stringify(batchErrors));
+ return {
+ status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
+ description: 'No valid entities in submission',
+ batchErrors,
+ inProcessEntities: entitiesToProcess,
+ };
+ }
+
+ // Get Active Submission or Open a new one
+ const activeSubmission = await getOrCreateActiveSubmission({ categoryId, userName, organization });
+ const activeSubmissionId = activeSubmission.id;
+
+ // Running Schema validation in the background do not need to wait
+ // Result of validations will be stored in database
+ validateFilesAsync(checkedEntities, {
+ schemasDictionary,
+ categoryId,
+ organization,
+ userName,
+ });
+
+ if (batchErrors.length === 0) {
+ return {
+ status: CREATE_SUBMISSION_STATUS.PROCESSING,
+ description: 'Submission files are being processed',
+ submissionId: activeSubmissionId,
+ batchErrors,
+ inProcessEntities: entitiesToProcess,
+ };
+ }
+
+ return {
+ status: CREATE_SUBMISSION_STATUS.PARTIAL_SUBMISSION,
+ description: 'Some Submission files are being processed while others were unable to process',
+ submissionId: activeSubmissionId,
+ batchErrors,
+ inProcessEntities: entitiesToProcess,
+ };
+ };
+
+ return {
+ commitSubmission,
+ deleteActiveSubmissionById,
+ deleteActiveSubmissionEntity,
+ getActiveSubmissionsByCategory,
+ getActiveSubmissionById,
+ getActiveSubmissionByOrganization,
+ getOrCreateActiveSubmission,
+ uploadSubmission,
+ };
+};
+
+export default service;
diff --git a/packages/data-provider/src/services/submittedData/searchDataRelations.ts b/packages/data-provider/src/services/submittedData/searchDataRelations.ts
new file mode 100644
index 0000000..d4330a3
--- /dev/null
+++ b/packages/data-provider/src/services/submittedData/searchDataRelations.ts
@@ -0,0 +1,88 @@
+import type { DataRecord } from '@overture-stack/lectern-client';
+import type { SubmittedData } from '@overture-stack/lyric-data-model';
+
+import type { BaseDependencies } from '../../config/config.js';
+import submittedRepository from '../../repository/submittedRepository.js';
+import type { SchemaChildNode } from '../../utils/dictionarySchemaRelations.js';
+import { mergeSubmittedDataAndDeduplicateById } from '../../utils/submittedDataUtils.js';
+
+const searchDataRelations = (dependencies: BaseDependencies) => {
+ const LOG_MODULE = 'SEARCH_DATA_RELATIONS_SERVICE';
+ const submittedDataRepo = submittedRepository(dependencies);
+ const { logger } = dependencies;
+ /**
+ * This function uses a dictionary children relations to query recursivaly
+ * to return all SubmittedData that relates
+ * @param input
+ * @param {DataRecord} input.data
+ * @param {Record} input.dictionaryRelations
+ * @param {string} input.entityName
+ * @param {string} input.organization
+ * @param {string} input.systemId
+ * @returns {Promise}
+ */
+ const searchDirectDependents = async ({
+ data,
+ dictionaryRelations,
+ entityName,
+ organization,
+ systemId,
+ }: {
+ data: DataRecord;
+ dictionaryRelations: Record;
+ entityName: string;
+ organization: string;
+ systemId: string;
+ }): Promise => {
+ const { getSubmittedDataFiltered } = submittedDataRepo;
+
+ // Check if entity has children relationships
+ if (Object.prototype.hasOwnProperty.call(dictionaryRelations, entityName)) {
+ // Array that represents the children fields to filter
+
+ const filterData: { entityName: string; dataField: string; dataValue: string | undefined }[] = Object.values(
+ dictionaryRelations[entityName],
+ )
+ .filter((childNode) => childNode.parent?.fieldName)
+ .map((childNode) => ({
+ entityName: childNode.schemaName,
+ dataField: childNode.fieldName,
+ dataValue: data[childNode.parent!.fieldName]?.toString(),
+ }));
+
+ logger.debug(
+ LOG_MODULE,
+ `Entity '${entityName}' has following dependencies filter'${JSON.stringify(filterData)}'`,
+ );
+
+ const directDependents = await getSubmittedDataFiltered(organization, filterData);
+
+ const additionalDepend = (
+ await Promise.all(
+ directDependents.map((record) =>
+ searchDirectDependents({
+ data: record.data,
+ dictionaryRelations,
+ entityName: record.entityName,
+ organization: record.organization,
+ systemId: record.systemId,
+ }),
+ ),
+ )
+ ).flatMap((item) => item);
+
+ const uniqueDependents = mergeSubmittedDataAndDeduplicateById(directDependents, additionalDepend);
+
+ logger.info(LOG_MODULE, `Found '${uniqueDependents.length}' records depending on system ID '${systemId}'`);
+
+ return uniqueDependents;
+ }
+
+ // return empty array when no dependents for this record
+ return [];
+ };
+
+ return { searchDirectDependents };
+};
+
+export default searchDataRelations;
diff --git a/packages/data-provider/src/services/submittedData/submmittedData.ts b/packages/data-provider/src/services/submittedData/submmittedData.ts
new file mode 100644
index 0000000..021a17e
--- /dev/null
+++ b/packages/data-provider/src/services/submittedData/submmittedData.ts
@@ -0,0 +1,472 @@
+import * as _ from 'lodash-es';
+
+import type { Dictionary as SchemasDictionary } from '@overture-stack/lectern-client';
+import { SQON } from '@overture-stack/sqon-builder';
+
+import { BaseDependencies } from '../../config/config.js';
+import categoryRepository from '../../repository/categoryRepository.js';
+import submittedRepository from '../../repository/submittedRepository.js';
+import { convertSqonToQuery } from '../../utils/convertSqonToQuery.js';
+import { getDictionarySchemaRelations } from '../../utils/dictionarySchemaRelations.js';
+import {
+ checkEntityFieldNames,
+ checkFileNames,
+ filterUpdatesFromDeletes,
+ mergeRecords,
+} from '../../utils/submissionUtils.js';
+import {
+ fetchDataErrorResponse,
+ getEntityNamesFromFilterOptions,
+ transformmSubmittedDataToSubmissionDeleteData,
+} from '../../utils/submittedDataUtils.js';
+import {
+ type BatchError,
+ CREATE_SUBMISSION_STATUS,
+ type CreateSubmissionStatus,
+ PaginationOptions,
+ SubmittedDataResponse,
+ VIEW_TYPE,
+ type ViewType,
+} from '../../utils/types.js';
+import processor from '../submission/processor.js';
+import submissionService from '../submission/submission.js';
+import searchDataRelations from './searchDataRelations.js';
+import viewMode from './viewMode.js';
+
+const PAGINATION_ERROR_MESSAGES = {
+ INVALID_CATEGORY_ID: 'Invalid Category ID',
+ NO_DATA_FOUND: 'No Submitted data found',
+} as const;
+
+const submittedData = (dependencies: BaseDependencies) => {
+ const LOG_MODULE = 'SUBMITTED_DATA_SERVICE';
+ const submittedDataRepo = submittedRepository(dependencies);
+ const { logger } = dependencies;
+ const { convertRecordsToCompoundDocuments } = viewMode(dependencies);
+ const { searchDirectDependents } = searchDataRelations(dependencies);
+
+ const deleteSubmittedDataBySystemId = async (
+ categoryId: number,
+ systemId: string,
+ userName: string,
+ ): Promise<{
+ description: string;
+ inProcessEntities: string[];
+ status: CreateSubmissionStatus;
+ submissionId?: string;
+ }> => {
+ const { getSubmittedDataBySystemId } = submittedDataRepo;
+ const { getActiveDictionaryByCategory } = categoryRepository(dependencies);
+ const { getOrCreateActiveSubmission } = submissionService(dependencies);
+ const { performDataValidation } = processor(dependencies);
+
+ // get SubmittedData by SystemId
+ const foundRecordToDelete = await getSubmittedDataBySystemId(systemId);
+
+ if (!foundRecordToDelete) {
+ return {
+ status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
+ description: `No Submitted data found with systemId '${systemId}'`,
+ inProcessEntities: [],
+ };
+ }
+ logger.info(LOG_MODULE, `Found Submitted Data with system ID '${systemId}'`);
+
+ if (foundRecordToDelete.dictionaryCategoryId !== categoryId) {
+ return {
+ status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
+ description: `Invalid Category ID '${categoryId}' for system ID '${systemId}'`,
+ inProcessEntities: [],
+ };
+ }
+
+ // get current dictionary
+ const currentDictionary = await getActiveDictionaryByCategory(categoryId);
+
+ if (!currentDictionary) {
+ return {
+ status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
+ description: `Dictionary not found`,
+ inProcessEntities: [],
+ };
+ }
+
+ // get dictionary relations
+ const dictionaryRelations = getDictionarySchemaRelations(currentDictionary.schemas);
+
+ const recordDependents = await searchDirectDependents({
+ data: foundRecordToDelete.data,
+ dictionaryRelations,
+ entityName: foundRecordToDelete.entityName,
+ organization: foundRecordToDelete.organization,
+ systemId: foundRecordToDelete.systemId,
+ });
+
+ const submittedDataToDelete = [foundRecordToDelete, ...recordDependents];
+
+ const recordsToDeleteMap = transformmSubmittedDataToSubmissionDeleteData(submittedDataToDelete);
+
+ // Get Active Submission or Open a new one
+ const activeSubmission = await getOrCreateActiveSubmission({
+ categoryId: foundRecordToDelete.dictionaryCategoryId,
+ userName,
+ organization: foundRecordToDelete.organization,
+ });
+
+ // Merge current Active Submission delete entities
+ const mergedSubmissionDeletes = mergeRecords(activeSubmission.data.deletes, recordsToDeleteMap);
+
+ const entitiesToProcess = Object.keys(mergedSubmissionDeletes);
+
+ // filter out update records found matching systemID on delete records
+ const filteredUpdates = filterUpdatesFromDeletes(activeSubmission.data.updates ?? {}, mergedSubmissionDeletes);
+
+ // Validate and update Active Submission
+ performDataValidation({
+ originalSubmission: activeSubmission,
+ submissionData: {
+ inserts: activeSubmission.data.inserts,
+ updates: filteredUpdates,
+ deletes: mergedSubmissionDeletes,
+ },
+ userName,
+ });
+
+ logger.info(LOG_MODULE, `Added '${entitiesToProcess.length}' records to be deleted on the Active Submission`);
+
+ return {
+ status: CREATE_SUBMISSION_STATUS.PROCESSING,
+ description: 'Submission data is being processed',
+ submissionId: activeSubmission.id.toString(),
+ inProcessEntities: entitiesToProcess,
+ };
+ };
+
+ const editSubmittedData = async ({
+ categoryId,
+ files,
+ organization,
+ userName,
+ }: {
+ categoryId: number;
+ files: Express.Multer.File[];
+ organization: string;
+ userName: string;
+ }): Promise<{
+ batchErrors: BatchError[];
+ description?: string;
+ inProcessEntities: string[];
+ submissionId?: number;
+ status: string;
+ }> => {
+ const { getActiveDictionaryByCategory } = categoryRepository(dependencies);
+ const { getOrCreateActiveSubmission } = submissionService(dependencies);
+ const { processEditFilesAsync } = processor(dependencies);
+ if (files.length === 0) {
+ return {
+ status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
+ description: 'No valid files for submission',
+ batchErrors: [],
+ inProcessEntities: [],
+ };
+ }
+
+ const currentDictionary = await getActiveDictionaryByCategory(categoryId);
+
+ if (_.isEmpty(currentDictionary)) {
+ return {
+ status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
+ description: `Dictionary in category '${categoryId}' not found`,
+ batchErrors: [],
+ inProcessEntities: [],
+ };
+ }
+
+ const schemasDictionary: SchemasDictionary = {
+ name: currentDictionary.name,
+ version: currentDictionary.version,
+ schemas: currentDictionary.schemas,
+ };
+
+ // step 1 Validation. Validate entity type (filename matches dictionary entities, remove duplicates)
+ const schemaNames: string[] = schemasDictionary.schemas.map((item) => item.name);
+ const { validFileEntity, batchErrors: fileNamesErrors } = await checkFileNames(files, schemaNames);
+
+ // step 2 Validation. Validate fieldNames (missing required fields based on schema)
+ const { checkedEntities, fieldNameErrors } = await checkEntityFieldNames(schemasDictionary, validFileEntity);
+
+ const batchErrors = [...fileNamesErrors, ...fieldNameErrors];
+ const entitiesToProcess = Object.keys(checkedEntities);
+
+ if (_.isEmpty(checkedEntities)) {
+ return {
+ status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
+ description: 'No valid entities in submission',
+ batchErrors,
+ inProcessEntities: entitiesToProcess,
+ };
+ }
+
+ // Get Active Submission or Open a new one
+ const activeSubmission = await getOrCreateActiveSubmission({ categoryId, userName, organization });
+
+ // Running Schema validation in the background do not need to wait
+ // Result of validations will be stored in database
+ processEditFilesAsync({
+ submission: activeSubmission,
+ files: checkedEntities,
+ schemasDictionary,
+ userName,
+ });
+
+ if (batchErrors.length === 0) {
+ return {
+ status: CREATE_SUBMISSION_STATUS.PROCESSING,
+ description: 'Submission files are being processed',
+ submissionId: activeSubmission.id,
+ batchErrors,
+ inProcessEntities: entitiesToProcess,
+ };
+ }
+
+ return {
+ status: CREATE_SUBMISSION_STATUS.PARTIAL_SUBMISSION,
+ description: 'Some Submission files are being processed while others were unable to process',
+ submissionId: activeSubmission.id,
+ batchErrors,
+ inProcessEntities: entitiesToProcess,
+ };
+ };
+
+ /**
+ * Fetches submitted data from the database based on the provided category ID, pagination options, and filter options.
+ *
+ * This function retrieves a list of submitted data associated with the specified `categoryId`.
+ * It also supports pagination, view representation and filtering based on entity names or a compound condition.
+ * The returned data includes both the retrieved records and metadata about the total number of records.
+ *
+ * @param categoryId - The ID of the category for which data is being fetched.
+ * @param paginationOptions - An object containing pagination options, such as page number and items per page.
+ * @param filterOptions - An object containing options for filtering the data.
+ * @param filterOptions.entityName - An optional array of entity names to filter the data by.
+ * @param filterOptions.view - An optional flag indicating the view type
+ * @returns A promise that resolves to an object containing:
+ * - `result`: An array of `SubmittedDataResponse` objects, representing the fetched data.
+ * - `metadata`: An object containing metadata about the fetched data, including the total number of records.
+ * If an error occurs during data retrieval, `metadata` will include an `errorMessage` property.
+ */
+ const getSubmittedDataByCategory = async (
+ categoryId: number,
+ paginationOptions: PaginationOptions,
+ filterOptions: { entityName?: string[]; view: ViewType },
+ ): Promise<{
+ result: SubmittedDataResponse[];
+ metadata: { totalRecords: number; errorMessage?: string };
+ }> => {
+ const { getSubmittedDataByCategoryIdPaginated, getTotalRecordsByCategoryId } = submittedDataRepo;
+
+ const { getCategoryById } = categoryRepository(dependencies);
+
+ const category = await getCategoryById(categoryId);
+
+ if (!category?.activeDictionary) {
+ return fetchDataErrorResponse(PAGINATION_ERROR_MESSAGES.INVALID_CATEGORY_ID);
+ }
+
+ const defaultCentricEntity = category.defaultCentricEntity || undefined;
+
+ let recordsPaginated = await getSubmittedDataByCategoryIdPaginated(categoryId, paginationOptions, {
+ entityNames: getEntityNamesFromFilterOptions(filterOptions, defaultCentricEntity),
+ });
+
+ if (recordsPaginated.length === 0) {
+ return fetchDataErrorResponse(PAGINATION_ERROR_MESSAGES.NO_DATA_FOUND);
+ }
+
+ if (filterOptions.view === VIEW_TYPE.Values.compound) {
+ recordsPaginated = await convertRecordsToCompoundDocuments({
+ dictionary: category.activeDictionary.dictionary,
+ records: recordsPaginated,
+ defaultCentricEntity: defaultCentricEntity,
+ });
+ }
+
+ const totalRecords = await getTotalRecordsByCategoryId(categoryId, {
+ entityNames: getEntityNamesFromFilterOptions(filterOptions, defaultCentricEntity),
+ });
+
+ logger.info(LOG_MODULE, `Retrieved '${recordsPaginated.length}' Submitted data on categoryId '${categoryId}'`);
+
+ return {
+ result: recordsPaginated,
+ metadata: {
+ totalRecords,
+ },
+ };
+ };
+
+ /**
+ * Fetches submitted data from the database based on the provided category ID, organization, pagination options, and optional filter options.
+ *
+ * This function retrieves a list of submitted data associated with the specified `categoryId` and `organization`.
+ * It supports a view representation, pagination and optional filtering using a structured query (`sqon`) or entity names.
+ * The result includes both the fetched data and metadata such as the total number of records and an error message if applicable.
+ *
+ * @param categoryId - The ID of the category for which data is being fetched.
+ * @param organization - The name of the organization to filter the data by.
+ * @param paginationOptions - An object containing pagination options, such as page number and items per page.
+ * @param filterOptions - Optional filtering options.
+ * @param filterOptions.sqon - An optional Structured Query Object Notation (SQON) for advanced filtering criteria.
+ * @param filterOptions.entityName - An optional array of entity names to filter the data by. Can include undefined entries.
+ * @param filterOptions.view - An optional flag indicating the view type
+ * @returns A promise that resolves to an object containing:
+ * - `result`: An array of `SubmittedDataResponse` objects, representing the fetched data.
+ * - `metadata`: An object containing metadata about the fetched data, including the total number of records.
+ * If an error occurs during data retrieval, `metadata` will include an `errorMessage` property.
+ */
+ const getSubmittedDataByOrganization = async (
+ categoryId: number,
+ organization: string,
+ paginationOptions: PaginationOptions,
+ filterOptions: { sqon?: SQON; entityName?: string[]; view: ViewType },
+ ): Promise<{ result: SubmittedDataResponse[]; metadata: { totalRecords: number; errorMessage?: string } }> => {
+ const { getSubmittedDataByCategoryIdAndOrganizationPaginated, getTotalRecordsByCategoryIdAndOrganization } =
+ submittedDataRepo;
+ const { getCategoryById } = categoryRepository(dependencies);
+
+ const category = await getCategoryById(categoryId);
+
+ if (!category?.activeDictionary) {
+ return fetchDataErrorResponse(PAGINATION_ERROR_MESSAGES.INVALID_CATEGORY_ID);
+ }
+
+ const defaultCentricEntity = category.defaultCentricEntity || undefined;
+
+ const sqonQuery = convertSqonToQuery(filterOptions?.sqon);
+
+ let recordsPaginated = await getSubmittedDataByCategoryIdAndOrganizationPaginated(
+ categoryId,
+ organization,
+ paginationOptions,
+ {
+ sql: sqonQuery,
+ entityNames: getEntityNamesFromFilterOptions(filterOptions, defaultCentricEntity),
+ },
+ );
+
+ if (recordsPaginated.length === 0) {
+ return fetchDataErrorResponse(PAGINATION_ERROR_MESSAGES.NO_DATA_FOUND);
+ }
+
+ if (filterOptions.view === VIEW_TYPE.Values.compound) {
+ recordsPaginated = await convertRecordsToCompoundDocuments({
+ dictionary: category.activeDictionary.dictionary,
+ records: recordsPaginated,
+ defaultCentricEntity: defaultCentricEntity,
+ });
+ }
+
+ const totalRecords = await getTotalRecordsByCategoryIdAndOrganization(categoryId, organization, {
+ sql: sqonQuery,
+ entityNames: getEntityNamesFromFilterOptions(filterOptions, defaultCentricEntity),
+ });
+
+ logger.info(
+ LOG_MODULE,
+ `Retrieved '${recordsPaginated.length}' Submitted data on categoryId '${categoryId}' organization '${organization}'`,
+ );
+
+ return {
+ result: recordsPaginated,
+ metadata: {
+ totalRecords,
+ },
+ };
+ };
+
+ /**
+ * Fetches submitted data from the database based on the specified category ID and system ID.
+ *
+ * This function retrieves the submitted data associated with a given `categoryId` and `systemId`.
+ * It supports a view representation defined by the `filterOptions`. The result includes both the
+ * fetched data and metadata, including an error message if applicable.
+ *
+ * @param categoryId - The ID of the category for which data is being fetched.
+ * @param systemId - The unique identifier for the system associated with the submitted data.
+ * @param filterOptions - An object containing options for data representation.
+ * @param filterOptions.view - The desired view type for the data representation, such as 'flat' or 'compound'.
+ * @returns A promise that resolves to an object containing:
+ * - `result`: The fetched `SubmittedDataResponse`, or `undefined` if no data is found.
+ * - `metadata`: An object containing metadata about the fetched data, including an optional `errorMessage` property.
+ */
+ const getSubmittedDataBySystemId = async (
+ categoryId: number,
+ systemId: string,
+ filterOptions: { view: ViewType },
+ ): Promise<{
+ result: SubmittedDataResponse | undefined;
+ metadata: { errorMessage?: string };
+ }> => {
+ // get SubmittedData by SystemId
+ const foundRecord = await submittedDataRepo.getSubmittedDataBySystemId(systemId);
+ logger.info(LOG_MODULE, `Found Submitted Data with system ID '${systemId}'`);
+
+ if (!foundRecord) {
+ return { result: undefined, metadata: { errorMessage: `No Submitted data found with systemId '${systemId}'` } };
+ }
+
+ if (foundRecord.dictionaryCategoryId !== categoryId) {
+ return {
+ result: undefined,
+ metadata: { errorMessage: `Invalid Category ID '${categoryId}' for system ID '${systemId}'` },
+ };
+ }
+
+ let recordResponse: SubmittedDataResponse = {
+ data: foundRecord.data,
+ entityName: foundRecord.entityName,
+ isValid: foundRecord.isValid,
+ organization: foundRecord.organization,
+ systemId: foundRecord.systemId,
+ };
+
+ if (filterOptions.view === VIEW_TYPE.Values.compound) {
+ const { getCategoryById } = categoryRepository(dependencies);
+
+ const category = await getCategoryById(foundRecord.dictionaryCategoryId);
+
+ if (!category?.activeDictionary) {
+ return { result: undefined, metadata: { errorMessage: `Invalid Category ID` } };
+ }
+
+ const defaultCentricEntity = category.defaultCentricEntity || undefined;
+
+ // Convert to compound records if the record matches the default centric entity type.
+ // If no default centric entity is defined, the record's entity type will be used
+ if (!defaultCentricEntity || defaultCentricEntity === foundRecord.entityName) {
+ const [convertedRecord] = await convertRecordsToCompoundDocuments({
+ dictionary: category.activeDictionary.dictionary,
+ records: [recordResponse],
+ defaultCentricEntity: defaultCentricEntity,
+ });
+
+ recordResponse = convertedRecord;
+ }
+ }
+
+ return {
+ result: recordResponse,
+ metadata: {},
+ };
+ };
+
+ return {
+ deleteSubmittedDataBySystemId,
+ editSubmittedData,
+ getSubmittedDataByCategory,
+ getSubmittedDataByOrganization,
+ getSubmittedDataBySystemId,
+ };
+};
+
+export default submittedData;
diff --git a/packages/data-provider/src/services/submittedData/viewMode.ts b/packages/data-provider/src/services/submittedData/viewMode.ts
new file mode 100644
index 0000000..b3d585b
--- /dev/null
+++ b/packages/data-provider/src/services/submittedData/viewMode.ts
@@ -0,0 +1,222 @@
+import type { Schema } from '@overture-stack/lectern-client';
+
+import type { BaseDependencies } from '../../config/config.js';
+import submittedRepository from '../../repository/submittedRepository.js';
+import { generateHierarchy, type TreeNode } from '../../utils/dictionarySchemaRelations.js';
+import { InternalServerError } from '../../utils/errors.js';
+import { pluralizeSchemaName } from '../../utils/submissionUtils.js';
+import { groupByEntityName } from '../../utils/submittedDataUtils.js';
+import { type DataRecordNested, ORDER_TYPE, type SubmittedDataResponse } from '../../utils/types.js';
+
+const viewMode = (dependencies: BaseDependencies) => {
+ const LOG_MODULE = 'VIEW_MODE_SERVICE';
+ const submittedDataRepo = submittedRepository(dependencies);
+ const recordHierarchy = dependencies.features?.recordHierarchy;
+ const { logger } = dependencies;
+
+ const convertRecordsToCompoundDocuments = async ({
+ dictionary,
+ records,
+ defaultCentricEntity,
+ }: {
+ dictionary: Schema[];
+ records: SubmittedDataResponse[];
+ defaultCentricEntity?: string;
+ }) => {
+ // get dictionary hierarchy structure
+ const hierarchyStructureDesc = generateHierarchy(dictionary, ORDER_TYPE.Values.desc);
+ const hierarchyStructureAsc = generateHierarchy(dictionary, ORDER_TYPE.Values.asc);
+
+ return await Promise.all(
+ records.map(async (record) => {
+ try {
+ const childNodes = await traverseChildNodes({
+ data: record.data,
+ entityName: record.entityName,
+ organization: record.organization,
+ schemaCentric: defaultCentricEntity || record.entityName,
+ treeNode: hierarchyStructureDesc,
+ });
+
+ const parentNodes = await traverseParentNodes({
+ data: record.data,
+ entityName: record.entityName,
+ organization: record.organization,
+ schemaCentric: defaultCentricEntity || record.entityName,
+ treeNode: hierarchyStructureAsc,
+ });
+
+ record.data = { ...record.data, ...childNodes, ...parentNodes };
+ } catch (error) {
+ logger.error(`Error converting record ${record.systemId} into compound document`, error);
+ throw new InternalServerError(`An error occurred while converting records into compound view`);
+ }
+ return record;
+ }),
+ );
+ };
+
+ /**
+ * Recursively traverses parent nodes of a schema tree and queries for related SubmittedData.
+ *
+ * This function takes in the current data record, entity name, organization, schema-centric information,
+ * and tree node structure, then filters and queries dependent records recursively, constructing a nested
+ * structure of related data.
+ *
+ * @param data - The current data record to traverse.
+ * @param entityName - The name of the entity (schema) associated with the current data.
+ * @param organization - The organization to which the data belongs.
+ * @param schemaCentric - The schema-centric identifier for filtering parent nodes.
+ * @param treeNode - The hierarchical structure representing schema relationships.
+ *
+ * @returns A promise that resolves to a nested `DataRecordNested` object, containing the traversed and filtered dependent data.
+ * If no parent nodes or dependencies exist, it returns an empty object.
+ *
+ */
+ const traverseParentNodes = async ({
+ data,
+ entityName,
+ organization,
+ schemaCentric,
+ treeNode,
+ }: {
+ data: DataRecordNested;
+ entityName: string;
+ organization: string;
+ schemaCentric: string;
+ treeNode: TreeNode[];
+ }): Promise => {
+ const { getSubmittedDataFiltered } = submittedDataRepo;
+
+ const parentNode = treeNode.find((node) => node.schemaName === schemaCentric)?.parent;
+
+ if (!parentNode || !parentNode.parentFieldName || !parentNode.schemaName) {
+ // return empty array when no dependents for this record
+ return {};
+ }
+
+ const filterData: { entityName: string; dataField: string; dataValue: string | undefined } = {
+ entityName: parentNode.schemaName,
+ dataField: parentNode.fieldName || '',
+ dataValue: data[parentNode.parentFieldName!]?.toString() || '',
+ };
+
+ logger.debug(
+ LOG_MODULE,
+ `Entity '${entityName}' has following dependencies filter '${JSON.stringify(filterData)}'`,
+ );
+
+ const directDependants = await getSubmittedDataFiltered(organization, [filterData]);
+
+ const groupedDependants = groupByEntityName(directDependants);
+
+ const result: DataRecordNested = {};
+ for (const [entityName, records] of Object.entries(groupedDependants)) {
+ const additionalRecordsForEntity = await Promise.all(
+ records.map(async (record) => {
+ const additional = await traverseParentNodes({
+ data: record.data,
+ entityName: record.entityName,
+ organization: record.organization,
+ schemaCentric: record.entityName,
+ treeNode,
+ });
+ return { ...record.data, ...additional };
+ }),
+ );
+
+ // Getting the first record as record can have only 1 parent
+ result[entityName] = additionalRecordsForEntity[0];
+ }
+
+ return result;
+ };
+
+ /**
+ * Recursively traverses child nodes of a schema tree and queries for related SubmittedData.
+ *
+ * This function takes in the current data record, entity name, organization, schema-centric information,
+ * and tree node structure, then filters and queries dependent records recursively, constructing a nested
+ * structure of related data.
+ *
+ * @param data - The current data record to traverse.
+ * @param entityName - The name of the entity (schema) associated with the current data.
+ * @param organization - The organization to which the data belongs.
+ * @param schemaCentric - The schema-centric identifier for filtering child nodes.
+ * @param treeNode - The hierarchical structure representing schema relationships.
+ *
+ * @returns A promise that resolves to a nested `DataRecordNested` object, containing the traversed and filtered dependent data.
+ * If no child nodes or dependencies exist, it returns an empty object.
+ *
+ */
+ const traverseChildNodes = async ({
+ data,
+ entityName,
+ organization,
+ schemaCentric,
+ treeNode,
+ }: {
+ data: DataRecordNested;
+ entityName: string;
+ organization: string;
+ schemaCentric: string;
+ treeNode: TreeNode[];
+ }): Promise => {
+ const { getSubmittedDataFiltered } = submittedDataRepo;
+
+ const childNode = treeNode
+ .find((node) => node.schemaName === schemaCentric)
+ ?.children?.filter((childNode) => childNode.parentFieldName && childNode.schemaName);
+
+ if (!childNode || childNode.length === 0) {
+ // return empty array when no dependents for this record
+ return {};
+ }
+
+ const filterData: { entityName: string; dataField: string; dataValue: string | undefined }[] = childNode.map(
+ (childNode) => ({
+ entityName: childNode.schemaName,
+ dataField: childNode.fieldName || '',
+ dataValue: data[childNode.parentFieldName!]?.toString() || '',
+ }),
+ );
+
+ logger.debug(
+ LOG_MODULE,
+ `Entity '${entityName}' has following dependencies filter '${JSON.stringify(filterData)}'`,
+ );
+
+ const directDependants = await getSubmittedDataFiltered(organization, filterData);
+
+ const groupedDependants = groupByEntityName(directDependants);
+
+ const result: DataRecordNested = {};
+ for (const [entityName, records] of Object.entries(groupedDependants)) {
+ // if enabled ensures that schema names are consistently pluralized
+ const dependantKeyName = recordHierarchy?.pluralizeSchemasName ? pluralizeSchemaName(entityName) : entityName;
+
+ const additionalRecordsForEntity = await Promise.all(
+ records.map(async (record) => {
+ const additional = await traverseChildNodes({
+ data: record.data,
+ entityName: record.entityName,
+ organization: record.organization,
+ schemaCentric: record.entityName,
+ treeNode,
+ });
+ return { ...record.data, ...additional };
+ }),
+ );
+
+ result[dependantKeyName] = additionalRecordsForEntity;
+ }
+
+ return result;
+ };
+
+ return {
+ convertRecordsToCompoundDocuments,
+ };
+};
+
+export default viewMode;
diff --git a/packages/data-provider/src/services/submittedDataService.ts b/packages/data-provider/src/services/submittedDataService.ts
deleted file mode 100644
index bfb5f5a..0000000
--- a/packages/data-provider/src/services/submittedDataService.ts
+++ /dev/null
@@ -1,395 +0,0 @@
-import * as _ from 'lodash-es';
-
-import type { DataRecord, Dictionary as SchemasDictionary } from '@overture-stack/lectern-client';
-import { SubmittedData } from '@overture-stack/lyric-data-model';
-import { SQON } from '@overture-stack/sqon-builder';
-
-import { BaseDependencies } from '../config/config.js';
-import categoryRepository from '../repository/categoryRepository.js';
-import submittedRepository from '../repository/submittedRepository.js';
-import submissionService from '../services/submissionService.js';
-import { convertSqonToQuery } from '../utils/convertSqonToQuery.js';
-import { getDictionarySchemaRelations, SchemaChildNode } from '../utils/dictionarySchemaRelations.js';
-import {
- checkEntityFieldNames,
- checkFileNames,
- filterUpdatesFromDeletes,
- mergeRecords,
-} from '../utils/submissionUtils.js';
-import {
- fetchDataErrorResponse,
- mergeSubmittedDataAndDeduplicateById,
- transformmSubmittedDataToSubmissionDeleteData,
-} from '../utils/submittedDataUtils.js';
-import {
- type BatchError,
- CREATE_SUBMISSION_STATUS,
- type CreateSubmissionStatus,
- PaginationOptions,
- SubmittedDataResponse,
-} from '../utils/types.js';
-
-const PAGINATION_ERROR_MESSAGES = {
- INVALID_CATEGORY_ID: 'Invalid Category ID',
- NO_DATA_FOUND: 'No Submitted data found',
-} as const;
-
-const service = (dependencies: BaseDependencies) => {
- const LOG_MODULE = 'SUBMITTED_DATA_SERVICE';
- const submittedDataRepo = submittedRepository(dependencies);
- const { logger } = dependencies;
-
- const deleteSubmittedDataBySystemId = async (
- categoryId: number,
- systemId: string,
- userName: string,
- ): Promise<{
- description: string;
- inProcessEntities: string[];
- status: CreateSubmissionStatus;
- submissionId?: string;
- }> => {
- const { getSubmittedDataBySystemId } = submittedDataRepo;
- const { getActiveDictionaryByCategory } = categoryRepository(dependencies);
- const { performDataValidation, getOrCreateActiveSubmission } = submissionService(dependencies);
-
- // get SubmittedData by SystemId
- const foundRecordToDelete = await getSubmittedDataBySystemId(systemId);
-
- if (!foundRecordToDelete) {
- return {
- status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
- description: `No Submitted data found with systemId '${systemId}'`,
- inProcessEntities: [],
- };
- }
- logger.info(LOG_MODULE, `Found Submitted Data with system ID '${systemId}'`);
-
- if (foundRecordToDelete.dictionaryCategoryId !== categoryId) {
- return {
- status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
- description: `Invalid Category ID '${categoryId}' for system ID '${systemId}'`,
- inProcessEntities: [],
- };
- }
-
- // get current dictionary
- const currentDictionary = await getActiveDictionaryByCategory(categoryId);
-
- if (!currentDictionary) {
- return {
- status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
- description: `Dictionary not found`,
- inProcessEntities: [],
- };
- }
-
- // get dictionary relations
- const dictionaryRelations = getDictionarySchemaRelations(currentDictionary.schemas);
-
- const recordDependents = await searchDirectDependents({
- data: foundRecordToDelete.data,
- dictionaryRelations,
- entityName: foundRecordToDelete.entityName,
- organization: foundRecordToDelete.organization,
- systemId: foundRecordToDelete.systemId,
- });
-
- const submittedDataToDelete = [foundRecordToDelete, ...recordDependents];
-
- const recordsToDeleteMap = transformmSubmittedDataToSubmissionDeleteData(submittedDataToDelete);
-
- // Get Active Submission or Open a new one
- const activeSubmission = await getOrCreateActiveSubmission({
- categoryId: foundRecordToDelete.dictionaryCategoryId,
- userName,
- organization: foundRecordToDelete.organization,
- });
-
- // Merge current Active Submission delete entities
- const mergedSubmissionDeletes = mergeRecords(activeSubmission.data.deletes, recordsToDeleteMap);
-
- const entitiesToProcess = Object.keys(mergedSubmissionDeletes);
-
- // filter out update records found matching systemID on delete records
- const filteredUpdates = filterUpdatesFromDeletes(activeSubmission.data.updates ?? {}, mergedSubmissionDeletes);
-
- // Validate and update Active Submission
- performDataValidation({
- originalSubmission: activeSubmission,
- submissionData: {
- inserts: activeSubmission.data.inserts,
- updates: filteredUpdates,
- deletes: mergedSubmissionDeletes,
- },
- userName,
- });
-
- logger.info(LOG_MODULE, `Added '${entitiesToProcess.length}' records to be deleted on the Active Submission`);
-
- return {
- status: CREATE_SUBMISSION_STATUS.PROCESSING,
- description: 'Submission data is being processed',
- submissionId: activeSubmission.id.toString(),
- inProcessEntities: entitiesToProcess,
- };
- };
-
- const editSubmittedData = async ({
- categoryId,
- files,
- organization,
- userName,
- }: {
- categoryId: number;
- files: Express.Multer.File[];
- organization: string;
- userName: string;
- }): Promise<{
- batchErrors: BatchError[];
- description?: string;
- inProcessEntities: string[];
- submissionId?: number;
- status: string;
- }> => {
- const { getActiveDictionaryByCategory } = categoryRepository(dependencies);
- const { processEditFilesAsync, getOrCreateActiveSubmission } = submissionService(dependencies);
- if (files.length === 0) {
- return {
- status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
- description: 'No valid files for submission',
- batchErrors: [],
- inProcessEntities: [],
- };
- }
-
- const currentDictionary = await getActiveDictionaryByCategory(categoryId);
-
- if (_.isEmpty(currentDictionary)) {
- return {
- status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
- description: `Dictionary in category '${categoryId}' not found`,
- batchErrors: [],
- inProcessEntities: [],
- };
- }
-
- const schemasDictionary: SchemasDictionary = {
- name: currentDictionary.name,
- version: currentDictionary.version,
- schemas: currentDictionary.schemas,
- };
-
- // step 1 Validation. Validate entity type (filename matches dictionary entities, remove duplicates)
- const schemaNames: string[] = schemasDictionary.schemas.map((item) => item.name);
- const { validFileEntity, batchErrors: fileNamesErrors } = await checkFileNames(files, schemaNames);
-
- // step 2 Validation. Validate fieldNames (missing required fields based on schema)
- const { checkedEntities, fieldNameErrors } = await checkEntityFieldNames(schemasDictionary, validFileEntity);
-
- const batchErrors = [...fileNamesErrors, ...fieldNameErrors];
- const entitiesToProcess = Object.keys(checkedEntities);
-
- if (_.isEmpty(checkedEntities)) {
- return {
- status: CREATE_SUBMISSION_STATUS.INVALID_SUBMISSION,
- description: 'No valid entities in submission',
- batchErrors,
- inProcessEntities: entitiesToProcess,
- };
- }
-
- // Get Active Submission or Open a new one
- const activeSubmission = await getOrCreateActiveSubmission({ categoryId, userName, organization });
-
- // Running Schema validation in the background do not need to wait
- // Result of validations will be stored in database
- processEditFilesAsync({
- submission: activeSubmission,
- files: checkedEntities,
- schemasDictionary,
- userName,
- });
-
- if (batchErrors.length === 0) {
- return {
- status: CREATE_SUBMISSION_STATUS.PROCESSING,
- description: 'Submission files are being processed',
- submissionId: activeSubmission.id,
- batchErrors,
- inProcessEntities: entitiesToProcess,
- };
- }
-
- return {
- status: CREATE_SUBMISSION_STATUS.PARTIAL_SUBMISSION,
- description: 'Some Submission files are being processed while others were unable to process',
- submissionId: activeSubmission.id,
- batchErrors,
- inProcessEntities: entitiesToProcess,
- };
- };
-
- const getSubmittedDataByCategory = async (
- categoryId: number,
- paginationOptions: PaginationOptions,
- filterOptions: { entityName?: (string | undefined)[] },
- ): Promise<{
- data: SubmittedDataResponse[];
- metadata: { totalRecords: number; errorMessage?: string };
- }> => {
- const { getSubmittedDataByCategoryIdPaginated, getTotalRecordsByCategoryId } = submittedDataRepo;
-
- const { categoryIdExists } = categoryRepository(dependencies);
-
- const isValidCategory = await categoryIdExists(categoryId);
-
- if (!isValidCategory) {
- return fetchDataErrorResponse(PAGINATION_ERROR_MESSAGES.INVALID_CATEGORY_ID);
- }
-
- const recordsPaginated = await getSubmittedDataByCategoryIdPaginated(categoryId, paginationOptions, {
- entityNames: filterOptions?.entityName,
- });
-
- if (recordsPaginated.length === 0) {
- return fetchDataErrorResponse(PAGINATION_ERROR_MESSAGES.NO_DATA_FOUND);
- }
-
- const totalRecords = await getTotalRecordsByCategoryId(categoryId, { entityNames: filterOptions?.entityName });
-
- logger.info(LOG_MODULE, `Retrieved '${recordsPaginated.length}' Submitted data on categoryId '${categoryId}'`);
-
- return {
- data: recordsPaginated,
- metadata: {
- totalRecords,
- },
- };
- };
-
- const getSubmittedDataByOrganization = async (
- categoryId: number,
- organization: string,
- paginationOptions: PaginationOptions,
- filterOptions?: { sqon?: SQON; entityName?: (string | undefined)[] },
- ): Promise<{ data: SubmittedDataResponse[]; metadata: { totalRecords: number; errorMessage?: string } }> => {
- const { getSubmittedDataByCategoryIdAndOrganizationPaginated, getTotalRecordsByCategoryIdAndOrganization } =
- submittedDataRepo;
- const { categoryIdExists } = categoryRepository(dependencies);
-
- const isValidCategory = await categoryIdExists(categoryId);
-
- if (!isValidCategory) {
- return fetchDataErrorResponse(PAGINATION_ERROR_MESSAGES.INVALID_CATEGORY_ID);
- }
-
- const sqonQuery = convertSqonToQuery(filterOptions?.sqon);
-
- const recordsPaginated = await getSubmittedDataByCategoryIdAndOrganizationPaginated(
- categoryId,
- organization,
- paginationOptions,
- { sql: sqonQuery, entityNames: filterOptions?.entityName },
- );
-
- if (recordsPaginated.length === 0) {
- return fetchDataErrorResponse(PAGINATION_ERROR_MESSAGES.NO_DATA_FOUND);
- }
-
- const totalRecords = await getTotalRecordsByCategoryIdAndOrganization(categoryId, organization, {
- sql: sqonQuery,
- entityNames: filterOptions?.entityName,
- });
-
- logger.info(
- LOG_MODULE,
- `Retrieved '${recordsPaginated.length}' Submitted data on categoryId '${categoryId}' organization '${organization}'`,
- );
-
- return {
- data: recordsPaginated,
- metadata: {
- totalRecords,
- },
- };
- };
-
- /**
- * This function uses a dictionary children relations to query recursivaly
- * to return all SubmittedData that relates
- * @param {Record} dictionaryRelations
- * @param {SubmittedData} submittedData
- * @returns {Promise}
- */
- const searchDirectDependents = async ({
- data,
- dictionaryRelations,
- entityName,
- organization,
- systemId,
- }: {
- data: DataRecord;
- dictionaryRelations: Record;
- entityName: string;
- organization: string;
- systemId: string;
- }): Promise => {
- const { getSubmittedDataFiltered } = submittedDataRepo;
-
- // Check if entity has children relationships
- if (Object.prototype.hasOwnProperty.call(dictionaryRelations, entityName)) {
- // Array that represents the children fields to filter
-
- const filterData: { entityName: string; dataField: string; dataValue: string | undefined }[] = Object.values(
- dictionaryRelations[entityName],
- )
- .filter((childNode) => childNode.parent?.fieldName)
- .map((childNode) => ({
- entityName: childNode.schemaName,
- dataField: childNode.fieldName,
- dataValue: data[childNode.parent!.fieldName]?.toString(),
- }));
-
- logger.debug(
- LOG_MODULE,
- `Entity '${entityName}' has following dependencies filter'${JSON.stringify(filterData)}'`,
- );
-
- const directDependents = await getSubmittedDataFiltered(organization, filterData);
-
- const additionalDepend = (
- await Promise.all(
- directDependents.map((record) =>
- searchDirectDependents({
- data: record.data,
- dictionaryRelations,
- entityName: record.entityName,
- organization: record.organization,
- systemId: record.systemId,
- }),
- ),
- )
- ).flatMap((item) => item);
-
- const uniqueDependents = mergeSubmittedDataAndDeduplicateById(directDependents, additionalDepend);
-
- logger.info(LOG_MODULE, `Found '${uniqueDependents.length}' records depending on system ID '${systemId}'`);
-
- return uniqueDependents;
- }
-
- // return empty array when no dependents for this record
- return [];
- };
-
- return {
- deleteSubmittedDataBySystemId,
- editSubmittedData,
- getSubmittedDataByCategory,
- getSubmittedDataByOrganization,
- searchDirectDependents,
- };
-};
-
-export default service;
diff --git a/packages/data-provider/src/utils/auditUtils.ts b/packages/data-provider/src/utils/auditUtils.ts
index 9e4b81f..88e29a3 100644
--- a/packages/data-provider/src/utils/auditUtils.ts
+++ b/packages/data-provider/src/utils/auditUtils.ts
@@ -30,7 +30,7 @@ export const convertToAuditEvent = (value: string): AuditAction | undefined => {
const parseResult = AUDIT_ACTION.safeParse(value.toUpperCase());
if (parseResult.success) {
- return parseResult.data as AuditAction;
+ return parseResult.data;
}
return undefined;
};
diff --git a/packages/data-provider/src/utils/convertSqonToQuery.ts b/packages/data-provider/src/utils/convertSqonToQuery.ts
index c47443a..05d80f3 100644
--- a/packages/data-provider/src/utils/convertSqonToQuery.ts
+++ b/packages/data-provider/src/utils/convertSqonToQuery.ts
@@ -155,11 +155,18 @@ export const parseSQON = (input: unknown): SQON | undefined => {
// TODO: SQL sanitization (https://github.com/overture-stack/lyric/issues/43)
} catch (error: unknown) {
if (isZodError(error)) {
- throw new BadRequest('Invalid SQON format', (error as ZodError).issues);
+ throw new BadRequest('Invalid SQON format', error.issues);
}
}
};
-const isZodError = (error: unknown) => {
- return error && typeof error === 'object' && 'name' in error && error.name === 'ZodError';
+const isZodError = (error: unknown): error is ZodError => {
+ if (error instanceof ZodError) {
+ return true;
+ }
+
+ if (error && typeof error === 'object' && 'name' in error && error.name === 'ZodError') {
+ return true;
+ }
+ return false;
};
diff --git a/packages/data-provider/src/utils/dictionarySchemaRelations.ts b/packages/data-provider/src/utils/dictionarySchemaRelations.ts
index 43922db..df4fa8d 100644
--- a/packages/data-provider/src/utils/dictionarySchemaRelations.ts
+++ b/packages/data-provider/src/utils/dictionarySchemaRelations.ts
@@ -1,5 +1,7 @@
import { Schema } from '@overture-stack/lectern-client';
+import { ORDER_TYPE, type OrderType, SCHEMA_RELATION_TYPE, type SchemaRelationType } from './types.js';
+
export interface SchemaParentNode {
schemaName: string;
fieldName: string;
@@ -47,3 +49,122 @@ export const getDictionarySchemaRelations = (
return acc;
}, {});
};
+
+export interface TreeNode {
+ schemaName: string;
+ parentFieldName?: string;
+ fieldName?: string;
+ children?: TreeNode[];
+ parent?: TreeNode;
+}
+
+/**
+ * Function to find or create a node in the tree
+ * @param tree
+ * @param schemaName
+ * @param order
+ * @returns
+ */
+const findOrCreateNode = (tree: TreeNode[], schemaName: string, order: OrderType): TreeNode => {
+ let node = tree.find((n) => n.schemaName === schemaName);
+ if (!node) {
+ node = {
+ schemaName,
+ ...(order === ORDER_TYPE.Values.desc ? { children: [] } : { parent: undefined }),
+ };
+ tree.push(node);
+ }
+ return node;
+};
+
+/**
+ * Finds a matching schema name within a nested object.
+ * Return true only if any matching schema name is found.
+ * @param treeNode
+ * @param schemaName
+ * @param type
+ * @returns
+ */
+const hasNestedNode = (treeNode: TreeNode, schemaName: string, type: SchemaRelationType): boolean => {
+ if (type === SCHEMA_RELATION_TYPE.Values.parent) {
+ if (!treeNode.parent) return false;
+ return hasNestedNode(treeNode.parent, schemaName, type);
+ } else {
+ if (!treeNode.children) return false;
+ return treeNode.children.some(
+ (node) =>
+ node.schemaName === schemaName ||
+ node.children?.some((innerNode) => hasNestedNode(innerNode, schemaName, type)),
+ );
+ }
+};
+
+/**
+ * Builds a hierarchy tree by linking the given schema to its parent schema based on foreign keys.
+ * This function recursively creates or finds nodes in the `tree` and establishes parent-child relationships
+ * between schemas using their foreign key mappings. It ensures that duplicate relationships are not created.
+ * @param schema The current schema definition to be added to the hierarchy tree
+ * @param tree The current tree of schema nodes, which gets updated with parent-child relationships as the hierarchy is built.
+ * @param order Order of the structure
+ * @returns
+ */
+const buildHierarchyTree = (schema: SchemaDefinition, tree: TreeNode[], order: OrderType) => {
+ // Create a node for the current schema
+ const node = findOrCreateNode(tree, schema.name, order);
+
+ schema.restrictions?.foreignKey?.forEach((foreignKey) => {
+ // Find the related schema by its name
+ const relatedSchema = findOrCreateNode(tree, foreignKey.schema, order);
+
+ // Use the first mapping for parent-child field relationship
+ const mapping = foreignKey.mappings[0];
+
+ // remove duplicates. skip mapping when schema is already linked
+ if (order === ORDER_TYPE.Values.desc) {
+ const cloneNode = {
+ ...node,
+ fieldName: mapping.local,
+ parentFieldName: mapping.foreign,
+ };
+ relatedSchema.children = (
+ relatedSchema.children?.filter(
+ (item) => !hasNestedNode(cloneNode, item.schemaName, SCHEMA_RELATION_TYPE.Values.children),
+ ) || []
+ ).concat(cloneNode);
+ } else {
+ const cloneNode = {
+ ...relatedSchema,
+ fieldName: mapping.foreign,
+ parentFieldName: mapping.local,
+ };
+
+ // Remove duplicates. Skip mapping when schema is already linked
+ node.parent = !hasNestedNode(node, cloneNode.schemaName, SCHEMA_RELATION_TYPE.Values.parent)
+ ? cloneNode
+ : node.parent;
+ }
+ });
+};
+
+/**
+ * Function to generate the hierarchy tree of a dictionary schemas
+ * Order by `asc` should return children to parent relations
+ * Order by `desc` should return parent to chilren relations
+ * @param source The list of all schemas.
+ * @param order Order of the structed.
+ * @returns The hierarchical tree structure.
+ */
+export const generateHierarchy = (source: SchemaDefinition[], order: OrderType): TreeNode[] => {
+ const tree: TreeNode[] = [];
+
+ source
+ .sort((schemaA, schemaB) => {
+ // Sorting starts with the schemas that have no foreign keys (root nodes)
+ const a = schemaA.restrictions?.foreignKey ? schemaA.restrictions.foreignKey.length : 0;
+ const b = schemaB.restrictions?.foreignKey ? schemaB.restrictions.foreignKey.length : 0;
+ return order === ORDER_TYPE.Values.desc ? b - a : a - b;
+ })
+ .forEach((schema) => buildHierarchyTree(schema, tree, order));
+
+ return tree;
+};
diff --git a/packages/data-provider/src/utils/formatUtils.ts b/packages/data-provider/src/utils/formatUtils.ts
index 68c1568..cdc4728 100644
--- a/packages/data-provider/src/utils/formatUtils.ts
+++ b/packages/data-provider/src/utils/formatUtils.ts
@@ -62,11 +62,15 @@ export function isValidDateFormat(value: string): boolean {
/**
* Ensure a value is wrapped in an array.
*
- * If passed an array, return it without change. If passed a single item, wrap it in an array.
+ * If passed an array, return it returns the same array. If passed a single item, wrap it in an array.
+ * The function then filters out any empty strings and `undefined` values
* @param val an item or array
* @return an array
*/
-export const asArray = (val: T | T[]): T[] => (Array.isArray(val) ? val : [val]);
+export const asArray = (val: T | T[]): T[] => {
+ const result = Array.isArray(val) ? val : [val];
+ return result.filter((item) => item !== null && item !== '' && item !== undefined);
+};
/**
* Performs a deep comparison between two values to determine if they are deeply equal.
@@ -81,14 +85,19 @@ export const deepCompare = (obj1: unknown, obj2: unknown): boolean => {
return false; // Ensure both are non-null objects
}
+ // Ensure obj1 and obj2 are both records (i.e., objects)
+ if (!isObject(obj1) || !isObject(obj2)) {
+ return false;
+ }
+
const keys1 = Object.keys(obj1);
const keys2 = Object.keys(obj2);
if (keys1.length !== keys2.length) return false; // Different number of keys
for (const key of keys1) {
- const val1 = (obj1 as Record)[key];
- const val2 = (obj2 as Record)[key];
+ const val1 = obj1[key];
+ const val2 = obj2[key];
if (!keys2.includes(key) || !deepCompare(val1, val2)) {
return false;
@@ -97,3 +106,8 @@ export const deepCompare = (obj1: unknown, obj2: unknown): boolean => {
return true;
};
+
+// Helper function to check if an object is a plain object
+function isObject(obj: unknown): obj is Record {
+ return typeof obj === 'object' && obj !== null && !Array.isArray(obj);
+}
diff --git a/packages/data-provider/src/utils/schemas.ts b/packages/data-provider/src/utils/schemas.ts
index ccf8154..b0d2082 100644
--- a/packages/data-provider/src/utils/schemas.ts
+++ b/packages/data-provider/src/utils/schemas.ts
@@ -8,6 +8,7 @@ import { isAuditEventValid, isSubmissionActionTypeValid } from './auditUtils.js'
import { parseSQON } from './convertSqonToQuery.js';
import { isValidDateFormat, isValidIdNumber } from './formatUtils.js';
import { RequestValidation } from './requestValidation.js';
+import { VIEW_TYPE } from './types.js';
const auditEventTypeSchema = z
.string()
@@ -15,6 +16,8 @@ const auditEventTypeSchema = z
.min(1)
.refine((value) => isAuditEventValid(value), 'invalid Event Type');
+const viewSchema = z.string().toLowerCase().trim().min(1).pipe(VIEW_TYPE);
+
const categoryIdSchema = z
.string()
.trim()
@@ -200,7 +203,8 @@ export const cagegoryDetailsRequestSchema: RequestValidation = {
query: z
.object({
entityName: z.union([entityNameSchema, entityNameSchema.array()]).optional(),
+ view: viewSchema.optional(),
})
- .merge(paginationQuerySchema),
+ .merge(paginationQuerySchema)
+ .superRefine((data, ctx) => {
+ if (data.view === VIEW_TYPE.Values.compound && data.entityName && data.entityName?.length > 0) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: 'is incompatible with `compound` view',
+ path: ['entityName'],
+ });
+ }
+ }),
pathParams: categoryPathParamsSchema,
};
@@ -324,12 +344,22 @@ export const dataGetByOrganizationRequestSchema: RequestValidation<
query: z
.object({
entityName: z.union([entityNameSchema, entityNameSchema.array()]).optional(),
+ view: viewSchema.optional(),
})
- .merge(paginationQuerySchema),
+ .merge(paginationQuerySchema)
+ .superRefine((data, ctx) => {
+ if (data.view === VIEW_TYPE.Values.compound && data.entityName && data.entityName?.length > 0) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: 'is incompatible with `compound` view',
+ path: ['entityName'],
+ });
+ }
+ }),
pathParams: categoryOrganizationPathParamsSchema,
};
-export const dataGetByQueryRequestschema: RequestValidation = {
+export const dataGetByQueryRequestSchema: RequestValidation = {
body: sqonSchema,
query: z
.object({
@@ -338,3 +368,22 @@ export const dataGetByQueryRequestschema: RequestValidation = {
+ query: z.object({
+ view: viewSchema.optional(),
+ }),
+ pathParams: z.object({
+ systemId: stringNotEmpty,
+ categoryId: categoryIdSchema,
+ }),
+};
diff --git a/packages/data-provider/src/utils/submissionUtils.ts b/packages/data-provider/src/utils/submissionUtils.ts
index 491ff8d..430278c 100644
--- a/packages/data-provider/src/utils/submissionUtils.ts
+++ b/packages/data-provider/src/utils/submissionUtils.ts
@@ -1,4 +1,5 @@
import * as _ from 'lodash-es';
+import plur from 'plur';
import {
type DataRecord,
@@ -28,12 +29,10 @@ import {
ActiveSubmissionSummaryResponse,
BATCH_ERROR_TYPE,
BatchError,
- CategoryActiveSubmission,
type DataDeletesActiveSubmissionSummary,
DataInsertsActiveSubmissionSummary,
DataRecordReference,
type DataUpdatesActiveSubmissionSummary,
- DictionaryActiveSubmission,
type EditSubmittedDataReference,
MERGE_REFERENCE_TYPE,
type NewSubmittedDataReference,
@@ -358,12 +357,12 @@ export const groupSchemaErrorsByEntity = (input: {
* This function extracts the Schema Data from the Active Submission
* and maps it to it's original reference Id
* The result mapping is used to perform the cross schema validation
- * @param {number | undefined} activeSubmissionId
+ * @param {number} activeSubmissionId
* @param {Record} activeSubmissionInsertDataEntities
* @returns {Record}
*/
export const mapInsertDataToRecordReferences = (
- activeSubmissionId: number | undefined,
+ activeSubmissionId: number,
activeSubmissionInsertDataEntities: Record,
): Record => {
return _.mapValues(activeSubmissionInsertDataEntities, (submissionInsertData) =>
@@ -374,7 +373,7 @@ export const mapInsertDataToRecordReferences = (
submissionId: activeSubmissionId,
type: MERGE_REFERENCE_TYPE.NEW_SUBMITTED_DATA,
index: index,
- } as NewSubmittedDataReference,
+ },
};
}),
);
@@ -615,8 +614,8 @@ export const parseActiveSubmissionResponse = (
return {
id: submission.id,
data: submission.data,
- dictionary: submission.dictionary as DictionaryActiveSubmission,
- dictionaryCategory: submission.dictionaryCategory as CategoryActiveSubmission,
+ dictionary: submission.dictionary,
+ dictionaryCategory: submission.dictionaryCategory,
errors: submission.errors,
organization: _.toString(submission.organization),
status: submission.status,
@@ -668,8 +667,8 @@ export const parseActiveSubmissionSummaryResponse = (
return {
id: submission.id,
data: { inserts: dataInsertsSummary, updates: dataUpdatesSummary, deletes: dataDeletesSummary },
- dictionary: submission.dictionary as DictionaryActiveSubmission,
- dictionaryCategory: submission.dictionaryCategory as CategoryActiveSubmission,
+ dictionary: submission.dictionary,
+ dictionaryCategory: submission.dictionaryCategory,
errors: submission.errors,
organization: _.toString(submission.organization),
status: submission.status,
@@ -680,6 +679,10 @@ export const parseActiveSubmissionSummaryResponse = (
};
};
+export const pluralizeSchemaName = (schemaName: string) => {
+ return plur(schemaName);
+};
+
export const removeItemsFromSubmission = (
submissionData: SubmissionData,
filter: { actionType: SubmissionActionType; entityName: string; index: number | null },
diff --git a/packages/data-provider/src/utils/submittedDataUtils.ts b/packages/data-provider/src/utils/submittedDataUtils.ts
index 86ee504..41f9473 100644
--- a/packages/data-provider/src/utils/submittedDataUtils.ts
+++ b/packages/data-provider/src/utils/submittedDataUtils.ts
@@ -19,6 +19,8 @@ import {
MERGE_REFERENCE_TYPE,
type MutableDataDiff,
type MutableDataRecord,
+ VIEW_TYPE,
+ type ViewType,
} from './types.js';
/**
@@ -80,6 +82,23 @@ export const computeDataDiff = (oldRecord: DataRecord | null, newRecord: DataRec
return diff;
};
+/**
+ * Convert a value into it's View type if it matches.
+ * Otherwise it returns `undefined`
+ * @param {unknown} value
+ * @returns {ViewType | undefined}
+ */
+export const convertToViewType = (value: unknown): ViewType | undefined => {
+ if (typeof value === 'string') {
+ const parseResult = VIEW_TYPE.safeParse(value.trim().toLowerCase());
+
+ if (parseResult.success) {
+ return parseResult.data;
+ }
+ }
+ return undefined;
+};
+
/**
* Abstract Error response
* @param error
@@ -88,11 +107,11 @@ export const computeDataDiff = (oldRecord: DataRecord | null, newRecord: DataRec
export const fetchDataErrorResponse = (
error: string,
): {
- data: [];
+ result: [];
metadata: { totalRecords: number; errorMessage?: string };
} => {
return {
- data: [],
+ result: [],
metadata: {
totalRecords: 0,
errorMessage: error,
@@ -100,6 +119,31 @@ export const fetchDataErrorResponse = (
};
};
+/**
+ * Returns a list of entity names based on the provided filter options
+ *
+ * If the `view` flag is set in the `filterOptions` and a `defaultCentricEntity` exists
+ * it returns an array containing the `defaultCentricEntity`.
+ * Otherwise, it returns the `entityName` from `filterOptions`, if provided.
+ *
+ * @param filterOptions An object containing the view flag and the entity name array.
+ * @param filterOptions.view A flag indicating the type of view to represent the records
+ * @param filterOptions.entityName An array of entity names, used if view is not compound.
+ * @param defaultCentricEntity The default centric entity name
+ * @returns An array of entity names or empty array if no conditions are met.
+ */
+export const getEntityNamesFromFilterOptions = (
+ filterOptions: { view: ViewType; entityName?: string[] },
+ defaultCentricEntity?: string,
+): string[] => {
+ if (filterOptions.view === VIEW_TYPE.Values.compound && defaultCentricEntity) {
+ return [defaultCentricEntity];
+ } else if (filterOptions.entityName) {
+ return filterOptions.entityName.filter((name) => name);
+ }
+ return []; // Return an empty array if no conditions are met
+};
+
/**
* Groupd Submitted Data by entityName
* @param dataArray Array of data to group
diff --git a/packages/data-provider/src/utils/types.ts b/packages/data-provider/src/utils/types.ts
index 6d7b0df..1f8e76d 100644
--- a/packages/data-provider/src/utils/types.ts
+++ b/packages/data-provider/src/utils/types.ts
@@ -3,11 +3,14 @@ import { z } from 'zod';
import {
type DataRecord,
+ type DataRecordValue,
Dictionary as SchemasDictionary,
DictionaryValidationRecordErrorDetails,
} from '@overture-stack/lectern-client';
import {
+ type Category,
type DataDiff,
+ type Dictionary,
NewSubmittedData,
Submission,
SubmissionData,
@@ -222,8 +225,8 @@ export type CategoryActiveSubmission = {
export type ActiveSubmissionResponse = {
id: number;
data: SubmissionData;
- dictionary: DictionaryActiveSubmission | null;
- dictionaryCategory: CategoryActiveSubmission | null;
+ dictionary: DictionaryActiveSubmission;
+ dictionaryCategory: CategoryActiveSubmission;
errors: Record> | null;
organization: string;
status: SubmissionStatus | null;
@@ -251,8 +254,8 @@ export type ActiveSubmissionSummaryResponse = Omit;
+ dictionaryCategory: Pick;
errors: Record> | null;
organization: string | null;
status: SubmissionStatus | null;
@@ -264,7 +267,7 @@ export type ActiveSubmissionSummaryRepository = {
export type CategoryDetailsResponse = {
id: number;
- dictionary?: { name: string; version: string };
+ dictionary?: Pick;
name: string;
organizations: string[];
createdAt: string;
@@ -289,7 +292,7 @@ export type ListAllCategoriesResponse = {
* Submitted Raw Data information
*/
export type SubmittedDataResponse = {
- data: DataRecord;
+ data: DataRecordNested;
entityName: string;
isValid: boolean;
organization: string;
@@ -368,6 +371,10 @@ export type DataRecordReference = {
reference: SubmittedDataReference | NewSubmittedDataReference | EditSubmittedDataReference;
};
+export interface DataRecordNested {
+ [key: string]: DataRecordValue | DataRecordNested | DataRecordNested[];
+}
+
/**
* Keys of an object type as a union
*
@@ -399,3 +406,21 @@ export type Values = T extends infer U ? U[keyof U] : never;
* This will display type as an object with key: value pairs instead as an alias name.
*/
export type Clean = T extends infer U ? { [K in keyof U]: U[K] } : never;
+
+/**
+ * Enum matching Schema relationships types
+ */
+export const SCHEMA_RELATION_TYPE = z.enum(['parent', 'children']);
+export type SchemaRelationType = z.infer;
+
+/**
+ * Enum matching Schema relationships order types
+ */
+export const ORDER_TYPE = z.enum(['asc', 'desc']);
+export type OrderType = z.infer;
+
+/**
+ * Enum matching Retrieve data views
+ */
+export const VIEW_TYPE = z.enum(['flat', 'compound']);
+export type ViewType = z.infer;
diff --git a/packages/data-provider/test/utils/dictionarySchemaRelations.spec.ts b/packages/data-provider/test/utils/dictionarySchemaRelations.spec.ts
index 1f55e69..5878f4e 100644
--- a/packages/data-provider/test/utils/dictionarySchemaRelations.spec.ts
+++ b/packages/data-provider/test/utils/dictionarySchemaRelations.spec.ts
@@ -5,28 +5,280 @@ import { Schema } from '@overture-stack/lectern-client';
interface SchemaDefinition extends Schema {}
-import { getDictionarySchemaRelations } from '../../src/utils/dictionarySchemaRelations.js';
-import { dictionarySportStats, dictionarySportStatsNodeGraph } from './fixtures/dictionarySchemasTestData.js';
+import { generateHierarchy, getDictionarySchemaRelations } from '../../src/utils/dictionarySchemaRelations.js';
+import {
+ dictionaryClinicalSchemas,
+ dictionarySportStats,
+ dictionarySportStatsNodeGraph,
+} from './fixtures/dictionarySchemasTestData.js';
describe('Dictionary Schema Relations', () => {
- it('should return the schema children nodes on a Dictionary', () => {
- const result = getDictionarySchemaRelations(dictionarySportStats.dictionary);
+ describe('Determine child relations by schema in a dictionary', () => {
+ it('should return the schema children nodes on a Dictionary', () => {
+ const result = getDictionarySchemaRelations(dictionarySportStats.dictionary);
- expect(result).to.deep.equal(dictionarySportStatsNodeGraph);
+ expect(result).to.deep.equal(dictionarySportStatsNodeGraph);
+ });
+ it('should return an empty schema children for a schema with no children', () => {
+ const dictionarySchemas: SchemaDefinition[] = [
+ {
+ name: 'sports',
+ fields: [
+ { name: 'sport_id', valueType: 'integer' },
+ { name: 'name', valueType: 'string' },
+ ],
+ },
+ ];
+
+ const result = getDictionarySchemaRelations(dictionarySchemas);
+
+ expect(result).to.eql({ sports: [] });
+ });
});
- it('should return an empty schema children for a schema with no children', () => {
- const dictionarySchemas: SchemaDefinition[] = [
- {
- name: 'sports',
- fields: [
- { name: 'sport_id', valueType: 'integer' },
- { name: 'name', valueType: 'string' },
- ],
- },
- ];
-
- const result = getDictionarySchemaRelations(dictionarySchemas);
-
- expect(result).to.eql({ sports: [] });
+
+ describe('find the hierarchical descendant structure between schemas in the dictionary', () => {
+ it('should return only one unrelated element the tree', () => {
+ const schemas: SchemaDefinition[] = [
+ {
+ name: 'sample',
+ fields: [
+ {
+ name: 'id',
+ valueType: 'integer',
+ },
+ ],
+ restrictions: {},
+ },
+ ];
+
+ const response = generateHierarchy(schemas, 'desc');
+ expect(response.length).to.eql(1);
+ expect(response).to.eql([
+ {
+ schemaName: 'sample',
+ children: [],
+ },
+ ]);
+ });
+
+ it('should return 2 unrelated elements', () => {
+ const schemas: SchemaDefinition[] = [
+ {
+ name: 'food',
+ fields: [
+ {
+ name: 'id',
+ valueType: 'integer',
+ },
+ ],
+ restrictions: {},
+ },
+ {
+ name: 'sports',
+ fields: [
+ {
+ name: 'id',
+ valueType: 'integer',
+ },
+ ],
+ restrictions: {},
+ },
+ ];
+
+ const response = generateHierarchy(schemas, 'desc');
+ expect(response.length).to.eql(2);
+ expect(response).to.eql([
+ {
+ schemaName: 'food',
+ children: [],
+ },
+ {
+ schemaName: 'sports',
+ children: [],
+ },
+ ]);
+ });
+
+ it('should return the hierarchy tree between 4 schemas', () => {
+ const response = generateHierarchy(dictionaryClinicalSchemas, 'desc');
+ expect(response.length).to.eq(4);
+ expect(response).to.eql([
+ {
+ schemaName: 'sample',
+ children: [],
+ },
+ {
+ schemaName: 'study',
+ children: [
+ {
+ schemaName: 'participant',
+ children: [
+ {
+ schemaName: 'specimen',
+ children: [
+ {
+ schemaName: 'sample',
+ children: [],
+ fieldName: 'submitter_specimen_id',
+ parentFieldName: 'submitter_specimen_id',
+ },
+ ],
+ fieldName: 'submitter_participant_id',
+ parentFieldName: 'submitter_participant_id',
+ },
+ ],
+ fieldName: 'study_id',
+ parentFieldName: 'study_id',
+ },
+ ],
+ },
+ {
+ schemaName: 'participant',
+ children: [
+ {
+ schemaName: 'specimen',
+ children: [
+ {
+ schemaName: 'sample',
+ children: [],
+ fieldName: 'submitter_specimen_id',
+ parentFieldName: 'submitter_specimen_id',
+ },
+ ],
+ fieldName: 'submitter_participant_id',
+ parentFieldName: 'submitter_participant_id',
+ },
+ ],
+ },
+ {
+ schemaName: 'specimen',
+ children: [
+ {
+ schemaName: 'sample',
+ children: [],
+ fieldName: 'submitter_specimen_id',
+ parentFieldName: 'submitter_specimen_id',
+ },
+ ],
+ },
+ ]);
+ });
+ });
+
+ describe('find the hierarchical ascendant structure between schemas in the dictionary', () => {
+ it('should return only one unrelated element the tree', () => {
+ const schemas: SchemaDefinition[] = [
+ {
+ name: 'sample',
+ fields: [
+ {
+ name: 'id',
+ valueType: 'integer',
+ },
+ ],
+ restrictions: {},
+ },
+ ];
+
+ const response = generateHierarchy(schemas, 'asc');
+ expect(response.length).to.eql(1);
+ expect(response).to.eql([
+ {
+ schemaName: 'sample',
+ parent: undefined,
+ },
+ ]);
+ });
+
+ it('should return 2 unrelated elements', () => {
+ const schemas: SchemaDefinition[] = [
+ {
+ name: 'food',
+ fields: [
+ {
+ name: 'id',
+ valueType: 'integer',
+ },
+ ],
+ restrictions: {},
+ },
+ {
+ name: 'sports',
+ fields: [
+ {
+ name: 'id',
+ valueType: 'integer',
+ },
+ ],
+ restrictions: {},
+ },
+ ];
+
+ const response = generateHierarchy(schemas, 'asc');
+ expect(response.length).to.eql(2);
+ expect(response).to.eql([
+ {
+ schemaName: 'food',
+ parent: undefined,
+ },
+ {
+ schemaName: 'sports',
+ parent: undefined,
+ },
+ ]);
+ });
+
+ it('should return the hierarchy tree between 4 schemas', () => {
+ const response = generateHierarchy(dictionaryClinicalSchemas, 'asc');
+ expect(response.length).to.eq(4);
+ expect(response).to.eql([
+ {
+ schemaName: 'study',
+ parent: undefined,
+ },
+ {
+ schemaName: 'participant',
+ parent: {
+ schemaName: 'study',
+ parent: undefined,
+ fieldName: 'study_id',
+ parentFieldName: 'study_id',
+ },
+ },
+ {
+ schemaName: 'specimen',
+ parent: {
+ schemaName: 'participant',
+ parent: {
+ schemaName: 'study',
+ parent: undefined,
+ fieldName: 'study_id',
+ parentFieldName: 'study_id',
+ },
+ fieldName: 'submitter_participant_id',
+ parentFieldName: 'submitter_participant_id',
+ },
+ },
+ {
+ schemaName: 'sample',
+ parent: {
+ schemaName: 'specimen',
+ parent: {
+ schemaName: 'participant',
+ parent: {
+ schemaName: 'study',
+ parent: undefined,
+ fieldName: 'study_id',
+ parentFieldName: 'study_id',
+ },
+ fieldName: 'submitter_participant_id',
+ parentFieldName: 'submitter_participant_id',
+ },
+ fieldName: 'submitter_specimen_id',
+ parentFieldName: 'submitter_specimen_id',
+ },
+ },
+ ]);
+ });
});
});
diff --git a/packages/data-provider/test/utils/fixtures/dictionarySchemasTestData.ts b/packages/data-provider/test/utils/fixtures/dictionarySchemasTestData.ts
index 560d8a6..ffe4816 100644
--- a/packages/data-provider/test/utils/fixtures/dictionarySchemasTestData.ts
+++ b/packages/data-provider/test/utils/fixtures/dictionarySchemasTestData.ts
@@ -221,3 +221,88 @@ export const dictionarySportStatsNodeGraph = {
player: [],
game: [],
} as const;
+
+export const dictionaryClinicalSchemas: Schema[] = [
+ {
+ name: 'study',
+ fields: [],
+ },
+ {
+ name: 'participant',
+ fields: [],
+ restrictions: {
+ foreignKey: [
+ {
+ schema: 'study',
+ mappings: [
+ {
+ local: 'study_id',
+ foreign: 'study_id',
+ },
+ ],
+ },
+ ],
+ },
+ },
+ {
+ name: 'sample',
+ fields: [],
+ restrictions: {
+ foreignKey: [
+ {
+ schema: 'study',
+ mappings: [
+ {
+ local: 'study_id',
+ foreign: 'study_id',
+ },
+ ],
+ },
+ {
+ schema: 'participant',
+ mappings: [
+ {
+ local: 'submitter_participant_id',
+ foreign: 'submitter_participant_id',
+ },
+ ],
+ },
+ {
+ schema: 'specimen',
+ mappings: [
+ {
+ local: 'submitter_specimen_id',
+ foreign: 'submitter_specimen_id',
+ },
+ ],
+ },
+ ],
+ },
+ },
+ {
+ name: 'specimen',
+ fields: [],
+ restrictions: {
+ foreignKey: [
+ {
+ schema: 'study',
+ mappings: [
+ {
+ local: 'study_id',
+ foreign: 'study_id',
+ },
+ ],
+ },
+ {
+ schema: 'participant',
+ mappings: [
+ {
+ local: 'submitter_participant_id',
+ foreign: 'submitter_participant_id',
+ },
+ ],
+ },
+ ],
+ },
+ },
+];
diff --git a/packages/data-provider/test/utils/formatUtils.spec.ts b/packages/data-provider/test/utils/formatUtils.spec.ts
index 2396b54..1d5387e 100644
--- a/packages/data-provider/test/utils/formatUtils.spec.ts
+++ b/packages/data-provider/test/utils/formatUtils.spec.ts
@@ -2,6 +2,7 @@ import { expect } from 'chai';
import { describe, it } from 'mocha';
import {
+ asArray,
isArrayWithValues,
isEmptyString,
isValidDateFormat,
@@ -10,6 +11,47 @@ import {
} from '../../src/utils/formatUtils.js';
describe('Format Utils', () => {
+ describe('Wrap any value into array', () => {
+ it('should return an empty array when the value is undefined', () => {
+ const result = asArray(undefined);
+ expect(result).to.eql([]); // Expecting an empty array
+ });
+
+ it('should return an empty array when the value is null', () => {
+ const result = asArray(null);
+ expect(result).to.eql([]); // Expecting an empty array
+ });
+
+ it('should return the same array when the value is already an array', () => {
+ const input = ['a', 'b', 'c'];
+ const result = asArray(input);
+ expect(result).to.eql(input);
+ });
+
+ it('should return an array with a single value when a single value is passed', () => {
+ const input = 'singleValue';
+ const result = asArray(input);
+ expect(result).to.eql([input]);
+ });
+
+ it('should exclude undefined values when the value is an array with undefined', () => {
+ const input = ['a', undefined, 'b'];
+ const result = asArray(input);
+ expect(result).to.eql(['a', 'b']);
+ });
+
+ it('should exclude undefined values when a single value is undefined', () => {
+ const input = undefined;
+ const result = asArray(input);
+ expect(result).to.eql([]);
+ });
+
+ it('should filter out empty strings and undefined values ', () => {
+ const input = [0, false, '', undefined, 'valid'];
+ const result = asArray(input);
+ expect(result).to.eql([0, false, 'valid']);
+ });
+ });
describe('Validate if input is a valid ID number', () => {
it('should return true if input is a valid number', () => {
const validNumber = 1;
diff --git a/packages/data-provider/test/utils/submissionUtils.spec.ts b/packages/data-provider/test/utils/submissionUtils.spec.ts
index a1ca69d..bed39c2 100644
--- a/packages/data-provider/test/utils/submissionUtils.spec.ts
+++ b/packages/data-provider/test/utils/submissionUtils.spec.ts
@@ -1270,8 +1270,8 @@ describe('Submission Utils', () => {
const activeSubmissionSummaryRepository: ActiveSubmissionSummaryRepository = {
id: 2,
data: {},
- dictionary: {},
- dictionaryCategory: {},
+ dictionary: { name: 'books', version: '1' },
+ dictionaryCategory: { name: 'favorite books', id: 1 },
errors: {},
organization: 'oicr',
status: SUBMISSION_STATUS.OPEN,
@@ -1284,8 +1284,8 @@ describe('Submission Utils', () => {
expect(response).to.eql({
id: 2,
data: {},
- dictionary: {},
- dictionaryCategory: {},
+ dictionary: { name: 'books', version: '1' },
+ dictionaryCategory: { name: 'favorite books', id: 1 },
errors: {},
organization: 'oicr',
status: SUBMISSION_STATUS.OPEN,
@@ -1330,8 +1330,8 @@ describe('Submission Utils', () => {
],
},
},
- dictionary: {},
- dictionaryCategory: {},
+ dictionary: { name: 'books', version: '1.1' },
+ dictionaryCategory: { name: 'favorite books', id: 1 },
errors: {},
organization: 'oicr',
status: SUBMISSION_STATUS.OPEN,
@@ -1375,8 +1375,8 @@ describe('Submission Utils', () => {
],
},
},
- dictionary: {},
- dictionaryCategory: {},
+ dictionary: { name: 'books', version: '1.1' },
+ dictionaryCategory: { name: 'favorite books', id: 1 },
errors: {},
organization: 'oicr',
status: SUBMISSION_STATUS.OPEN,
@@ -1392,8 +1392,8 @@ describe('Submission Utils', () => {
const activeSubmissionSummaryRepository: ActiveSubmissionSummaryRepository = {
id: 4,
data: {},
- dictionary: {},
- dictionaryCategory: {},
+ dictionary: { name: 'books', version: '1' },
+ dictionaryCategory: { name: 'favorite books', id: 1 },
errors: {},
organization: 'oicr',
status: SUBMISSION_STATUS.VALID,
@@ -1410,8 +1410,8 @@ describe('Submission Utils', () => {
updates: undefined,
deletes: undefined,
},
- dictionary: {},
- dictionaryCategory: {},
+ dictionary: { name: 'books', version: '1' },
+ dictionaryCategory: { name: 'favorite books', id: 1 },
errors: {},
organization: 'oicr',
status: SUBMISSION_STATUS.VALID,
@@ -1456,8 +1456,8 @@ describe('Submission Utils', () => {
],
},
},
- dictionary: {},
- dictionaryCategory: {},
+ dictionary: { name: 'books', version: '1' },
+ dictionaryCategory: { name: 'favorite books', id: 1 },
errors: {},
organization: 'oicr',
status: SUBMISSION_STATUS.VALID,
@@ -1487,8 +1487,8 @@ describe('Submission Utils', () => {
},
},
},
- dictionary: {},
- dictionaryCategory: {},
+ dictionary: { name: 'books', version: '1' },
+ dictionaryCategory: { name: 'favorite books', id: 1 },
errors: {},
organization: 'oicr',
status: SUBMISSION_STATUS.VALID,
diff --git a/packages/data-provider/test/utils/submittedDataUtils.spec.ts b/packages/data-provider/test/utils/submittedDataUtils.spec.ts
index 49e10dd..5e9b1c0 100644
--- a/packages/data-provider/test/utils/submittedDataUtils.spec.ts
+++ b/packages/data-provider/test/utils/submittedDataUtils.spec.ts
@@ -7,11 +7,13 @@ import type { NewSubmittedData, SubmittedData } from '@overture-stack/lyric-data
import {
computeDataDiff,
fetchDataErrorResponse,
+ getEntityNamesFromFilterOptions,
groupErrorsByIndex,
groupSchemaDataByEntityName,
hasErrorsByIndex,
transformmSubmittedDataToSubmissionDeleteData,
} from '../../src/utils/submittedDataUtils.js';
+import { VIEW_TYPE } from '../../src/utils/types.js';
describe('Submitted Data Utils', () => {
const todaysDate = new Date();
@@ -106,15 +108,42 @@ describe('Submitted Data Utils', () => {
const response = fetchDataErrorResponse('Error fetching data');
expect(response.metadata.errorMessage).to.eql('Error fetching data');
expect(response.metadata.totalRecords).to.eq(0);
- expect(response.data).to.eql([]);
+ expect(response.result).to.eql([]);
});
it('should return a response with empty message', () => {
const response = fetchDataErrorResponse('');
expect(response.metadata.errorMessage).to.eql('');
expect(response.metadata.totalRecords).to.eq(0);
- expect(response.data).to.eql([]);
+ expect(response.result).to.eql([]);
});
});
+
+ describe('Determine the entity names based on the provided filter', () => {
+ it('should return an array with defaultCentricEntity if view is compound', () => {
+ const filterOptions = { view: VIEW_TYPE.Values.compound, entityName: ['entity1', 'entity2'] };
+ const result = getEntityNamesFromFilterOptions(filterOptions, 'defaultEntity');
+ expect(result).to.eql(['defaultEntity']);
+ });
+
+ it('should return entityName array if view is not compound and entityName is provided', () => {
+ const filterOptions = { view: VIEW_TYPE.Values.flat, entityName: ['entity1', 'entity2'] };
+ const result = getEntityNamesFromFilterOptions(filterOptions, undefined);
+ expect(result).to.eql(['entity1', 'entity2']);
+ });
+
+ it('should return an empty array if neither defaultCentricEntity nor entityName are provided', () => {
+ const filterOptions = { view: VIEW_TYPE.Values.flat, entityName: [] };
+ const result = getEntityNamesFromFilterOptions(filterOptions, undefined);
+ expect(result).to.eql([]);
+ });
+
+ it('should return an empty array if entityName is undefined and view is not compound', () => {
+ const filterOptions = { view: VIEW_TYPE.Values.flat };
+ const result = getEntityNamesFromFilterOptions(filterOptions, undefined);
+ expect(result).to.eql([]);
+ });
+ });
+
describe('Group validation errors by index', () => {
it('should return the errors by index', () => {
const listOfErrors: SchemaRecordError[] = [
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 5210c53..8e8e7c4 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -181,6 +181,9 @@ importers:
pg:
specifier: ^8.12.0
version: 8.12.0
+ plur:
+ specifier: ^5.1.0
+ version: 5.1.0
winston:
specifier: ^3.13.1
version: 3.13.1
@@ -2831,6 +2834,11 @@ packages:
engines: {node: '>= 0.10'}
dev: false
+ /irregular-plurals@3.5.0:
+ resolution: {integrity: sha512-1ANGLZ+Nkv1ptFb2pa8oG8Lem4krflKuX/gINiHJHjJUKaJHk/SXk5x6K3J+39/p0h1RQ2saROclJJ+QLvETCQ==}
+ engines: {node: '>=8'}
+ dev: false
+
/is-arrayish@0.3.2:
resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}
dev: false
@@ -3491,6 +3499,13 @@ packages:
engines: {node: '>=8.6'}
dev: true
+ /plur@5.1.0:
+ resolution: {integrity: sha512-VP/72JeXqak2KiOzjgKtQen5y3IZHn+9GOuLDafPv0eXa47xq0At93XahYBs26MsifCQ4enGKwbjBTKgb9QJXg==}
+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+ dependencies:
+ irregular-plurals: 3.5.0
+ dev: false
+
/postgres-array@2.0.0:
resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==}
engines: {node: '>=4'}