Add WAS finding model and sync scan

backend/src/api/scans.ts

@@ -238,6 +238,15 @@ export const SCAN_SCHEMA: ScanSchema = {
     description:
       'Open source tool that fingerprints web technologies based on HTTP responses'
   },
+  wasSync: {
+    type: 'fargate',
+    isPassive: true,
+    global: true,
+    description:
+      'Pull in WAS vulnerability data from PEs Vulnerability database',
+    cpu: '1024',
+    memory: '8192'
+  },
   webscraper: {
     type: 'fargate',
     isPassive: true,

backend/src/models/connection.ts

@@ -46,7 +46,8 @@ import {
   TicketEvent,
   Ticket,
   TrustymailScan,
-  VulnScan
+  VulnScan,
+  WasFinding
 } from '.';
 
 let connection: Connection | null = null;
@@ -87,7 +88,8 @@ const connectDl2 = async (logging?: boolean) => {
       TicketEvent,
       Ticket,
       TrustymailScan,
-      VulnScan
+      VulnScan,
+      WasFinding
     ],
     synchronize: false,
     name: 'default2',
@@ -146,7 +148,8 @@ const connectDl = async (logging?: boolean) => {
       TicketEvent,
       Ticket,
       TrustymailScan,
-      VulnScan
+      VulnScan,
+      WasFinding
     ],
     synchronize: false,
     name: 'default',
@@ -187,7 +190,6 @@ const connectDb = async (logging?: boolean) => {
       Response,
       Role,
       SavedSearch,
-      OrganizationTag,
       Notification,
       Scan,
       ScanTask,

backend/src/models/index.ts

@@ -44,3 +44,4 @@ export * from './mini_data_lake/ticket_events';
 export * from './mini_data_lake/tickets';
 export * from './mini_data_lake/trustymail_scans';
 export * from './mini_data_lake/vuln_scans';
+export * from './mini_data_lake/was_findings';

backend/src/models/mini_data_lake/organizations.ts

@@ -30,6 +30,7 @@ import { HostScan } from './host_scans';
 import { Host } from './hosts';
 import { Ticket } from './tickets';
 import { PortScan } from './port_scans';
+import { WasFinding } from './was_findings';
 @Entity()
 export class Organization extends BaseEntity {
   @PrimaryGeneratedColumn('uuid')
@@ -223,6 +224,12 @@ export class Organization extends BaseEntity {
   })
   vulnScans: VulnScan[];
 
+  @OneToMany((type) => WasFinding, (was_finding) => was_finding.organization, {
+    onDelete: 'CASCADE',
+    onUpdate: 'CASCADE'
+  })
+  wasFindings: WasFinding[];
+
   @OneToMany((type) => HostScan, (host_scan) => host_scan.organization, {
     onDelete: 'CASCADE',
     onUpdate: 'CASCADE'

backend/src/models/mini_data_lake/was_findings.ts

@@ -0,0 +1,141 @@
+// The data in this table is derived from the Vulnerability Scans Database,
+// the [vuln_scans Collection] (https://github.com/cisagov/ncats-data-dictionary/blob/develop/NCATS_Data_Dictionary.md#vuln_scans-collection).
+
+import {
+  Entity,
+  Column,
+  PrimaryColumn,
+  BaseEntity,
+  ManyToMany,
+  ManyToOne,
+  JoinTable,
+  OneToMany
+} from 'typeorm';
+
+import { Organization } from './organizations';
+
+@Entity()
+export class WasFinding extends BaseEntity {
+  @PrimaryColumn()
+  id: string;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  webappId: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  wasOrgId: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  owaspCategory: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'integer'
+  })
+  severity: number | null;
+
+  @Column({
+    nullable: true,
+    type: 'integer'
+  })
+  timesDetected: number | null;
+
+  @Column({
+    nullable: true,
+    type: 'decimal'
+  })
+  baseScore: number | null;
+
+  @Column({
+    nullable: true,
+    type: 'decimal'
+  })
+  temporalScore: number | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  fstatus: string | null;
+
+  @Column({ nullable: true, type: 'timestamp' })
+  lastDetected: Date | null;
+
+  @Column({ nullable: true, type: 'timestamp' })
+  firstDetected: Date | null;
+
+  @Column({ nullable: true })
+  isRemediated: boolean;
+
+  @Column({ nullable: true })
+  potential: boolean;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  webappUrl: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  webappName: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  name: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  cvssV3AttackVector: string | null;
+
+  @Column('int', { array: true, default: [] })
+  cweList: number[];
+
+  @Column({
+    type: 'jsonb',
+    default: []
+  })
+  wascList: object;
+
+  @Column({ nullable: true, type: 'timestamp' })
+  lastTested: Date | null;
+
+  @Column({ nullable: true, type: 'timestamp' })
+  fixedDate: Date | null;
+
+  @Column({ nullable: true })
+  isIgnored: boolean;
+
+  @Column({
+    nullable: true,
+    type: 'varchar'
+  })
+  url: string | null;
+
+  @Column({
+    nullable: true,
+    type: 'integer'
+  })
+  qid: number | null;
+
+  @ManyToOne((type) => Organization, (org) => org.wasFindings, {
+    onDelete: 'CASCADE',
+    nullable: true
+  })
+  organization: Organization;
+}

backend/src/tasks/helpers/saveWasFindingToDb.ts

@@ -0,0 +1,33 @@
+import { plainToClass } from 'class-transformer';
+import { WasFinding, connectToDatalake2 } from '../../models';
+
+export default async (wasFindingObj: WasFinding): Promise<string | null> => {
+  console.log(
+    `Starting to save WAS finding to datalake: ${wasFindingObj.name}`
+  );
+
+  const datalakeConnnection = await connectToDatalake2();
+  const dl_was_finding = datalakeConnnection.getRepository(WasFinding);
+  const wasFindingValues = Object.keys(wasFindingObj)
+    .map((key) => {
+      if (['id'].indexOf(key) > -1) return '';
+      else if (key === 'organization') return 'organizationId';
+      return wasFindingObj[key] != null ? key : '';
+    })
+    .filter((key) => key !== '');
+  console.log(wasFindingObj);
+  const was_finding_id: string = (
+    await dl_was_finding
+      .createQueryBuilder()
+      .insert()
+      .values(wasFindingObj)
+      .orUpdate({
+        conflict_target: ['id'],
+        overwrite: wasFindingValues
+      })
+      .returning('id')
+      .execute()
+  ).identifiers[0].id;
+
+  return was_finding_id;
+};