Enable Dynamic Schema Support for File Type Definitions in Analysis Types

pom.xml

@@ -57,9 +57,12 @@
             <url>https://jcenter.bintray.com/</url>
         </repository>
         <repository>
-            <id>spring-releases</id>
+            <id>lib-releases</id>
             <name>Spring Releases</name>
             <url>https://repo.spring.io/libs-release</url>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
         </repository>
     </repositories>
 

song-core/src/main/java/bio/overture/song/core/model/AnalysisType.java

@@ -18,6 +18,7 @@ public class AnalysisType {
   @NotNull private String name;
   @NotNull private Integer version;
   private LocalDateTime createdAt;
+  private AnalysisTypeOptions options;
 
   @JsonInclude(JsonInclude.Include.NON_NULL)
   private JsonNode schema;

song-core/src/main/java/bio/overture/song/core/model/AnalysisTypeOptions.java

@@ -0,0 +1,12 @@
+package bio.overture.song.core.model;
+
+import java.util.List;
+import lombok.*;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class AnalysisTypeOptions {
+  private List<String> fileTypes;
+}

song-server/pom.xml

@@ -25,13 +25,6 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>song-server</artifactId>
-    <repositories>
-        <repository>
-            <id>spring-releases</id>
-            <name>Spring Releases</name>
-            <url>https://repo.spring.io/libs-release</url>
-        </repository>
-    </repositories>
 
     <dependencies>
 

song-server/src/main/java/bio/overture/song/server/controller/analysisType/AnalysisTypeController.java

@@ -108,7 +108,8 @@ public String getAnalysisTypeRegistrationSchema() {
   public @ResponseBody AnalysisType register(
       @RequestHeader(value = AUTHORIZATION, required = false) final String accessToken,
       @RequestBody RegisterAnalysisTypeRequest request) {
-    return analysisTypeService.register(request.getName(), request.getSchema());
+    return analysisTypeService.register(
+        request.getName(), request.getOptions(), request.getSchema());
   }
 
   @ApiImplicitParams({

song-server/src/main/java/bio/overture/song/server/model/dto/schema/RegisterAnalysisTypeRequest.java

@@ -2,6 +2,7 @@
 
 import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT;
 
+import bio.overture.song.core.model.AnalysisTypeOptions;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.JsonNode;
 import lombok.AllArgsConstructor;
@@ -17,4 +18,5 @@
 public class RegisterAnalysisTypeRequest {
   private String name;
   private JsonNode schema;
+  private AnalysisTypeOptions options;
 }

song-server/src/main/java/bio/overture/song/server/model/entity/AnalysisSchema.java

@@ -1,29 +1,20 @@
 package bio.overture.song.server.model.entity;
 
-import static bio.overture.song.server.model.enums.TableAttributeNames.ID;
-import static bio.overture.song.server.model.enums.TableAttributeNames.NAME;
-import static bio.overture.song.server.model.enums.TableAttributeNames.SCHEMA;
-import static bio.overture.song.server.model.enums.TableAttributeNames.VERSION;
+import static bio.overture.song.server.model.enums.TableAttributeNames.*;
 import static bio.overture.song.server.repository.CustomJsonType.CUSTOM_JSON_TYPE_PKG_PATH;
 import static com.google.common.collect.Sets.newHashSet;
 
 import bio.overture.song.server.model.analysis.Analysis;
 import bio.overture.song.server.model.enums.ModelAttributeNames;
 import bio.overture.song.server.model.enums.TableAttributeNames;
 import bio.overture.song.server.model.enums.TableNames;
+import bio.overture.song.server.utils.StringListConverter;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.databind.JsonNode;
 import java.time.LocalDateTime;
+import java.util.List;
 import java.util.Set;
-import javax.persistence.CascadeType;
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.FetchType;
-import javax.persistence.GeneratedValue;
-import javax.persistence.GenerationType;
-import javax.persistence.Id;
-import javax.persistence.OneToMany;
-import javax.persistence.Table;
+import javax.persistence.*;
 import javax.validation.constraints.NotNull;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -63,6 +54,10 @@ public class AnalysisSchema {
   @Type(type = CUSTOM_JSON_TYPE_PKG_PATH)
   private JsonNode schema;
 
+  @Column(name = FILE_TYPES, columnDefinition = "text[]")
+  @Convert(converter = StringListConverter.class)
+  private List<String> fileTypes;
+
   @JsonIgnore
   @Builder.Default
   @ToString.Exclude

song-server/src/main/java/bio/overture/song/server/model/enums/TableAttributeNames.java

@@ -76,6 +76,7 @@ public class TableAttributeNames {
   public static final String DATA_TYPE = "data_type";
   public static final String TUMOUR_NORMAL_DESIGNATION = "tumour_normal_designation";
   public static final String TISSUE_SOURCE = "tissue_source";
+  public static final String FILE_TYPES = "file_types";
 
   public static final String INITIAL_STATE = "initial_state";
   public static final String UPDATED_STATE = "updated_state";

song-server/src/main/java/bio/overture/song/server/repository/AnalysisSchemaRepository.java

@@ -18,6 +18,7 @@
 package bio.overture.song.server.repository;
 
 import bio.overture.song.server.model.entity.AnalysisSchema;
+import java.util.List;
 import java.util.Optional;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
@@ -30,4 +31,6 @@ public interface AnalysisSchemaRepository
   Integer countAllByNameAndIdLessThanEqual(String name, Integer id);
 
   Optional<AnalysisSchema> findByNameAndVersion(String name, Integer version);
+
+  List<AnalysisSchema> findByName(String name);
 }

song-server/src/main/java/bio/overture/song/server/repository/search/IdSearchRequest.java

@@ -47,7 +47,13 @@ public class IdSearchRequest {
   private final String submitterSpecimenIds;
 
   public static IdSearchRequest createIdSearchRequest(
-      String donorId, String sampleId, String specimenId, String objectId, String submitterSampleId, String submitterDonorIds, String submitterSpecimenIds) {
+      String donorId,
+      String sampleId,
+      String specimenId,
+      String objectId,
+      String submitterSampleId,
+      String submitterDonorIds,
+      String submitterSpecimenIds) {
     return new IdSearchRequest(
         getGlobPattern(donorId),
         getGlobPattern(sampleId),

song-server/src/main/java/bio/overture/song/server/service/AnalysisTypeService.java

@@ -42,6 +42,7 @@
 import bio.overture.song.core.exceptions.ServerErrors;
 import bio.overture.song.core.model.AnalysisType;
 import bio.overture.song.core.model.AnalysisTypeId;
+import bio.overture.song.core.model.AnalysisTypeOptions;
 import bio.overture.song.core.model.PageDTO;
 import bio.overture.song.server.controller.analysisType.AnalysisTypeController;
 import bio.overture.song.server.model.entity.AnalysisSchema;
@@ -50,14 +51,15 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.io.IOException;
-import java.util.Collection;
+import java.util.*;
 import java.util.function.Supplier;
 import java.util.regex.Pattern;
 import javax.transaction.Transactional;
 import lombok.NonNull;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
+import org.apache.commons.collections.CollectionUtils;
 import org.everit.json.schema.Schema;
 import org.everit.json.schema.SchemaException;
 import org.everit.json.schema.ValidationException;
@@ -149,21 +151,29 @@ public AnalysisSchema getAnalysisSchema(String name, Integer version) {
   public AnalysisType getAnalysisType(
       @NonNull AnalysisTypeId analysisTypeId, boolean unrenderedOnly) {
     val analysisSchema = getAnalysisSchema(analysisTypeId);
+
     val resolvedSchemaJson =
         resolveSchemaJsonView(analysisSchema.getSchema(), unrenderedOnly, false);
+
+    List<String> fileTypes =
+        (analysisSchema.getFileTypes() != null && !analysisSchema.getFileTypes().isEmpty())
+            ? analysisSchema.getFileTypes()
+            : new ArrayList<>();
     return AnalysisType.builder()
         .name(analysisTypeId.getName())
         .version(analysisTypeId.getVersion())
         .createdAt(analysisSchema.getCreatedAt())
         .schema(resolvedSchemaJson)
+        .options(AnalysisTypeOptions.builder().fileTypes(fileTypes).build())
         .build();
   }
 
   @Transactional
-  public AnalysisType register(@NonNull String analysisTypeName, JsonNode analysisTypeSchema) {
+  public AnalysisType register(
+      @NonNull String analysisTypeName, AnalysisTypeOptions options, JsonNode analysisTypeSchema) {
     validateAnalysisTypeName(analysisTypeName);
     validateAnalysisTypeSchema(analysisTypeSchema);
-    return commitAnalysisType(analysisTypeName, analysisTypeSchema);
+    return commitAnalysisType(analysisTypeName, analysisTypeSchema, options);
   }
 
   public PageDTO<AnalysisType> listAnalysisTypes(
@@ -246,9 +256,23 @@ private void validateAnalysisTypeSchema(JsonNode analysisTypeSchema) {
 
   @SneakyThrows
   private AnalysisType commitAnalysisType(
-      @NonNull String analysisTypeName, @NonNull JsonNode analysisTypeSchema) {
+      @NonNull String analysisTypeName,
+      @NonNull JsonNode analysisTypeSchema,
+      AnalysisTypeOptions options) {
+
+    List<String> fileTypes = new ArrayList<>();
+
+    if (options != null && CollectionUtils.isNotEmpty(options.getFileTypes())) {
+      fileTypes = options.getFileTypes();
+    }
     val analysisSchema =
-        AnalysisSchema.builder().name(analysisTypeName).schema(analysisTypeSchema).build();
+        AnalysisSchema.builder()
+            .name(analysisTypeName)
+            .schema(analysisTypeSchema)
+            .fileTypes(fileTypes)
+            .build();
+
+    log.debug("Creating analysisSchema with file types: {}  " + fileTypes);
     analysisSchemaRepository.save(analysisSchema);
     val version =
         analysisSchemaRepository.countAllByNameAndIdLessThanEqual(
@@ -275,6 +299,7 @@ private AnalysisType commitAnalysisType(
         .version(version)
         .createdAt(createdAt)
         .schema(resolvedSchemaJson)
+        .options(options)
         .build();
   }
 

song-server/src/main/java/bio/overture/song/server/service/ValidationService.java

@@ -35,11 +35,15 @@
 import bio.overture.song.server.validation.SchemaValidator;
 import bio.overture.song.server.validation.ValidationResponse;
 import com.fasterxml.jackson.databind.JsonNode;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
 import java.util.function.Supplier;
 import lombok.NonNull;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
+import org.apache.commons.collections.CollectionUtils;
 import org.everit.json.schema.Schema;
 import org.everit.json.schema.ValidationException;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -73,6 +77,7 @@ public ValidationService(
     this.analysisTypeIdSchema = analysisTypeIdSchemaSupplier.get();
   }
 
+  @SneakyThrows
   public Optional<String> validate(@NonNull JsonNode payload) {
     String errors = null;
     try {
@@ -85,6 +90,14 @@ public Optional<String> validate(@NonNull JsonNode payload) {
               "Found Analysis type: name=%s  version=%s",
               analysisType.getName(), analysisType.getVersion()));
 
+      List<String> fileTypes = new ArrayList<>();
+
+      if (analysisType.getOptions() != null && analysisType.getOptions().getFileTypes() != null) {
+        fileTypes = analysisType.getOptions().getFileTypes();
+      }
+
+      validateFileType(fileTypes, payload);
+
       val schema = buildSchema(analysisType.getSchema());
       validateWithSchema(schema, payload);
     } catch (ValidationException e) {
@@ -94,6 +107,26 @@ public Optional<String> validate(@NonNull JsonNode payload) {
     return Optional.ofNullable(errors);
   }
 
+  private void validateFileType(List<String> fileTypes, @NonNull JsonNode payload) {
+
+    if (CollectionUtils.isNotEmpty(fileTypes)) {
+      JsonNode files = payload.get("files");
+      if (files.isArray()) {
+        for (JsonNode file : files) {
+          log.info("file is " + file);
+          String fileType = file.get("fileType").asText();
+          String fileName = file.get("fileName").asText();
+          if (!fileTypes.contains(fileType)) {
+            throw new ValidationException(
+                String.format(
+                    "%s name is not supported, supported formats are %s",
+                    fileName, String.join(", ", fileTypes)));
+          }
+        }
+      }
+    }
+  }
+
   public void update(@NonNull String uploadId, String errorMessages) {
     if (isNull(errorMessages)) {
       updateAsValid(uploadId);

song-server/src/main/java/bio/overture/song/server/utils/StringListConverter.java

@@ -0,0 +1,32 @@
+package bio.overture.song.server.utils;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+@Converter
+public class StringListConverter implements AttributeConverter<List<String>, String> {
+
+  @Override
+  public String convertToDatabaseColumn(List<String> attribute) {
+    if (attribute == null || attribute.isEmpty()) {
+      return "{}";
+    }
+    return "{"
+        + String.join(
+            ",", attribute.stream().map(s -> "\"" + s + "\"").collect(Collectors.toList()))
+        + "}";
+  }
+
+  @Override
+  public List<String> convertToEntityAttribute(String dbData) {
+    if (dbData == null || dbData.equals("{}")) {
+      return Arrays.asList();
+    }
+    return Arrays.stream(dbData.substring(1, dbData.length() - 1).split(","))
+        .map(s -> s.replace("\"", ""))
+        .collect(Collectors.toList());
+  }
+}

song-server/src/main/resources/db/migration/V1_20__update_file_from_enum.sql

@@ -0,0 +1,5 @@
+ALTER TABLE public.file
+ALTER COLUMN type TYPE text USING type::text;
+
+ALTER TABLE public.analysis_schema
+ADD COLUMN file_types text[];

song-server/src/main/resources/schemas/analysis/analysisBase.json

@@ -18,22 +18,7 @@
     },
     "file": {
       "fileType": {
-        "type": "string",
-        "enum": [
-          "FASTA",
-          "FAI",
-          "FASTQ",
-          "BAM",
-          "BAI",
-          "VCF",
-          "TBI",
-          "IDX",
-          "XML",
-          "TGZ",
-          "CRAM",
-          "CRAI",
-          "TXT"
-        ]
+        "type": "string"
       },
       "fileData": {
         "type": "object",
@@ -267,4 +252,4 @@
       "items": { "$ref": "#/definitions/file/fileData" }
     }
   }
-}
+}