diff --git a/pom.xml b/pom.xml index d983cdadf..98905d8e0 100644 --- a/pom.xml +++ b/pom.xml @@ -57,9 +57,12 @@ https://jcenter.bintray.com/ - spring-releases + lib-releases Spring Releases https://repo.spring.io/libs-release + + false + diff --git a/song-core/src/main/java/bio/overture/song/core/model/AnalysisType.java b/song-core/src/main/java/bio/overture/song/core/model/AnalysisType.java index f1c1ae9e7..f7464eb3d 100644 --- a/song-core/src/main/java/bio/overture/song/core/model/AnalysisType.java +++ b/song-core/src/main/java/bio/overture/song/core/model/AnalysisType.java @@ -18,6 +18,7 @@ public class AnalysisType { @NotNull private String name; @NotNull private Integer version; private LocalDateTime createdAt; + private AnalysisTypeOptions options; @JsonInclude(JsonInclude.Include.NON_NULL) private JsonNode schema; diff --git a/song-core/src/main/java/bio/overture/song/core/model/AnalysisTypeOptions.java b/song-core/src/main/java/bio/overture/song/core/model/AnalysisTypeOptions.java new file mode 100644 index 000000000..4aca09844 --- /dev/null +++ b/song-core/src/main/java/bio/overture/song/core/model/AnalysisTypeOptions.java @@ -0,0 +1,12 @@ +package bio.overture.song.core.model; + +import java.util.List; +import lombok.*; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class AnalysisTypeOptions { + private List fileTypes; +} diff --git a/song-server/pom.xml b/song-server/pom.xml index 1ee2eb7f4..6a8260d38 100644 --- a/song-server/pom.xml +++ b/song-server/pom.xml @@ -25,13 +25,6 @@ 4.0.0 song-server - - - spring-releases - Spring Releases - https://repo.spring.io/libs-release - - diff --git a/song-server/src/main/java/bio/overture/song/server/controller/analysisType/AnalysisTypeController.java b/song-server/src/main/java/bio/overture/song/server/controller/analysisType/AnalysisTypeController.java index a340d8d54..69023a9fd 100644 --- a/song-server/src/main/java/bio/overture/song/server/controller/analysisType/AnalysisTypeController.java +++ b/song-server/src/main/java/bio/overture/song/server/controller/analysisType/AnalysisTypeController.java @@ -108,7 +108,8 @@ public String getAnalysisTypeRegistrationSchema() { public @ResponseBody AnalysisType register( @RequestHeader(value = AUTHORIZATION, required = false) final String accessToken, @RequestBody RegisterAnalysisTypeRequest request) { - return analysisTypeService.register(request.getName(), request.getSchema()); + return analysisTypeService.register( + request.getName(), request.getOptions(), request.getSchema()); } @ApiImplicitParams({ diff --git a/song-server/src/main/java/bio/overture/song/server/model/dto/schema/RegisterAnalysisTypeRequest.java b/song-server/src/main/java/bio/overture/song/server/model/dto/schema/RegisterAnalysisTypeRequest.java index e8a805cda..a2abf0009 100644 --- a/song-server/src/main/java/bio/overture/song/server/model/dto/schema/RegisterAnalysisTypeRequest.java +++ b/song-server/src/main/java/bio/overture/song/server/model/dto/schema/RegisterAnalysisTypeRequest.java @@ -2,6 +2,7 @@ import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT; +import bio.overture.song.core.model.AnalysisTypeOptions; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.JsonNode; import lombok.AllArgsConstructor; @@ -17,4 +18,5 @@ public class RegisterAnalysisTypeRequest { private String name; private JsonNode schema; + private AnalysisTypeOptions options; } diff --git a/song-server/src/main/java/bio/overture/song/server/model/entity/AnalysisSchema.java b/song-server/src/main/java/bio/overture/song/server/model/entity/AnalysisSchema.java index 356e843b9..f5595353d 100644 --- a/song-server/src/main/java/bio/overture/song/server/model/entity/AnalysisSchema.java +++ b/song-server/src/main/java/bio/overture/song/server/model/entity/AnalysisSchema.java @@ -1,9 +1,6 @@ package bio.overture.song.server.model.entity; -import static bio.overture.song.server.model.enums.TableAttributeNames.ID; -import static bio.overture.song.server.model.enums.TableAttributeNames.NAME; -import static bio.overture.song.server.model.enums.TableAttributeNames.SCHEMA; -import static bio.overture.song.server.model.enums.TableAttributeNames.VERSION; +import static bio.overture.song.server.model.enums.TableAttributeNames.*; import static bio.overture.song.server.repository.CustomJsonType.CUSTOM_JSON_TYPE_PKG_PATH; import static com.google.common.collect.Sets.newHashSet; @@ -11,19 +8,13 @@ import bio.overture.song.server.model.enums.ModelAttributeNames; import bio.overture.song.server.model.enums.TableAttributeNames; import bio.overture.song.server.model.enums.TableNames; +import bio.overture.song.server.utils.StringListConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import java.time.LocalDateTime; +import java.util.List; import java.util.Set; -import javax.persistence.CascadeType; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.OneToMany; -import javax.persistence.Table; +import javax.persistence.*; import javax.validation.constraints.NotNull; import lombok.AllArgsConstructor; import lombok.Builder; @@ -63,6 +54,10 @@ public class AnalysisSchema { @Type(type = CUSTOM_JSON_TYPE_PKG_PATH) private JsonNode schema; + @Column(name = FILE_TYPES, columnDefinition = "text[]") + @Convert(converter = StringListConverter.class) + private List fileTypes; + @JsonIgnore @Builder.Default @ToString.Exclude diff --git a/song-server/src/main/java/bio/overture/song/server/model/enums/TableAttributeNames.java b/song-server/src/main/java/bio/overture/song/server/model/enums/TableAttributeNames.java index f1a7b82c8..29e847ef1 100644 --- a/song-server/src/main/java/bio/overture/song/server/model/enums/TableAttributeNames.java +++ b/song-server/src/main/java/bio/overture/song/server/model/enums/TableAttributeNames.java @@ -76,6 +76,7 @@ public class TableAttributeNames { public static final String DATA_TYPE = "data_type"; public static final String TUMOUR_NORMAL_DESIGNATION = "tumour_normal_designation"; public static final String TISSUE_SOURCE = "tissue_source"; + public static final String FILE_TYPES = "file_types"; public static final String INITIAL_STATE = "initial_state"; public static final String UPDATED_STATE = "updated_state"; diff --git a/song-server/src/main/java/bio/overture/song/server/repository/AnalysisSchemaRepository.java b/song-server/src/main/java/bio/overture/song/server/repository/AnalysisSchemaRepository.java index 6a7d47d53..7712c815f 100644 --- a/song-server/src/main/java/bio/overture/song/server/repository/AnalysisSchemaRepository.java +++ b/song-server/src/main/java/bio/overture/song/server/repository/AnalysisSchemaRepository.java @@ -18,6 +18,7 @@ package bio.overture.song.server.repository; import bio.overture.song.server.model.entity.AnalysisSchema; +import java.util.List; import java.util.Optional; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; @@ -30,4 +31,6 @@ public interface AnalysisSchemaRepository Integer countAllByNameAndIdLessThanEqual(String name, Integer id); Optional findByNameAndVersion(String name, Integer version); + + List findByName(String name); } diff --git a/song-server/src/main/java/bio/overture/song/server/repository/search/IdSearchRequest.java b/song-server/src/main/java/bio/overture/song/server/repository/search/IdSearchRequest.java index 870be41bf..fc59595ea 100644 --- a/song-server/src/main/java/bio/overture/song/server/repository/search/IdSearchRequest.java +++ b/song-server/src/main/java/bio/overture/song/server/repository/search/IdSearchRequest.java @@ -47,7 +47,13 @@ public class IdSearchRequest { private final String submitterSpecimenIds; public static IdSearchRequest createIdSearchRequest( - String donorId, String sampleId, String specimenId, String objectId, String submitterSampleId, String submitterDonorIds, String submitterSpecimenIds) { + String donorId, + String sampleId, + String specimenId, + String objectId, + String submitterSampleId, + String submitterDonorIds, + String submitterSpecimenIds) { return new IdSearchRequest( getGlobPattern(donorId), getGlobPattern(sampleId), diff --git a/song-server/src/main/java/bio/overture/song/server/service/AnalysisTypeService.java b/song-server/src/main/java/bio/overture/song/server/service/AnalysisTypeService.java index 5a84c8f85..0b52bba59 100644 --- a/song-server/src/main/java/bio/overture/song/server/service/AnalysisTypeService.java +++ b/song-server/src/main/java/bio/overture/song/server/service/AnalysisTypeService.java @@ -42,6 +42,7 @@ import bio.overture.song.core.exceptions.ServerErrors; import bio.overture.song.core.model.AnalysisType; import bio.overture.song.core.model.AnalysisTypeId; +import bio.overture.song.core.model.AnalysisTypeOptions; import bio.overture.song.core.model.PageDTO; import bio.overture.song.server.controller.analysisType.AnalysisTypeController; import bio.overture.song.server.model.entity.AnalysisSchema; @@ -50,7 +51,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; -import java.util.Collection; +import java.util.*; import java.util.function.Supplier; import java.util.regex.Pattern; import javax.transaction.Transactional; @@ -58,6 +59,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.apache.commons.collections.CollectionUtils; import org.everit.json.schema.Schema; import org.everit.json.schema.SchemaException; import org.everit.json.schema.ValidationException; @@ -149,21 +151,29 @@ public AnalysisSchema getAnalysisSchema(String name, Integer version) { public AnalysisType getAnalysisType( @NonNull AnalysisTypeId analysisTypeId, boolean unrenderedOnly) { val analysisSchema = getAnalysisSchema(analysisTypeId); + val resolvedSchemaJson = resolveSchemaJsonView(analysisSchema.getSchema(), unrenderedOnly, false); + + List fileTypes = + (analysisSchema.getFileTypes() != null && !analysisSchema.getFileTypes().isEmpty()) + ? analysisSchema.getFileTypes() + : new ArrayList<>(); return AnalysisType.builder() .name(analysisTypeId.getName()) .version(analysisTypeId.getVersion()) .createdAt(analysisSchema.getCreatedAt()) .schema(resolvedSchemaJson) + .options(AnalysisTypeOptions.builder().fileTypes(fileTypes).build()) .build(); } @Transactional - public AnalysisType register(@NonNull String analysisTypeName, JsonNode analysisTypeSchema) { + public AnalysisType register( + @NonNull String analysisTypeName, AnalysisTypeOptions options, JsonNode analysisTypeSchema) { validateAnalysisTypeName(analysisTypeName); validateAnalysisTypeSchema(analysisTypeSchema); - return commitAnalysisType(analysisTypeName, analysisTypeSchema); + return commitAnalysisType(analysisTypeName, analysisTypeSchema, options); } public PageDTO listAnalysisTypes( @@ -246,9 +256,23 @@ private void validateAnalysisTypeSchema(JsonNode analysisTypeSchema) { @SneakyThrows private AnalysisType commitAnalysisType( - @NonNull String analysisTypeName, @NonNull JsonNode analysisTypeSchema) { + @NonNull String analysisTypeName, + @NonNull JsonNode analysisTypeSchema, + AnalysisTypeOptions options) { + + List fileTypes = new ArrayList<>(); + + if (options != null && CollectionUtils.isNotEmpty(options.getFileTypes())) { + fileTypes = options.getFileTypes(); + } val analysisSchema = - AnalysisSchema.builder().name(analysisTypeName).schema(analysisTypeSchema).build(); + AnalysisSchema.builder() + .name(analysisTypeName) + .schema(analysisTypeSchema) + .fileTypes(fileTypes) + .build(); + + log.debug("Creating analysisSchema with file types: {} " + fileTypes); analysisSchemaRepository.save(analysisSchema); val version = analysisSchemaRepository.countAllByNameAndIdLessThanEqual( @@ -275,6 +299,7 @@ private AnalysisType commitAnalysisType( .version(version) .createdAt(createdAt) .schema(resolvedSchemaJson) + .options(options) .build(); } diff --git a/song-server/src/main/java/bio/overture/song/server/service/ValidationService.java b/song-server/src/main/java/bio/overture/song/server/service/ValidationService.java index 52b3cc6d7..f688bf99f 100644 --- a/song-server/src/main/java/bio/overture/song/server/service/ValidationService.java +++ b/song-server/src/main/java/bio/overture/song/server/service/ValidationService.java @@ -35,11 +35,15 @@ import bio.overture.song.server.validation.SchemaValidator; import bio.overture.song.server.validation.ValidationResponse; import com.fasterxml.jackson.databind.JsonNode; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.function.Supplier; import lombok.NonNull; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.apache.commons.collections.CollectionUtils; import org.everit.json.schema.Schema; import org.everit.json.schema.ValidationException; import org.springframework.beans.factory.annotation.Autowired; @@ -73,6 +77,7 @@ public ValidationService( this.analysisTypeIdSchema = analysisTypeIdSchemaSupplier.get(); } + @SneakyThrows public Optional validate(@NonNull JsonNode payload) { String errors = null; try { @@ -85,6 +90,14 @@ public Optional validate(@NonNull JsonNode payload) { "Found Analysis type: name=%s version=%s", analysisType.getName(), analysisType.getVersion())); + List fileTypes = new ArrayList<>(); + + if (analysisType.getOptions() != null && analysisType.getOptions().getFileTypes() != null) { + fileTypes = analysisType.getOptions().getFileTypes(); + } + + validateFileType(fileTypes, payload); + val schema = buildSchema(analysisType.getSchema()); validateWithSchema(schema, payload); } catch (ValidationException e) { @@ -94,6 +107,26 @@ public Optional validate(@NonNull JsonNode payload) { return Optional.ofNullable(errors); } + private void validateFileType(List fileTypes, @NonNull JsonNode payload) { + + if (CollectionUtils.isNotEmpty(fileTypes)) { + JsonNode files = payload.get("files"); + if (files.isArray()) { + for (JsonNode file : files) { + log.info("file is " + file); + String fileType = file.get("fileType").asText(); + String fileName = file.get("fileName").asText(); + if (!fileTypes.contains(fileType)) { + throw new ValidationException( + String.format( + "%s name is not supported, supported formats are %s", + fileName, String.join(", ", fileTypes))); + } + } + } + } + } + public void update(@NonNull String uploadId, String errorMessages) { if (isNull(errorMessages)) { updateAsValid(uploadId); diff --git a/song-server/src/main/java/bio/overture/song/server/utils/StringListConverter.java b/song-server/src/main/java/bio/overture/song/server/utils/StringListConverter.java new file mode 100644 index 000000000..718f613f8 --- /dev/null +++ b/song-server/src/main/java/bio/overture/song/server/utils/StringListConverter.java @@ -0,0 +1,32 @@ +package bio.overture.song.server.utils; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +@Converter +public class StringListConverter implements AttributeConverter, String> { + + @Override + public String convertToDatabaseColumn(List attribute) { + if (attribute == null || attribute.isEmpty()) { + return "{}"; + } + return "{" + + String.join( + ",", attribute.stream().map(s -> "\"" + s + "\"").collect(Collectors.toList())) + + "}"; + } + + @Override + public List convertToEntityAttribute(String dbData) { + if (dbData == null || dbData.equals("{}")) { + return Arrays.asList(); + } + return Arrays.stream(dbData.substring(1, dbData.length() - 1).split(",")) + .map(s -> s.replace("\"", "")) + .collect(Collectors.toList()); + } +} diff --git a/song-server/src/main/resources/db/migration/V1_20__update_file_from_enum.sql b/song-server/src/main/resources/db/migration/V1_20__update_file_from_enum.sql new file mode 100644 index 000000000..a77642b4a --- /dev/null +++ b/song-server/src/main/resources/db/migration/V1_20__update_file_from_enum.sql @@ -0,0 +1,5 @@ +ALTER TABLE public.file +ALTER COLUMN type TYPE text USING type::text; + +ALTER TABLE public.analysis_schema +ADD COLUMN file_types text[]; \ No newline at end of file diff --git a/song-server/src/main/resources/schemas/analysis/analysisBase.json b/song-server/src/main/resources/schemas/analysis/analysisBase.json index b73b16834..b91cea09c 100644 --- a/song-server/src/main/resources/schemas/analysis/analysisBase.json +++ b/song-server/src/main/resources/schemas/analysis/analysisBase.json @@ -18,22 +18,7 @@ }, "file": { "fileType": { - "type": "string", - "enum": [ - "FASTA", - "FAI", - "FASTQ", - "BAM", - "BAI", - "VCF", - "TBI", - "IDX", - "XML", - "TGZ", - "CRAM", - "CRAI", - "TXT" - ] + "type": "string" }, "fileData": { "type": "object", @@ -267,4 +252,4 @@ "items": { "$ref": "#/definitions/file/fileData" } } } -} +} \ No newline at end of file diff --git a/song-server/src/test/java/bio/overture/song/server/controller/AnalysisTypeControllerTest.java b/song-server/src/test/java/bio/overture/song/server/controller/AnalysisTypeControllerTest.java index 0a8dba9f2..825c9a658 100644 --- a/song-server/src/test/java/bio/overture/song/server/controller/AnalysisTypeControllerTest.java +++ b/song-server/src/test/java/bio/overture/song/server/controller/AnalysisTypeControllerTest.java @@ -51,6 +51,7 @@ import bio.overture.song.core.exceptions.ServerError; import bio.overture.song.core.model.AnalysisType; import bio.overture.song.core.model.AnalysisTypeId; +import bio.overture.song.core.model.AnalysisTypeOptions; import bio.overture.song.core.utils.RandomGenerator; import bio.overture.song.core.utils.ResourceFetcher; import bio.overture.song.server.model.dto.schema.RegisterAnalysisTypeRequest; @@ -849,7 +850,7 @@ public static List generateData( i -> { val name = names.get(i % repeats); val schema = generateRandomRegistrationPayload(randomGenerator); - val out = analysisTypeService.register(name, schema); + val out = analysisTypeService.register(name, new AnalysisTypeOptions(), schema); return out; }) .collect(toImmutableList());