Skip to content

Commit cded800

Browse files
Azher2AliAzher2Ali
authored
Merge pull request #858 from overture-stack/feature/test_changes
Enable Dynamic Schema Support for File Type Definitions in Analysis Types
2 parents 573a4b2 + 571d79f commit cded800

File tree

16 files changed

+144
-46
lines changed

16 files changed

+144
-46
lines changed

‎pom.xml

+4-1
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,12 @@
5757
<url>https://jcenter.bintray.com/</url>
5858
</repository>
5959
<repository>
60-
<id>spring-releases</id>
60+
<id>lib-releases</id>
6161
<name>Spring Releases</name>
6262
<url>https://repo.spring.io/libs-release</url>
63+
<snapshots>
64+
<enabled>false</enabled>
65+
</snapshots>
6366
</repository>
6467
</repositories>
6568

‎song-core/src/main/java/bio/overture/song/core/model/AnalysisType.java

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ public class AnalysisType {
1818
@NotNull private String name;
1919
@NotNull private Integer version;
2020
private LocalDateTime createdAt;
21+
private AnalysisTypeOptions options;
2122

2223
@JsonInclude(JsonInclude.Include.NON_NULL)
2324
private JsonNode schema;

‎song-core/src/main/java/bio/overture/song/core/model/AnalysisTypeOptions.java

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package bio.overture.song.core.model;
2+
3+
import java.util.List;
4+
import lombok.*;
5+
6+
@Data
7+
@Builder
8+
@NoArgsConstructor
9+
@AllArgsConstructor
10+
public class AnalysisTypeOptions {
11+
private List<String> fileTypes;
12+
}

‎song-server/pom.xml

-7
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,6 @@
2525
<modelVersion>4.0.0</modelVersion>
2626

2727
<artifactId>song-server</artifactId>
28-
<repositories>
29-
<repository>
30-
<id>spring-releases</id>
31-
<name>Spring Releases</name>
32-
<url>https://repo.spring.io/libs-release</url>
33-
</repository>
34-
</repositories>
3528

3629
<dependencies>
3730

‎song-server/src/main/java/bio/overture/song/server/controller/analysisType/AnalysisTypeController.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,8 @@ public String getAnalysisTypeRegistrationSchema() {
108108
public @ResponseBody AnalysisType register(
109109
@RequestHeader(value = AUTHORIZATION, required = false) final String accessToken,
110110
@RequestBody RegisterAnalysisTypeRequest request) {
111-
return analysisTypeService.register(request.getName(), request.getSchema());
111+
return analysisTypeService.register(
112+
request.getName(), request.getOptions(), request.getSchema());
112113
}
113114

114115
@ApiImplicitParams({

‎song-server/src/main/java/bio/overture/song/server/model/dto/schema/RegisterAnalysisTypeRequest.java

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_ABSENT;
44

5+
import bio.overture.song.core.model.AnalysisTypeOptions;
56
import com.fasterxml.jackson.annotation.JsonInclude;
67
import com.fasterxml.jackson.databind.JsonNode;
78
import lombok.AllArgsConstructor;
@@ -17,4 +18,5 @@
1718
public class RegisterAnalysisTypeRequest {
1819
private String name;
1920
private JsonNode schema;
21+
private AnalysisTypeOptions options;
2022
}

‎song-server/src/main/java/bio/overture/song/server/model/entity/AnalysisSchema.java

+8-13
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,20 @@
11
package bio.overture.song.server.model.entity;
22

3-
import static bio.overture.song.server.model.enums.TableAttributeNames.ID;
4-
import static bio.overture.song.server.model.enums.TableAttributeNames.NAME;
5-
import static bio.overture.song.server.model.enums.TableAttributeNames.SCHEMA;
6-
import static bio.overture.song.server.model.enums.TableAttributeNames.VERSION;
3+
import static bio.overture.song.server.model.enums.TableAttributeNames.*;
74
import static bio.overture.song.server.repository.CustomJsonType.CUSTOM_JSON_TYPE_PKG_PATH;
85
import static com.google.common.collect.Sets.newHashSet;
96

107
import bio.overture.song.server.model.analysis.Analysis;
118
import bio.overture.song.server.model.enums.ModelAttributeNames;
129
import bio.overture.song.server.model.enums.TableAttributeNames;
1310
import bio.overture.song.server.model.enums.TableNames;
11+
import bio.overture.song.server.utils.StringListConverter;
1412
import com.fasterxml.jackson.annotation.JsonIgnore;
1513
import com.fasterxml.jackson.databind.JsonNode;
1614
import java.time.LocalDateTime;
15+
import java.util.List;
1716
import java.util.Set;
18-
import javax.persistence.CascadeType;
19-
import javax.persistence.Column;
20-
import javax.persistence.Entity;
21-
import javax.persistence.FetchType;
22-
import javax.persistence.GeneratedValue;
23-
import javax.persistence.GenerationType;
24-
import javax.persistence.Id;
25-
import javax.persistence.OneToMany;
26-
import javax.persistence.Table;
17+
import javax.persistence.*;
2718
import javax.validation.constraints.NotNull;
2819
import lombok.AllArgsConstructor;
2920
import lombok.Builder;
@@ -63,6 +54,10 @@ public class AnalysisSchema {
6354
@Type(type = CUSTOM_JSON_TYPE_PKG_PATH)
6455
private JsonNode schema;
6556

57+
@Column(name = FILE_TYPES, columnDefinition = "text[]")
58+
@Convert(converter = StringListConverter.class)
59+
private List<String> fileTypes;
60+
6661
@JsonIgnore
6762
@Builder.Default
6863
@ToString.Exclude

‎song-server/src/main/java/bio/overture/song/server/model/enums/TableAttributeNames.java

+1
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ public class TableAttributeNames {
7676
public static final String DATA_TYPE = "data_type";
7777
public static final String TUMOUR_NORMAL_DESIGNATION = "tumour_normal_designation";
7878
public static final String TISSUE_SOURCE = "tissue_source";
79+
public static final String FILE_TYPES = "file_types";
7980

8081
public static final String INITIAL_STATE = "initial_state";
8182
public static final String UPDATED_STATE = "updated_state";

‎song-server/src/main/java/bio/overture/song/server/repository/AnalysisSchemaRepository.java

+3
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package bio.overture.song.server.repository;
1919

2020
import bio.overture.song.server.model.entity.AnalysisSchema;
21+
import java.util.List;
2122
import java.util.Optional;
2223
import org.springframework.data.jpa.repository.JpaRepository;
2324
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
@@ -30,4 +31,6 @@ public interface AnalysisSchemaRepository
3031
Integer countAllByNameAndIdLessThanEqual(String name, Integer id);
3132

3233
Optional<AnalysisSchema> findByNameAndVersion(String name, Integer version);
34+
35+
List<AnalysisSchema> findByName(String name);
3336
}

‎song-server/src/main/java/bio/overture/song/server/repository/search/IdSearchRequest.java

+7-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,13 @@ public class IdSearchRequest {
4747
private final String submitterSpecimenIds;
4848

4949
public static IdSearchRequest createIdSearchRequest(
50-
String donorId, String sampleId, String specimenId, String objectId, String submitterSampleId, String submitterDonorIds, String submitterSpecimenIds) {
50+
String donorId,
51+
String sampleId,
52+
String specimenId,
53+
String objectId,
54+
String submitterSampleId,
55+
String submitterDonorIds,
56+
String submitterSpecimenIds) {
5157
return new IdSearchRequest(
5258
getGlobPattern(donorId),
5359
getGlobPattern(sampleId),

‎song-server/src/main/java/bio/overture/song/server/service/AnalysisTypeService.java

+30-5
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import bio.overture.song.core.exceptions.ServerErrors;
4343
import bio.overture.song.core.model.AnalysisType;
4444
import bio.overture.song.core.model.AnalysisTypeId;
45+
import bio.overture.song.core.model.AnalysisTypeOptions;
4546
import bio.overture.song.core.model.PageDTO;
4647
import bio.overture.song.server.controller.analysisType.AnalysisTypeController;
4748
import bio.overture.song.server.model.entity.AnalysisSchema;
@@ -50,14 +51,15 @@
5051
import com.fasterxml.jackson.databind.node.ArrayNode;
5152
import com.fasterxml.jackson.databind.node.ObjectNode;
5253
import java.io.IOException;
53-
import java.util.Collection;
54+
import java.util.*;
5455
import java.util.function.Supplier;
5556
import java.util.regex.Pattern;
5657
import javax.transaction.Transactional;
5758
import lombok.NonNull;
5859
import lombok.SneakyThrows;
5960
import lombok.extern.slf4j.Slf4j;
6061
import lombok.val;
62+
import org.apache.commons.collections.CollectionUtils;
6163
import org.everit.json.schema.Schema;
6264
import org.everit.json.schema.SchemaException;
6365
import org.everit.json.schema.ValidationException;
@@ -149,21 +151,29 @@ public AnalysisSchema getAnalysisSchema(String name, Integer version) {
149151
public AnalysisType getAnalysisType(
150152
@NonNull AnalysisTypeId analysisTypeId, boolean unrenderedOnly) {
151153
val analysisSchema = getAnalysisSchema(analysisTypeId);
154+
152155
val resolvedSchemaJson =
153156
resolveSchemaJsonView(analysisSchema.getSchema(), unrenderedOnly, false);
157+
158+
List<String> fileTypes =
159+
(analysisSchema.getFileTypes() != null && !analysisSchema.getFileTypes().isEmpty())
160+
? analysisSchema.getFileTypes()
161+
: new ArrayList<>();
154162
return AnalysisType.builder()
155163
.name(analysisTypeId.getName())
156164
.version(analysisTypeId.getVersion())
157165
.createdAt(analysisSchema.getCreatedAt())
158166
.schema(resolvedSchemaJson)
167+
.options(AnalysisTypeOptions.builder().fileTypes(fileTypes).build())
159168
.build();
160169
}
161170

162171
@Transactional
163-
public AnalysisType register(@NonNull String analysisTypeName, JsonNode analysisTypeSchema) {
172+
public AnalysisType register(
173+
@NonNull String analysisTypeName, AnalysisTypeOptions options, JsonNode analysisTypeSchema) {
164174
validateAnalysisTypeName(analysisTypeName);
165175
validateAnalysisTypeSchema(analysisTypeSchema);
166-
return commitAnalysisType(analysisTypeName, analysisTypeSchema);
176+
return commitAnalysisType(analysisTypeName, analysisTypeSchema, options);
167177
}
168178

169179
public PageDTO<AnalysisType> listAnalysisTypes(
@@ -246,9 +256,23 @@ private void validateAnalysisTypeSchema(JsonNode analysisTypeSchema) {
246256

247257
@SneakyThrows
248258
private AnalysisType commitAnalysisType(
249-
@NonNull String analysisTypeName, @NonNull JsonNode analysisTypeSchema) {
259+
@NonNull String analysisTypeName,
260+
@NonNull JsonNode analysisTypeSchema,
261+
AnalysisTypeOptions options) {
262+
263+
List<String> fileTypes = new ArrayList<>();
264+
265+
if (options != null && CollectionUtils.isNotEmpty(options.getFileTypes())) {
266+
fileTypes = options.getFileTypes();
267+
}
250268
val analysisSchema =
251-
AnalysisSchema.builder().name(analysisTypeName).schema(analysisTypeSchema).build();
269+
AnalysisSchema.builder()
270+
.name(analysisTypeName)
271+
.schema(analysisTypeSchema)
272+
.fileTypes(fileTypes)
273+
.build();
274+
275+
log.debug("Creating analysisSchema with file types: {} " + fileTypes);
252276
analysisSchemaRepository.save(analysisSchema);
253277
val version =
254278
analysisSchemaRepository.countAllByNameAndIdLessThanEqual(
@@ -275,6 +299,7 @@ private AnalysisType commitAnalysisType(
275299
.version(version)
276300
.createdAt(createdAt)
277301
.schema(resolvedSchemaJson)
302+
.options(options)
278303
.build();
279304
}
280305

‎song-server/src/main/java/bio/overture/song/server/service/ValidationService.java

+33
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,15 @@
3535
import bio.overture.song.server.validation.SchemaValidator;
3636
import bio.overture.song.server.validation.ValidationResponse;
3737
import com.fasterxml.jackson.databind.JsonNode;
38+
import java.util.ArrayList;
39+
import java.util.List;
3840
import java.util.Optional;
3941
import java.util.function.Supplier;
4042
import lombok.NonNull;
43+
import lombok.SneakyThrows;
4144
import lombok.extern.slf4j.Slf4j;
4245
import lombok.val;
46+
import org.apache.commons.collections.CollectionUtils;
4347
import org.everit.json.schema.Schema;
4448
import org.everit.json.schema.ValidationException;
4549
import org.springframework.beans.factory.annotation.Autowired;
@@ -73,6 +77,7 @@ public ValidationService(
7377
this.analysisTypeIdSchema = analysisTypeIdSchemaSupplier.get();
7478
}
7579

80+
@SneakyThrows
7681
public Optional<String> validate(@NonNull JsonNode payload) {
7782
String errors = null;
7883
try {
@@ -85,6 +90,14 @@ public Optional<String> validate(@NonNull JsonNode payload) {
8590
"Found Analysis type: name=%s version=%s",
8691
analysisType.getName(), analysisType.getVersion()));
8792

93+
List<String> fileTypes = new ArrayList<>();
94+
95+
if (analysisType.getOptions() != null && analysisType.getOptions().getFileTypes() != null) {
96+
fileTypes = analysisType.getOptions().getFileTypes();
97+
}
98+
99+
validateFileType(fileTypes, payload);
100+
88101
val schema = buildSchema(analysisType.getSchema());
89102
validateWithSchema(schema, payload);
90103
} catch (ValidationException e) {
@@ -94,6 +107,26 @@ public Optional<String> validate(@NonNull JsonNode payload) {
94107
return Optional.ofNullable(errors);
95108
}
96109

110+
private void validateFileType(List<String> fileTypes, @NonNull JsonNode payload) {
111+
112+
if (CollectionUtils.isNotEmpty(fileTypes)) {
113+
JsonNode files = payload.get("files");
114+
if (files.isArray()) {
115+
for (JsonNode file : files) {
116+
log.info("file is " + file);
117+
String fileType = file.get("fileType").asText();
118+
String fileName = file.get("fileName").asText();
119+
if (!fileTypes.contains(fileType)) {
120+
throw new ValidationException(
121+
String.format(
122+
"%s name is not supported, supported formats are %s",
123+
fileName, String.join(", ", fileTypes)));
124+
}
125+
}
126+
}
127+
}
128+
}
129+
97130
public void update(@NonNull String uploadId, String errorMessages) {
98131
if (isNull(errorMessages)) {
99132
updateAsValid(uploadId);

‎song-server/src/main/java/bio/overture/song/server/utils/StringListConverter.java

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package bio.overture.song.server.utils;
2+
3+
import java.util.Arrays;
4+
import java.util.List;
5+
import java.util.stream.Collectors;
6+
import javax.persistence.AttributeConverter;
7+
import javax.persistence.Converter;
8+
9+
@Converter
10+
public class StringListConverter implements AttributeConverter<List<String>, String> {
11+
12+
@Override
13+
public String convertToDatabaseColumn(List<String> attribute) {
14+
if (attribute == null || attribute.isEmpty()) {
15+
return "{}";
16+
}
17+
return "{"
18+
+ String.join(
19+
",", attribute.stream().map(s -> "\"" + s + "\"").collect(Collectors.toList()))
20+
+ "}";
21+
}
22+
23+
@Override
24+
public List<String> convertToEntityAttribute(String dbData) {
25+
if (dbData == null || dbData.equals("{}")) {
26+
return Arrays.asList();
27+
}
28+
return Arrays.stream(dbData.substring(1, dbData.length() - 1).split(","))
29+
.map(s -> s.replace("\"", ""))
30+
.collect(Collectors.toList());
31+
}
32+
}

‎song-server/src/main/resources/db/migration/V1_20__update_file_from_enum.sql

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
ALTER TABLE public.file
2+
ALTER COLUMN type TYPE text USING type::text;
3+
4+
ALTER TABLE public.analysis_schema
5+
ADD COLUMN file_types text[];

‎song-server/src/main/resources/schemas/analysis/analysisBase.json

+2-17
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,7 @@
1818
},
1919
"file": {
2020
"fileType": {
21-
"type": "string",
22-
"enum": [
23-
"FASTA",
24-
"FAI",
25-
"FASTQ",
26-
"BAM",
27-
"BAI",
28-
"VCF",
29-
"TBI",
30-
"IDX",
31-
"XML",
32-
"TGZ",
33-
"CRAM",
34-
"CRAI",
35-
"TXT"
36-
]
21+
"type": "string"
3722
},
3823
"fileData": {
3924
"type": "object",
@@ -267,4 +252,4 @@
267252
"items": { "$ref": "#/definitions/file/fileData" }
268253
}
269254
}
270-
}
255+
}

0 commit comments

Comments
 (0)