Merge pull request #264 from Ferlab-Ste-Justine/feat/clin-4060

feat: CLIN-4060 use gnomad 4 for hc_complement calc

Author: meek0

datalake-spark3/src/main/scala/bio/ferlab/datalake/spark3/publictables/enriched/RareVariant.scala

@@ -13,7 +13,7 @@ import java.time.LocalDateTime
 case class RareVariant(rc: RuntimeETLContext) extends SimpleSingleETL(rc) {
 
   override val mainDestination: DatasetConf = conf.getDataset("enriched_rare_variant")
-  val gnomad: DatasetConf = conf.getDataset("normalized_gnomad_genomes_v3")
+  val gnomad: DatasetConf = conf.getDataset("normalized_gnomad_joint_v4")
 
   override def extract(lastRunValue: LocalDateTime,
                        currentRunValue: LocalDateTime): Map[String, DataFrame] = {
@@ -25,9 +25,9 @@ case class RareVariant(rc: RuntimeETLContext) extends SimpleSingleETL(rc) {
                                lastRunValue: LocalDateTime,
                                currentRunValue: LocalDateTime): DataFrame = {
     data(gnomad.id)
-      .select(columns.locus :+ col("af"): _*)
+      .select(columns.locus :+ col("af_joint"): _*)
       .groupByLocus()
-      .agg(max("af") as "af")
+      .agg(max("af_joint") as "af")
       .withColumn("is_rare", col("af") <= 0.01)
   }
 

datalake-spark3/src/test/scala/bio/ferlab/datalake/spark3/publictables/enriched/RareVariantSpec.scala

@@ -1,8 +1,9 @@
 package bio.ferlab.datalake.spark3.publictables.enriched
 
 import bio.ferlab.datalake.commons.config.DatasetConf
-import bio.ferlab.datalake.testutils.models.enriched.{EnrichedRareVariantInput, EnrichedRareVariantOutput}
+import bio.ferlab.datalake.testutils.models.enriched.EnrichedRareVariantOutput
 import bio.ferlab.datalake.spark3.testutils.WithTestConfig
+import bio.ferlab.datalake.testutils.models.normalized.NormalizedGnomadJoint4
 import bio.ferlab.datalake.testutils.{SparkSpec, TestETLContext}
 
 class RareVariantSpec extends SparkSpec with WithTestConfig {
@@ -13,14 +14,14 @@ class RareVariantSpec extends SparkSpec with WithTestConfig {
 
   val gnomad_df: DatasetConf = job.gnomad
 
-  "transformSingle" should "transform Gnomad v2 to rare variant" in {
+  "transformSingle" should "transform Gnomad v4 to rare variant" in {
     val inputData = Map(
       gnomad_df.id -> Seq(
-        EnrichedRareVariantInput(chromosome = "1", start = 1000, reference = "A", alternate = "T", af = 0.005),
-        EnrichedRareVariantInput(chromosome = "1", start = 1000, reference = "A", alternate = "T", af = 0.03),
-        EnrichedRareVariantInput(chromosome = "1", start = 2000, reference = "A", alternate = "T", af = 0.011),
-        EnrichedRareVariantInput(chromosome = "2", start = 1000, reference = "A", alternate = "T", af = 0.005),
-        EnrichedRareVariantInput(chromosome = "2", start = 1000, reference = "A", alternate = "T", af = 0.01)
+        NormalizedGnomadJoint4(chromosome = "1", start = 1000, reference = "A", alternate = "T", af_joint = 0.005),
+        NormalizedGnomadJoint4(chromosome = "1", start = 1000, reference = "A", alternate = "T", af_joint = 0.03),
+        NormalizedGnomadJoint4(chromosome = "1", start = 2000, reference = "A", alternate = "T", af_joint = 0.011),
+        NormalizedGnomadJoint4(chromosome = "2", start = 1000, reference = "A", alternate = "T", af_joint = 0.005),
+        NormalizedGnomadJoint4(chromosome = "2", start = 1000, reference = "A", alternate = "T", af_joint = 0.01)
 
       )
         .toDF()

datalake-test-utils/src/main/scala/bio/ferlab/datalake/testutils/models/enriched/EnrichedRareVariantInput.scala

@@ -4,7 +4,7 @@ case class EnrichedRareVariantInput(chromosome: String = "1",
                                     start: Long = 210862942,
                                     reference: String = "GGCA",
                                     alternate: String = "G",
-                                    af: Double = 1.0
+                                    af_joint: Double = 1.0
                                    ) {
 
 }