some updates

Author: lindaxiang

get-song-dump.py

@@ -14,7 +14,7 @@ def download(song_url, file_dump):
     with open(file_dump, 'w') as fp:
         for study in studies:
             if study in ['TEST-CA']: continue
-            analyses = requests.get(song_url + ('/studies/%s/analysis?analysisStates=PUBLISHED' % (study))).json()
+            analyses = requests.get(song_url + ('/studies/%s/analysis?analysisStates=PUBLISHED,UNPUBLISHED' % (study))).json()
             for analysis in analyses:
                 fp.write(json.dumps(analysis)+"\n")
 

get-stats.py

@@ -17,6 +17,7 @@ then
 fi
 
 echo "Total $TYPE calls          : " $( cat ${VCFDIR}/*.${TYPE}.vcf | grep -cv "^#" )
+echo "Total $TYPE calls(no LOWDEPTH): " $( cat ${VCFDIR}/*.${TYPE}.vcf | grep -v "LOWDEPTH" | grep -cv "^#" )
 echo "Total validated $TYPE calls: " $( cat ${VCFDIR}/*.${TYPE}.vcf | grep -c "PASS" )
 echo ""
 for caller in ${callers}

notebooks/compare_pcawg_sanger_qc.ipynb

@@ -7,7 +7,8 @@
 from argparse import ArgumentParser
 import time
 import sys
-
+import csv
+import ast
 
 def song_operation(endpoint, operation, token, data=None):
 
@@ -46,9 +47,11 @@ def main():
 
     if args.report:
         with open(args.report, 'r') as fp:
-            for fline in fp:
-                run = json.loads(fline)
-                for a in run.get('run_output_analysis_to_suppress'):
+            reader = csv.DictReader(fp, delimiter='\t')
+            for run in reader:
+                alist = ast.literal_eval(run.get('run_output_analysis_to_suppress')) 
+                for a in alist:
+                    print(a)
                     endpoint = "%s/studies/%s/analysis/suppress/%s" % (args.song_url, run.get('studyId'), a)
                     operation = 'analysis_suppress'
                     song_operation(endpoint, operation, args.token)

quickstat

@@ -178,11 +178,11 @@ def vcf2tsv(vcf_dir):
             os.remove(filename)
     #concatenate the query results for all donors
     for fp in glob.glob(os.path.join(vcf_dir, "*.query.txt"), recursive=True):
-        projectId, donorId = os.path.basename(fp).split(".")[0:2]
+        projectId, donorId, sampleId, experiment = os.path.basename(fp).split(".")[0:5]
         evtype = os.path.basename(fp).split(".")[-3]
         cat = f'cat {fp}'
         awk = f'awk \'{{printf "\\t%s\\t%d\\t%s\\t%s\\t%s\\t%s\\t%s\\n\",$1,$2,$3,$4,$5,$6,$7}}\''
-        sed = f'sed "s/^/{donorId}/g" >> {vcf_dir}.{evtype}.all'
+        sed = f'sed "s/^/{projectId}\t{donorId}\t{sampleId}\t{experiment}/g" >> {vcf_dir}.{evtype}.all'
         cmd = '|'.join([cat, awk, sed])
         run_cmd(cmd)
 
@@ -246,7 +246,7 @@ def union_vcf(data_dir, union_dir):
     donor = set()
 
     for fn in glob.glob(os.path.join(data_dir, "*_annot_vcf", "*-*", "*.query.txt"), recursive=True):
-        donor.add(os.path.basename(fn).split("2020")[0].rstrip('.'))
+        donor.add(os.path.basename(fn).split(".2020")[0].rstrip('.'))
 
     for evtype in ['snv', 'indel']:
         for do in donor:
@@ -363,7 +363,9 @@ def snv_readcount_annot(union_dir, validated_dir, readcount_dir):
         os.makedirs(validated_dir)
 
     for fn in glob.glob(os.path.join(union_dir, "*.snv.vcf"), recursive=True):
-        projectId, donorId, sampleId, library_strategy, evtype, fileformat = os.path.basename(fn).split(".")
+        projectId, donorId, sampleId, library_strategy = os.path.basename(fp).split(".")[0:5]
+        evtype, fileformat = os.path.basename(fp).split(".")[-2:]
+
         output_vcf = os.path.join(validated_dir, '.'.join([projectId, donorId, 'validated', evtype, fileformat]))
         normal_rc = glob.glob(os.path.join(readcount_dir, '.'.join([projectId, donorId, sample[donorId]['normal'], 'targeted-seq', '*', 'aln.bam.rc'])))[0]
         tumour_rc = glob.glob(os.path.join(readcount_dir, '.'.join([projectId, donorId, sample[donorId]['tumour'], 'targeted-seq', '*', 'aln.bam.rc'])))[0]

song-suppress.py

@@ -63,7 +63,7 @@ def main():
 
 
     #download data and annotate
-    for wf in ['sanger', 'mutect2', 'mutect2-bqsr']:
+    for wf in ['sanger', 'mutect2']:
         subfolder = args.mode + '/' + wf
         if not include.get(wf): continue 
         download(args.dump_path, 'snv', args.token, args.metadata_url, args.storage_url, include.get(wf), subfolder)