Skip to content

Commit c567d53

Browse files
Winda0001Winda0001
committed
Add PubMed daily audit workflow
1 parent 33f63c9 commit c567d53

16 files changed

Lines changed: 2145 additions & 0 deletions

‎.github/workflows/pubmed-screen-audit.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
name: PubMed Screening Audit
2+
3+
on:
4+
schedule:
5+
# Avoid the top of the hour, where scheduled workflows are more likely to queue.
6+
- cron: "17 21 * * *"
7+
workflow_dispatch:
8+
inputs:
9+
retmax:
10+
description: "PubMed records per topic"
11+
required: false
12+
default: "25"
13+
14+
permissions:
15+
contents: read
16+
17+
jobs:
18+
pubmed-screen-audit:
19+
runs-on: ubuntu-latest
20+
defaults:
21+
run:
22+
working-directory: hypertension
23+
strategy:
24+
fail-fast: false
25+
matrix:
26+
include:
27+
- domain: hypertension
28+
label: hypertension
29+
topics: config/pubmed_screen_topics.json
30+
- domain: nasal
31+
label: nasal
32+
topics: config/pubmed_screen_topics_nasal.json
33+
34+
steps:
35+
- name: Checkout
36+
uses: actions/checkout@v4
37+
38+
- name: Set up Python
39+
uses: actions/setup-python@v5
40+
with:
41+
python-version: "3.12"
42+
43+
- name: Install package
44+
run: |
45+
python -m pip install --upgrade pip
46+
python -m pip install -e .
47+
48+
- name: Run PubMed screening and predownload
49+
env:
50+
NCBI_API_KEY: ${{ secrets.NCBI_API_KEY }}
51+
NCBI_EMAIL: ${{ secrets.NCBI_EMAIL }}
52+
RETMAX: ${{ github.event.inputs.retmax || '25' }}
53+
DOMAIN: ${{ matrix.domain }}
54+
TOPICS: ${{ matrix.topics }}
55+
run: |
56+
set -euo pipefail
57+
AUDIT_DATE="$(date -u +%F)"
58+
DAILY_DIR="audits/daily/${AUDIT_DATE}/${DOMAIN}"
59+
PMC_DIR="staging/pmc_xml/${AUDIT_DATE}/${DOMAIN}"
60+
REPORT_DIR="reports/daily/${AUDIT_DATE}/${DOMAIN}"
61+
mkdir -p "${DAILY_DIR}" "${PMC_DIR}" "${REPORT_DIR}"
62+
63+
python scripts/auto_pubmed_screen.py \
64+
--topics "${TOPICS}" \
65+
--output "${DAILY_DIR}/pubmed-screen-${DOMAIN}.jsonl" \
66+
--retmax "${RETMAX}"
67+
68+
python scripts/download_from_audit.py \
69+
--audit "${DAILY_DIR}/pubmed-screen-${DOMAIN}.jsonl" \
70+
--decisions "${DAILY_DIR}/pubmed-download-decisions-${DOMAIN}.jsonl" \
71+
--citation-shells "${DAILY_DIR}/pubmed-citation-shells-${DOMAIN}.jsonl" \
72+
--output-dir "${PMC_DIR}" \
73+
--evidence-root evidence
74+
75+
python scripts/render_audit_report.py \
76+
--domain "${DOMAIN}" \
77+
--date "${AUDIT_DATE}" \
78+
--audit "${DAILY_DIR}/pubmed-screen-${DOMAIN}.jsonl" \
79+
--decisions "${DAILY_DIR}/pubmed-download-decisions-${DOMAIN}.jsonl" \
80+
--output-dir "${REPORT_DIR}"
81+
82+
- name: Upload audit artifacts
83+
uses: actions/upload-artifact@v4
84+
with:
85+
name: pubmed-screen-audit-${{ matrix.label }}
86+
path: |
87+
hypertension/audits/daily/**/*.jsonl
88+
hypertension/staging/pmc_xml/**/*.xml
89+
hypertension/reports/daily/**/*.html
90+
hypertension/reports/daily/**/*.md
91+
if-no-files-found: error

‎hypertension/config/pubmed_screen_topics.json‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
[
2+
{
3+
"name": "core_hypertension_guidelines",
4+
"query": "(hypertension[MeSH Terms] OR hypertension[tiab] OR \"high blood pressure\"[tiab]) AND (\"Practice Guideline\"[pt] OR guideline[ti] OR consensus[tiab] OR recommendation*[tiab])",
5+
"tags": ["hypertension", "guideline", "recommendations"],
6+
"special_topic": false
7+
},
8+
{
9+
"name": "hypertension_systematic_reviews",
10+
"query": "(hypertension[MeSH Terms] OR hypertension[tiab] OR antihypertensive[tiab]) AND (\"systematic review\"[pt] OR meta-analysis[pt] OR \"systematic review\"[tiab] OR meta-analysis[tiab]) AND (treatment[tiab] OR management[tiab] OR target[tiab])",
11+
"tags": ["hypertension", "systematic_review", "meta_analysis"],
12+
"special_topic": false
13+
},
14+
{
15+
"name": "first_line_antihypertensive_strategy",
16+
"query": "(hypertension[MeSH Terms] OR hypertension[tiab]) AND (\"first line\"[tiab] OR initial[tiab] OR monotherapy[tiab] OR combination[tiab]) AND (antihypertensive[tiab] OR \"blood pressure\"[tiab])",
17+
"tags": ["hypertension", "first_line_treatment", "combination_therapy"],
18+
"special_topic": false
19+
},
20+
{
21+
"name": "blood_pressure_targets",
22+
"query": "(hypertension[MeSH Terms] OR hypertension[tiab]) AND (\"blood pressure target\"[tiab] OR \"target blood pressure\"[tiab] OR intensive[tiab] OR standard[tiab]) AND (trial[tiab] OR guideline[tiab] OR meta-analysis[tiab])",
23+
"tags": ["hypertension", "bp_target", "intensive_control"],
24+
"special_topic": false
25+
},
26+
{
27+
"name": "elderly_hypertension_management",
28+
"query": "(hypertension[MeSH Terms] OR hypertension[tiab]) AND (elderly[tiab] OR older[tiab] OR geriatric[tiab] OR frail[tiab]) AND (treatment[tiab] OR target[tiab] OR management[tiab] OR antihypertensive[tiab])",
29+
"tags": ["hypertension", "elderly", "frailty"],
30+
"special_topic": false
31+
},
32+
{
33+
"name": "hypertension_ckd_diabetes",
34+
"query": "(hypertension[MeSH Terms] OR hypertension[tiab]) AND (\"chronic kidney disease\"[tiab] OR CKD[tiab] OR diabetes[tiab] OR diabetic[tiab]) AND (treatment[tiab] OR target[tiab] OR management[tiab] OR renin angiotensin[tiab])",
35+
"tags": ["hypertension", "ckd", "diabetes", "comorbidity"],
36+
"special_topic": false
37+
},
38+
{
39+
"name": "resistant_hypertension_management",
40+
"query": "(\"resistant hypertension\"[tiab] OR \"refractory hypertension\"[tiab]) AND (management[tiab] OR treatment[tiab] OR spironolactone[tiab] OR renal denervation[tiab] OR guideline[tiab])",
41+
"tags": ["resistant_hypertension", "spironolactone", "renal_denervation"],
42+
"special_topic": true
43+
},
44+
{
45+
"name": "secondary_hypertension_screening",
46+
"query": "(\"secondary hypertension\"[tiab] OR \"primary aldosteronism\"[tiab] OR \"renovascular hypertension\"[tiab] OR pheochromocytoma[tiab]) AND (screening[tiab] OR diagnosis[tiab] OR management[tiab] OR treatment[tiab])",
47+
"tags": ["secondary_hypertension", "screening", "diagnosis"],
48+
"special_topic": true
49+
},
50+
{
51+
"name": "pregnancy_postpartum_hypertension",
52+
"query": "(\"hypertension in pregnancy\"[tiab] OR \"gestational hypertension\"[tiab] OR preeclampsia[tiab] OR postpartum[tiab]) AND (management[tiab] OR treatment[tiab] OR antihypertensive[tiab] OR guideline[tiab])",
53+
"tags": ["pregnancy_hypertension", "preeclampsia", "postpartum"],
54+
"special_topic": true
55+
},
56+
{
57+
"name": "orthostatic_supine_nocturnal_hypertension",
58+
"query": "(\"supine hypertension\"[tiab] OR \"orthostatic hypotension\"[tiab] OR \"nocturnal hypertension\"[tiab] OR \"blood pressure variability\"[tiab]) AND (management[tiab] OR treatment[tiab] OR monitoring[tiab])",
59+
"tags": ["supine_hypertension", "orthostatic_hypotension", "nocturnal_hypertension"],
60+
"special_topic": true
61+
}
62+
]

‎hypertension/config/pubmed_screen_topics_nasal.json‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
[
2+
{
3+
"name": "chronic_rhinosinusitis",
4+
"query": "(chronic rhinosinusitis OR CRS) AND (treatment OR management OR guideline OR systematic review OR randomized controlled trial)",
5+
"tags": ["nasal", "chronic_rhinosinusitis", "rhinosinusitis"]
6+
},
7+
{
8+
"name": "nasal_polyps",
9+
"query": "(chronic rhinosinusitis with nasal polyps OR CRSwNP OR nasal polyps) AND (treatment OR management OR biologics OR intranasal corticosteroids OR surgery)",
10+
"tags": ["nasal", "nasal_polyps", "crswnp"]
11+
},
12+
{
13+
"name": "nasal_polyps_biologics",
14+
"query": "(nasal polyps OR CRSwNP) AND (dupilumab OR omalizumab OR mepolizumab OR biologic OR biologics) AND (trial OR systematic review OR guideline)",
15+
"tags": ["nasal", "nasal_polyps", "biologics"]
16+
},
17+
{
18+
"name": "allergic_rhinitis",
19+
"query": "(allergic rhinitis) AND (intranasal corticosteroids OR antihistamine OR immunotherapy OR treatment OR guideline OR systematic review)",
20+
"tags": ["nasal", "allergic_rhinitis", "rhinitis"]
21+
},
22+
{
23+
"name": "nonallergic_rhinitis",
24+
"query": "(nonallergic rhinitis OR non-allergic rhinitis OR vasomotor rhinitis) AND (treatment OR management OR intranasal corticosteroids OR ipratropium)",
25+
"tags": ["nasal", "nonallergic_rhinitis", "rhinitis"],
26+
"special_topic": true
27+
},
28+
{
29+
"name": "acute_bacterial_rhinosinusitis",
30+
"query": "(acute bacterial rhinosinusitis OR acute sinusitis) AND (antibiotics OR treatment OR guideline OR randomized controlled trial)",
31+
"tags": ["nasal", "acute_rhinosinusitis", "sinusitis"]
32+
},
33+
{
34+
"name": "epistaxis",
35+
"query": "(epistaxis OR nosebleed) AND (management OR treatment OR cautery OR packing OR tranexamic acid OR guideline)",
36+
"tags": ["nasal", "epistaxis"],
37+
"special_topic": true
38+
},
39+
{
40+
"name": "nasal_obstruction_septal_deviation",
41+
"query": "(nasal obstruction OR septal deviation OR turbinate hypertrophy) AND (septoplasty OR turbinate reduction OR treatment OR outcomes)",
42+
"tags": ["nasal", "nasal_obstruction", "septal_deviation"]
43+
},
44+
{
45+
"name": "endoscopic_sinus_surgery",
46+
"query": "(endoscopic sinus surgery OR functional endoscopic sinus surgery OR FESS) AND (chronic rhinosinusitis OR nasal polyps) AND (outcomes OR guideline OR systematic review)",
47+
"tags": ["nasal", "endoscopic_sinus_surgery", "rhinosinusitis"]
48+
},
49+
{
50+
"name": "saline_irrigation_intranasal_steroids",
51+
"query": "(saline irrigation OR intranasal corticosteroids OR intranasal steroid) AND (rhinosinusitis OR allergic rhinitis OR nasal polyps) AND (randomized controlled trial OR systematic review)",
52+
"tags": ["nasal", "intranasal_corticosteroids", "saline_irrigation"]
53+
}
54+
]

‎hypertension/scripts/auto_pubmed_screen.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
"""Run PubMed candidate discovery and automated screening.
2+
3+
Usage from the hypertension/ directory:
4+
python scripts/auto_pubmed_screen.py --output tmp/pubmed_screen.jsonl --retmax 10
5+
python scripts/auto_pubmed_screen.py --topics config/pubmed_screen_topics.json --output tmp/pubmed_screen.jsonl
6+
"""
7+
8+
from __future__ import annotations
9+
10+
from pathlib import Path
11+
12+
from hypertensiondb.ingest.auto_pubmed_screen import main
13+
from hypertensiondb.utils.env import load_env_files
14+
15+
16+
load_env_files(Path(__file__).resolve().parent)
17+
18+
19+
if __name__ == "__main__":
20+
raise SystemExit(main())

‎hypertension/scripts/download_from_audit.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
"""Download eligible PMC XML files from a PubMed screening audit JSONL.
2+
3+
Usage from the hypertension/ directory:
4+
python scripts/download_from_audit.py --audit audits/pubmed-screen.jsonl \
5+
--decisions staging/download-decisions.jsonl --output-dir staging/pmc_xml
6+
"""
7+
8+
from __future__ import annotations
9+
10+
from pathlib import Path
11+
12+
from hypertensiondb.ingest.audit_download import main
13+
from hypertensiondb.utils.env import load_env_files
14+
15+
16+
load_env_files(Path(__file__).resolve().parent)
17+
18+
19+
if __name__ == "__main__":
20+
raise SystemExit(main())

‎hypertension/scripts/render_audit_report.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
"""Render Chinese HTML/Markdown reports for PubMed audit artifacts."""
2+
3+
from __future__ import annotations
4+
5+
from pathlib import Path
6+
7+
from hypertensiondb.ingest.audit_report import main
8+
from hypertensiondb.utils.env import load_env_files
9+
10+
11+
load_env_files(Path(__file__).resolve().parent)
12+
13+
14+
if __name__ == "__main__":
15+
raise SystemExit(main())

0 commit comments

Comments
 (0)