Port to run as import process

Remove main methods
Port from lxml to xml
Rename for clarity

Author: Jongmassey

coding_systems/icd10/scrape.py

@@ -4,15 +4,14 @@
 import traceback
 from collections import defaultdict
 from pathlib import Path
+from tempfile import TemporaryDirectory
 from urllib import parse
 
 import requests as rq
 from bs4 import BeautifulSoup, element
 
 
 BASE_URL = "https://classbrowser.nhs.uk/"
-BASE_PATH = Path("icdscrape")
-
 errors = {}
 
 
@@ -50,7 +49,15 @@ def get_block_urls(menu_json_path):
     )
 
 
-def get_menu_chapters(menu):
+def get_menu_chapters(menu, html_dir):
+    def fetch_html(url, force_download=False):
+        htmls_dir = html_dir / "html_cache"
+        htmls_dir.mkdir(parents=True, exist_ok=True)
+        file_path = htmls_dir / os.path.split(parse.urlparse(url).path)[-1]
+        if force_download or not file_path.exists():
+            download_stream(url, file_path)
+        return file_path
+
     def split_text(text):
         return text.split(": ")
 
@@ -134,15 +141,6 @@ def get_blocks(chapter):
         )
 
 
-def fetch_html(url, force_download=False):
-    htmls_dir = BASE_PATH / "html_cache"
-    htmls_dir.mkdir(parents=True, exist_ok=True)
-    file_path = htmls_dir / os.path.split(parse.urlparse(url).path)[-1]
-    if force_download or not file_path.exists():
-        download_stream(url, file_path)
-    return file_path
-
-
 def label_text(elem):
     label = " ".join(
         "".join(
@@ -399,18 +397,6 @@ def get_html_body(html_path):
 
 def scrape():
     menu = fetch_menu_json()
-    chapters = {k: v for k, v in get_menu_chapters(menu)}
-
-    json.dump(
-        chapters, (BASE_PATH / "chapters.json").open("w"), indent=4, ensure_ascii=False
-    )
-    json.dump(
-        errors, (BASE_PATH / "errors.json").open("w"), indent=4, ensure_ascii=False
-    )
-
-    if errors:
-        print("Errors found, check errors.json")
-
-
-if __name__ == "__main__":
-    scrape()
+    with TemporaryDirectory() as temp:
+        chapters = {k: v for k, v in get_menu_chapters(menu, Path(temp))}
+    return chapters

coding_systems/icd10/json_to_claml.py coding_systems/icd10/scraped_to_claml.pycoding_systems/icd10/json_to_claml.py renamed to coding_systems/icd10/scraped_to_claml.py

@@ -1,14 +1,11 @@
-import argparse
-import json
 import re
+import xml.etree.ElementTree as etree
 from collections import defaultdict
 from dataclasses import dataclass, field
 from enum import Enum, EnumMeta, auto
 from itertools import count
 from pathlib import Path
 
-from lxml import etree
-
 
 JSON_PATH = Path("icdscrape/chapters.json")
 CLAML_PATH = Path("icdscrape/chapters.xml")
@@ -416,11 +413,8 @@ def append_subclasses():
             _class.subclasses.append(SubClass(code=child.code))
 
 
-def main(json_path, claml_path):
-    chapters = json.load(json_path.open())
-    claml = etree.fromstring(
-        CLAML_HEADER, parser=etree.XMLParser(remove_blank_text=True)
-    )
+def convert_chapters_to_claml(chapters, claml_path):
+    claml = etree.fromstring(CLAML_HEADER)
     claml.append(ClassKind.to_element())
     claml.append(UsageKind.to_element())
     claml.append(RubricKind.to_element())
@@ -433,27 +427,5 @@ def main(json_path, claml_path):
     for _class in classes_to_write:
         claml.append(_class.to_element())
     xml = etree.ElementTree(claml)
-    xml.write(claml_path, pretty_print=True, xml_declaration=True, encoding="UTF-8")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        prog="json_to_claml",
-        description="converts scraped chapters.json to ClaML format",
-    )
-    parser.add_argument(
-        "json_path",
-        nargs="?",
-        default=JSON_PATH,
-        help="Path to input json file",
-        type=Path,
-    )
-    parser.add_argument(
-        "claml_path",
-        nargs="?",
-        default=CLAML_PATH,
-        help="Output path for ClaML file",
-        type=Path,
-    )
-    args = parser.parse_args()
-    main(args.json_path, args.claml_path)
+    etree.indent(xml, space="\t", level=0)
+    xml.write(claml_path, xml_declaration=True, encoding="UTF-8")