fix: load sheets in subdirectory (#188)

Refactor VFS design and add recursive file loading for GitHub VFS

Author: XiangYyang

src/base/dom/drag-drop.ts

@@ -6,11 +6,11 @@
 
 import {
     DragAndDropFileSystem,
-    VirtualFileSystem,
+    type IFileSystem,
 } from "../../kicanvas/services/vfs";
 
 export class DropTarget {
-    constructor(elm: HTMLElement, callback: (fs: VirtualFileSystem) => void) {
+    constructor(elm: HTMLElement, callback: (fs: IFileSystem) => void) {
         elm.addEventListener(
             "dragenter",
             (e) => {

src/base/paths.ts

@@ -21,3 +21,31 @@ export function basename(path: string | URL) {
 export function extension(path: string) {
     return path.split(".").at(-1) ?? "";
 }
+
+/**
+ * Path.join and normalize the result,
+ * It DOES NOT check relative path or absolute path.
+ * + `('', '/qwq')` -> `qwq`
+ * + `('///', '/qwq/', 'file')` -> `qwq/file`
+ */
+export function normalize_join(...parts: string[]): string {
+    return parts
+        .flatMap((p) => p.split("/"))
+        .filter((s) => s !== "")
+        .join("/");
+}
+
+/**
+ * Return relative path of `absolute` based on `parent`.
+ * + `('qwq/abc', 'qwq/abc/def/file')` -> `def/file`
+ */
+export function based_on(parent: string, absolute: string): string {
+    if (parent === absolute) {
+        return "";
+    }
+    const base = normalize_join(parent);
+    const prefix = base.length > 0 ? base + "/" : "";
+    return absolute.startsWith(prefix)
+        ? absolute.slice(prefix.length)
+        : absolute;
+}

src/kicanvas/elements/kicanvas-embed.ts

@@ -20,7 +20,7 @@ import {
     FetchFileSystem,
     LocalFileSystem,
     MergedFileSystem,
-    VirtualFileSystem,
+    type IFileSystem,
 } from "../services/vfs";
 import type { KCBoardAppElement } from "./kc-board/app";
 import type { KCSchematicAppElement } from "./kc-schematic/app";
@@ -159,11 +159,12 @@ class KiCanvasEmbedElement extends KCUIElement {
         await this.#setup_project(vfs);
     }
 
-    async #setup_project(vfs: VirtualFileSystem) {
+    async #setup_project(vfs: IFileSystem) {
         this.loaded = false;
         this.loading = true;
 
         try {
+            await vfs.setup();
             await this.#project.load(vfs);
 
             this.loaded = true;

src/kicanvas/elements/kicanvas-shell.ts

@@ -13,7 +13,7 @@ import { sprites_url } from "../icons/sprites";
 import { Project } from "../project";
 import { GitHubFileSystem } from "../services/github-vfs";
 import { CodebergFileSystem } from "../services/codeberg-vfs";
-import { FetchFileSystem, type VirtualFileSystem } from "../services/vfs";
+import { FetchFileSystem, type IFileSystem } from "../services/vfs";
 import { KCBoardAppElement } from "./kc-board/app";
 import { KCSchematicAppElement } from "./kc-schematic/app";
 
@@ -88,15 +88,18 @@ class KiCanvasShellElement extends KCUIElement {
             ...url_params.getAll("repo"),
         ];
 
+        // Only load the first URL
+        const url = urls[0];
+
         later(async () => {
             if (this.src) {
                 const vfs = new FetchFileSystem([this.src]);
                 await this.setup_project(vfs);
                 return;
             }
 
-            if (urls.length) {
-                const vfs = await this.load_repo(...urls);
+            if (url) {
+                const vfs = await this.load_repo(url);
                 if (!vfs) {
                     return;
                 }
@@ -134,20 +137,19 @@ class KiCanvasShellElement extends KCUIElement {
         });
     }
 
-    private async load_repo(
-        ...url: string[]
-    ): Promise<VirtualFileSystem | null> {
+    private async load_repo(url: string): Promise<IFileSystem | null> {
         return (
-            (await GitHubFileSystem.fromURLs(...url)) ??
-            (await CodebergFileSystem.fromURLs(...url))
+            (await GitHubFileSystem.fromURLs(url)) ??
+            (await CodebergFileSystem.fromURLs(url))
         );
     }
 
-    private async setup_project(vfs: VirtualFileSystem) {
+    private async setup_project(vfs: IFileSystem) {
         this.loaded = false;
         this.loading = true;
 
         try {
+            await vfs.setup();
             await this.project.load(vfs);
             this.project.set_active_page(this.project.first_page);
             this.loaded = true;

src/kicanvas/project.ts

@@ -9,18 +9,19 @@ import { Barrier } from "../base/async";
 import { type IDisposable } from "../base/disposable";
 import { first, length, map } from "../base/iterator";
 import { Logger } from "../base/log";
+import { dirname, normalize_join } from "../base/paths";
 import { is_string, type Constructor } from "../base/types";
 import { KicadPCB, KicadSch, ProjectSettings } from "../kicad";
 import type {
     SchematicSheet,
     SchematicSheetInstance,
 } from "../kicad/schematic";
-import type { VirtualFileSystem } from "./services/vfs";
+import type { IFileSystem } from "./services/vfs";
 
 const log = new Logger("kicanvas:project");
 
 export class Project extends EventTarget implements IDisposable {
-    #fs: VirtualFileSystem;
+    #fs: IFileSystem;
     #files_by_name: Map<string, KicadPCB | KicadSch | null> = new Map();
     #pages_by_path: Map<string, ProjectPage> = new Map();
     #root_schematic_page?: ProjectPage;
@@ -33,7 +34,7 @@ export class Project extends EventTarget implements IDisposable {
         this.#pages_by_path.clear();
     }
 
-    public async load(fs: VirtualFileSystem) {
+    public async load(fs: IFileSystem) {
         log.info(`Loading project from ${fs.constructor.name}`);
 
         this.settings = new ProjectSettings();
@@ -42,30 +43,53 @@ export class Project extends EventTarget implements IDisposable {
 
         this.#fs = fs;
 
-        let promises = [];
+        const proj_promises = [];
 
+        // load project file
         for (const filename of this.#fs.list()) {
-            promises.push(this.#load_file(filename));
+            proj_promises.push(this.#load_file(filename));
         }
 
-        await Promise.all(promises);
+        await Promise.all(proj_promises);
 
-        while (promises.length) {
-            // 'Recursively' resolve all schematics until none remain
-            promises = [];
-            for (const schematic of this.schematics()) {
-                for (const sheet of schematic.sheets) {
-                    const sheet_sch = this.#files_by_name.get(
-                        sheet.sheetfile ?? "",
-                    ) as KicadSch;
+        // 'Recursively' resolve all schematics until none remain
+        let load_new = true;
+        const skipped_files: string[] = [];
+        while (load_new) {
+            load_new = false;
 
-                    if (!sheet_sch && sheet.sheetfile) {
-                        // Missing schematic, attempt to fetch
-                        promises.push(this.#load_file(sheet.sheetfile));
+            const loaded_file = Array.from(this.schematics());
+            for (const sch of loaded_file) {
+                const base_dir = dirname(sch.filename);
+                for (const subsch of sch.sheets) {
+                    if (!subsch.sheetfile) {
+                        continue;
+                    }
+
+                    const new_file = normalize_join(base_dir, subsch.sheetfile);
+
+                    const loaded = loaded_file.map((s) => s.filename);
+                    if (
+                        loaded.includes(new_file) ||
+                        skipped_files.includes(new_file)
+                    ) {
+                        // file loaded or skipped
+                        continue;
+                    }
+
+                    load_new = true;
+
+                    if (await this.#fs.has(new_file)) {
+                        // load file, it changes this.#files_by_name and causes calling
+                        // this.schematics() will return a new result.
+                        await this.#load_file(new_file);
+                    } else {
+                        // skip non-existent files to allow loading an incomplete schematics
+                        skipped_files.push(new_file);
+                        log.warn(`file "${new_file}" is not existed, skip it.`);
                     }
                 }
             }
-            await Promise.all(promises);
         }
 
         this.#determine_schematic_hierarchy();
@@ -251,6 +275,10 @@ export class Project extends EventTarget implements IDisposable {
 
         // Finally, if no root schematic was found, just use the first one we saw.
         this.#root_schematic_page = first(this.#pages_by_path.values());
+
+        if (!this.#root_schematic_page) {
+            log.error("No vaild root schematic was found.");
+        }
     }
 
     public *files() {

src/kicanvas/services/codeberg-vfs.ts

@@ -7,53 +7,50 @@ import { base64_decode } from "../../base/base64";
 import { initiate_download } from "../../base/dom/download";
 import { extension } from "../../base/paths";
 import { Codeberg, GetBlobResponse, RepoContentResponse } from "./codeberg";
-import { VirtualFileSystem } from "./vfs";
+import { type IFileSystem } from "./vfs";
 
-export class CodebergFileSystem extends VirtualFileSystem {
+export class CodebergFileSystem implements IFileSystem {
     static readonly kicad_extensions = ["kicad_pcb", "kicad_pro", "kicad_sch"];
 
-    constructor(private files_to_urls: Map<string, URL>) {
-        super();
-    }
+    async setup() {}
+    constructor(private files_to_urls: Map<string, URL>) {}
 
     public static async fromURLs(
-        ...urls: (string | URL)[]
+        url: string | URL,
     ): Promise<CodebergFileSystem | null> {
         const files_to_urls = new Map<string, URL>();
 
-        for (const url of urls) {
-            const info = Codeberg.parse_url(url);
-            if (!info) {
-                continue;
-            }
-
-            // API:
-            // https://codeberg.org/api/swagger#/repository/repoGetContents
-            const api_url = `repos/${info.owner}/${info.repo}/contents/${info.path}`;
+        const info = Codeberg.parse_url(url);
+        if (!info) {
+            return null;
+        }
 
-            let files = await Codeberg.request_json<
-                RepoContentResponse | RepoContentResponse[]
-            >(api_url);
+        // API:
+        // https://codeberg.org/api/swagger#/repository/repoGetContents
+        const api_url = `repos/${info.owner}/${info.repo}/contents/${info.path}`;
 
-            if (!Array.isArray(files)) {
-                files = [files];
-            }
+        let files = await Codeberg.request_json<
+            RepoContentResponse | RepoContentResponse[]
+        >(api_url);
 
-            for (const file of files) {
-                if (!file.name || !file.git_url || file.type !== "file") {
-                    continue;
-                }
+        if (!Array.isArray(files)) {
+            files = [files];
+        }
 
-                if (
-                    !CodebergFileSystem.kicad_extensions.includes(
-                        extension(file.name),
-                    )
-                ) {
-                    continue;
-                }
+        for (const file of files) {
+            if (!file.name || !file.git_url || file.type !== "file") {
+                continue;
+            }
 
-                files_to_urls.set(file.name, new URL(file.git_url));
+            if (
+                !CodebergFileSystem.kicad_extensions.includes(
+                    extension(file.name),
+                )
+            ) {
+                continue;
             }
+
+            files_to_urls.set(file.name, new URL(file.git_url));
         }
 
         if (files_to_urls.size == 0) {
@@ -64,11 +61,11 @@ export class CodebergFileSystem extends VirtualFileSystem {
         return new CodebergFileSystem(files_to_urls);
     }
 
-    override *list(): Generator<string> {
+    *list(): Generator<string> {
         yield* this.files_to_urls.keys();
     }
 
-    override async get(name: string): Promise<File> {
+    async get(name: string) {
         const url = this.files_to_urls.get(name);
         if (!url) {
             throw new Error(`File ${name} not found.`);
@@ -87,11 +84,11 @@ export class CodebergFileSystem extends VirtualFileSystem {
         return file;
     }
 
-    override has(name: string): Promise<boolean> {
+    async has(name: string) {
         return Promise.resolve(this.files_to_urls.has(name));
     }
 
-    override async download(name: string): Promise<void> {
+    async download(name: string) {
         initiate_download(await this.get(name));
     }
 }