fix: load sheets in subdirectory from Codeberg VFS (#189)

Author: XiangYyang

src/kicanvas/project.ts

@@ -54,42 +54,39 @@ export class Project extends EventTarget implements IDisposable {
 
         // 'Recursively' resolve all schematics until none remain
         let load_new = true;
-        const skipped_files: string[] = [];
+        const loaded_files: string[] = [];
         while (load_new) {
             load_new = false;
 
-            const loaded_file = Array.from(this.schematics());
-            for (const sch of loaded_file) {
+            const pending = [];
+            for (const sch of this.schematics()) {
                 const base_dir = dirname(sch.filename);
                 for (const subsch of sch.sheets) {
                     if (!subsch.sheetfile) {
                         continue;
                     }
 
                     const new_file = normalize_join(base_dir, subsch.sheetfile);
-
-                    const loaded = loaded_file.map((s) => s.filename);
-                    if (
-                        loaded.includes(new_file) ||
-                        skipped_files.includes(new_file)
-                    ) {
+                    if (loaded_files.includes(new_file)) {
                         // file loaded or skipped
                         continue;
                     }
 
                     load_new = true;
+                    loaded_files.push(new_file);
 
                     if (await this.#fs.has(new_file)) {
                         // load file, it changes this.#files_by_name and causes calling
                         // this.schematics() will return a new result.
-                        await this.#load_file(new_file);
+                        pending.push(this.#load_file(new_file));
                     } else {
                         // skip non-existent files to allow loading an incomplete schematics
-                        skipped_files.push(new_file);
                         log.warn(`file "${new_file}" is not existed, skip it.`);
                     }
                 }
             }
+
+            await Promise.all(pending);
         }
 
         this.#determine_schematic_hierarchy();

src/kicanvas/services/codeberg-vfs.ts

@@ -4,30 +4,52 @@
     Full text available at: https://opensource.org/licenses/MIT
 */
 import { base64_decode } from "../../base/base64";
-import { initiate_download } from "../../base/dom/download";
-import { extension } from "../../base/paths";
-import { Codeberg, GetBlobResponse, RepoContentResponse } from "./codeberg";
-import { type IFileSystem } from "./vfs";
+import { based_on, normalize_join } from "../../base/paths";
+import {
+    Codeberg,
+    CodebergRepoInfo,
+    GetBlobResponse,
+    RepoContentResponse,
+} from "./codeberg";
+import { FileEntry, FileSystemBase } from "./vfs";
+
+export class CodebergFileSystem extends FileSystemBase {
+    private download_urls: Map<string, URL> = new Map();
+
+    constructor(private repo: CodebergRepoInfo) {
+        super();
+    }
 
-export class CodebergFileSystem implements IFileSystem {
-    static readonly kicad_extensions = ["kicad_pcb", "kicad_pro", "kicad_sch"];
+    async load_file(path: string): Promise<File> {
+        const url = this.download_urls.get(path);
+        if (!url) {
+            throw new Error(`File ${path} not found.`);
+        }
 
-    async setup() {}
-    constructor(private files_to_urls: Map<string, URL>) {}
+        // API: https://codeberg.org/api/swagger#/repository/GetBlob
+        const blob = await Codeberg.request_json<GetBlobResponse>(url.pathname);
 
-    public static async fromURLs(
-        url: string | URL,
-    ): Promise<CodebergFileSystem | null> {
-        const files_to_urls = new Map<string, URL>();
+        if (blob.content.length === 0) {
+            throw new Error(`Blob returns empty when loading file ${path}`);
+        }
 
-        const info = Codeberg.parse_url(url);
-        if (!info) {
-            return null;
+        if (blob.encoding !== "base64") {
+            throw new Error(`Unsupported encoding: ${blob.encoding}`);
         }
 
+        const content = base64_decode(blob.content);
+        const file = new File([content], path);
+
+        return file;
+    }
+
+    async enumerate(cur_path: string): Promise<FileEntry[]> {
         // API:
         // https://codeberg.org/api/swagger#/repository/repoGetContents
-        const api_url = `repos/${info.owner}/${info.repo}/contents/${info.path}`;
+        const info = this.repo;
+        const base_dir = info.path;
+        const base_api = `repos/${info.owner}/${info.repo}/contents`;
+        const api_url = normalize_join(base_api, base_dir, cur_path);
 
         let files = await Codeberg.request_json<
             RepoContentResponse | RepoContentResponse[]
@@ -37,58 +59,39 @@ export class CodebergFileSystem implements IFileSystem {
             files = [files];
         }
 
-        for (const file of files) {
-            if (!file.name || !file.git_url || file.type !== "file") {
-                continue;
-            }
-
+        const result: FileEntry[] = [];
+        for (const it of files) {
             if (
-                !CodebergFileSystem.kicad_extensions.includes(
-                    extension(file.name),
-                )
+                it.type === "file" &&
+                CodebergFileSystem.is_kicad_file(it.name)
             ) {
-                continue;
+                const file_path = based_on(base_dir, it.path);
+
+                this.download_urls.set(file_path, new URL(it.git_url));
+
+                result.push({
+                    type: "file",
+                    path: file_path,
+                });
+            } else if (it.type === "dir") {
+                result.push({
+                    type: "directory",
+                    path: based_on(base_dir, it.path),
+                });
             }
-
-            files_to_urls.set(file.name, new URL(file.git_url));
-        }
-
-        if (files_to_urls.size == 0) {
-            // no valid URL and files, return null.
-            return null;
         }
 
-        return new CodebergFileSystem(files_to_urls);
-    }
-
-    *list(): Generator<string> {
-        yield* this.files_to_urls.keys();
+        return result;
     }
 
-    async get(name: string) {
-        const url = this.files_to_urls.get(name);
-        if (!url) {
-            throw new Error(`File ${name} not found.`);
-        }
-
-        // API: https://codeberg.org/api/swagger#/repository/GetBlob
-        const blob = await Codeberg.request_json<GetBlobResponse>(url.pathname);
-
-        if (blob.encoding !== "base64") {
-            throw new Error(`Unsupported encoding: ${blob.encoding}`);
+    public static async fromURLs(
+        url: string | URL,
+    ): Promise<CodebergFileSystem | null> {
+        const info = Codeberg.parse_url(url);
+        if (!info) {
+            return null;
         }
 
-        const content = base64_decode(blob.content);
-        const file = new File([content], name);
-
-        return file;
-    }
-
-    async has(name: string) {
-        return Promise.resolve(this.files_to_urls.has(name));
-    }
-
-    async download(name: string) {
-        initiate_download(await this.get(name));
+        return new CodebergFileSystem(info);
     }
 }

src/kicanvas/services/codeberg.ts

@@ -22,13 +22,20 @@ export class GetBlobResponse {
     url: string;
 }
 
+export class CodebergRepoInfo {
+    owner: string;
+    repo: string;
+    ref: string;
+    path: string;
+}
+
 export class Codeberg {
     static readonly host_name = "codeberg.org";
     static readonly html_base_url = "https://codeberg.org/";
     static readonly base_url = "https://codeberg.org/api/v1/";
     static readonly accept_header = "application/json";
 
-    static parse_url(url: string | URL) {
+    static parse_url(url: string | URL): CodebergRepoInfo | null {
         url = new URL(url, Codeberg.html_base_url);
         if (url.hostname !== Codeberg.host_name) {
             return null;
@@ -41,6 +48,9 @@ export class Codeberg {
         }
 
         const [, owner, repo, ...parts] = path_parts;
+        if (!owner || !repo) {
+            return null;
+        }
 
         let ref, path;
         if (parts.length > 0) {

src/kicanvas/services/github-vfs.ts

@@ -42,7 +42,7 @@ export class GitHubFileSystem extends FileSystemBase {
         }
     }
 
-    override async load_file(path: string): Promise<File> {
+    async load_file(path: string): Promise<File> {
         const download_url = this.download_urls.get(path);
         if (!download_url) {
             throw new Error(`File ${path} not found!`);
@@ -51,7 +51,7 @@ export class GitHubFileSystem extends FileSystemBase {
         return await gh_user_content.get(download_url);
     }
 
-    override async enumerate(cur_dir: string): Promise<FileEntry[]> {
+    async enumerate(cur_dir: string): Promise<FileEntry[]> {
         if (this.single_file) {
             // single file, return all files directly
             return Array.from(this.download_urls.keys()).map((v) => ({

src/kicanvas/services/vfs.ts

@@ -24,7 +24,7 @@ export interface IFileSystem {
     /** List all files */
     list(): Generator<string>;
 
-    /** Initialize it */
+    /** Initialize it. Call this function befoce using VFS */
     setup(): Promise<void>;
 
     /** Get a file */