Put prepare_vg.sh under CI

adamnovak · adamnovak · commit b6cb4c84b0f2 · 2025-02-11T11:07:16.000-05:00
diff --git a/scripts/prepare_vg.sh b/scripts/prepare_vg.sh
@@ -10,11 +10,10 @@ fi
 echo "${1}"
 echo "${1%.vg}"
 
-if [[ -e "${1%.vg}.vcf.gz" && -e "${1%.vg}.vcf.gz.tbi" ]]
-then
-    echo "Generating xg index and gbwt index from vg file and VCF"
-    vg index "${1}" -v "${1%.vg}.vcf.gz" -x "${1}.xg" --gbwt-name "${1}.gbwt"
-else
-    echo "Generating xg index from vg file"
-    vg index "${1}" -x "${1}.xg"
+echo "Generating xg index ${1}.xg from vg file"
+vg convert "${1}" -x >"${1}.xg"
+
+if [[ -e "${1%.vg}.vcf.gz" && -e "${1%.vg}.vcf.gz.tbi" ]] ; then
+    echo "Generating gbwt index ${1}.gbwt from vg file and VCF"
+    vg gbwt -x "${1}" -v "${1%.vg}.vcf.gz" -o "${1}.gbwt"
 fi
diff --git a/src/errors.mjs b/src/errors.mjs
@@ -0,0 +1,38 @@
+/// errors.mjs: Error type definitions for the tube map server
+
+// We can throw this error to trigger our error handling code instead of
+// Express's default. It covers input validation failures, and vaguely-expected
+// server-side errors we want to report in a controlled way (because they could
+// be caused by bad user input to vg).
+export class TubeMapError extends Error {
+  constructor(message) {
+    super(message);
+  }
+}
+
+// We can throw this error to make Express respond with a bad request error
+// message. We should throw it whenever we detect that user input is
+// unacceptable.
+export class BadRequestError extends TubeMapError {
+  constructor(message) {
+    super(message);
+    this.status = 400;
+  }
+}
+
+// We can throw this error to make Express respond with an internal server
+// error message
+export class InternalServerError extends TubeMapError {
+  constructor(message) {
+    super(message);
+    this.status = 500;
+  }
+}
+
+// We can throw this error to make Express respond with an internal server
+// error message about vg.
+export class VgExecutionError extends InternalServerError {
+  constructor(message) {
+    super(message);
+  }
+}
diff --git a/src/scripts.test.js b/src/scripts.test.js
@@ -0,0 +1,62 @@
+// Tests for data imp[ort scripts, to make sure vg still supports them.
+
+import "./config-server.mjs";
+
+import { find_vg } from "./vg.mjs";
+
+import { mkdtemp, rm, cp, open, access } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import child_process from 'node:child_process';
+import { promisify } from 'node:util';
+
+// This runs a command string and returns a promise for {stdout, stderr} that
+// rejects if the command fails.
+const exec = promisify(child_process.exec);
+
+// This takes a command file and an array of arguments and returns a promise
+// for {stdout, stderr} that rejects if the command fails.
+const execFile = promisify(child_process.execFile);
+
+const EXAMPLE_DATA = join(__dirname, "..", "exampleData");
+const SCRIPTS = join(__dirname, "..", "scripts");
+
+// We set this to a fresh empty directory for each test.
+let workDir = null;
+
+beforeEach(async () => {
+  // Each test gets a fresh directory
+  workDir = await mkdtemp(join(tmpdir(), 'test-'));
+});
+
+afterEach(async () => {
+  if (workDir) {
+    rm(workDir, {force: true, recursive: true});
+  }
+});
+
+it("can run prepare_vg.sh", async () => {
+  for (let filename of ["x.fa", "x.vcf.gz", "x.vcf.gz.tbi"]) {
+    // Get all the input data
+    await cp(join(EXAMPLE_DATA, filename), join(workDir, filename));
+  }
+  
+  // Build the graph
+  const vgBuffer = (await execFile(find_vg(), ["construct", "-r", join(workDir, "x.fa"), "-v", join(workDir, "x.vcf.gz"), "-a"], {encoding: "buffer"})).stdout
+  const graphPath = join(workDir, "x.vg");
+  console.log("Save graph to " + graphPath);
+  let file = await open(graphPath, "w");
+  await file.writeFile(vgBuffer);
+  await file.close();
+
+  // Do the call under test
+  // We can't use expect here because await expect(...).resolves doesn't actually detect rejections.
+  console.log("Call script");
+  let {stdout, stderr} = await execFile(join(SCRIPTS, "prepare_vg.sh"), [join(workDir, "x.vg")]);
+  console.log("stdout:", stdout);
+  console.log("stderr:", stderr);
+  await access(join(workDir, "x.vg.xg"));
+  await access(join(workDir, "x.vg.gbwt"));
+});
+
+
diff --git a/src/server.mjs b/src/server.mjs
@@ -6,6 +6,9 @@
 
 import "./config-server.mjs";
 import { config } from "./config-global.mjs";
+import { find_vg } from "./vg.mjs"
+import { TubeMapError, BadRequestError, InternalServerError, VgExecutionError } from "./errors.mjs"
+
 import assert from "assert";
 import { spawn } from "child_process";
 import express from "express";
@@ -35,65 +38,12 @@ import sanitize from "sanitize-filename";
 import { createHash } from "node:crypto";
 import cron from "node-cron";
 import { RWLock, combine } from "readers-writer-lock";
-import which from "which";
 
 if (process.env.NODE_ENV !== "production") {
   // Load any .env file config
   dotenv.config();
 }
 
-/// Return the command string to execute to run vg.
-/// Checks config.vgPath.
-/// An entry of "" in config.vgPath means to check PATH.
-function find_vg() {
-  if (find_vg.found_vg !== null) {
-    // Cache the answer and don't re-check all the time.
-    // Nobody shoudl be deleting vg.
-    return find_vg.found_vg;
-  }
-  for (let prefix of config.vgPath) {
-    if (prefix === "") {
-      // Empty string has special meaning of "use PATH".
-      console.log("Check for vg on PATH");
-      try {
-        find_vg.found_vg = which.sync("vg");
-        console.log("Found vg at:", find_vg.found_vg);
-        return find_vg.found_vg;
-      } catch (e) {
-        // vg is not on PATH
-        continue;
-      }
-    }
-    if (prefix.length > 0 && prefix[prefix.length - 1] !== "/") {
-      // Add trailing slash
-      prefix = prefix + "/";
-    }
-    let vg_filename = prefix + "vg";
-    console.log("Check for vg at:", vg_filename);
-    if (fs.existsSync(vg_filename)) {
-      if (!fs.statSync(vg_filename).isFile()) {
-        // This is a directory or something, not a binary we can run.
-        continue;
-      }
-      try {
-        // Pretend we will execute it
-        fs.accessSync(vg_filename, fs.constants.X_OK)
-      } catch (e) {
-        // Not executable
-        continue;
-      }
-      // If we get here it is executable.
-      find_vg.found_vg = vg_filename;
-      console.log("Found vg at:", find_vg.found_vg);
-      return find_vg.found_vg;
-    }
-  }
-  // If we get here we don't see vg at all.
-  throw new InternalServerError("The vg command was not found. Install vg to use the Sequence Tube Map: https://github.com/vgteam/vg?tab=readme-ov-file#installation");
-}
-find_vg.found_vg = null;
-
-
 const MOUNTED_DATA_PATH = config.dataPath;
 const INTERNAL_DATA_PATH = config.internalDataPath;
 // THis is where we will store uploaded files
@@ -1086,43 +1036,6 @@ function organizePathsTargetFirst(region, pathList) {
   }
 }
 
-// We can throw this error to trigger our error handling code instead of
-// Express's default. It covers input validation failures, and vaguely-expected
-// server-side errors we want to report in a controlled way (because they could
-// be caused by bad user input to vg).
-class TubeMapError extends Error {
-  constructor(message) {
-    super(message);
-  }
-}
-
-// We can throw this error to make Express respond with a bad request error
-// message. We should throw it whenever we detect that user input is
-// unacceptable.
-class BadRequestError extends TubeMapError {
-  constructor(message) {
-    super(message);
-    this.status = 400;
-  }
-}
-
-// We can throw this error to make Express respond with an internal server
-// error message
-class InternalServerError extends TubeMapError {
-  constructor(message) {
-    super(message);
-    this.status = 500;
-  }
-}
-
-// We can throw this error to make Express respond with an internal server
-// error message about vg.
-class VgExecutionError extends InternalServerError {
-  constructor(message) {
-    super(message);
-  }
-}
-
 // We can use this middleware to ensure that errors we synchronously throw or
 // next(err) will be sent along to the user. It does *not* happen on API
 // endpoint promise rejections until Express 5.
diff --git a/src/vg.mjs b/src/vg.mjs
@@ -0,0 +1,56 @@
+import { config } from "./config-global.mjs";
+import { InternalServerError } from "./errors.mjs";
+
+import fs from "fs-extra";
+import which from "which";
+
+/// Return the command string to execute to run vg.
+/// Checks config.vgPath.
+/// An entry of "" in config.vgPath means to check PATH.
+export function find_vg() {
+  if (find_vg.found_vg !== null) {
+    // Cache the answer and don't re-check all the time.
+    // Nobody should be deleting vg.
+    return find_vg.found_vg;
+  }
+  for (let prefix of config.vgPath) {
+    if (prefix === "") {
+      // Empty string has special meaning of "use PATH".
+      console.log("Check for vg on PATH");
+      try {
+        find_vg.found_vg = which.sync("vg");
+        console.log("Found vg at:", find_vg.found_vg);
+        return find_vg.found_vg;
+      } catch (e) {
+        // vg is not on PATH
+        continue;
+      }
+    }
+    if (prefix.length > 0 && prefix[prefix.length - 1] !== "/") {
+      // Add trailing slash
+      prefix = prefix + "/";
+    }
+    let vg_filename = prefix + "vg";
+    console.log("Check for vg at:", vg_filename);
+    if (fs.existsSync(vg_filename)) {
+      if (!fs.statSync(vg_filename).isFile()) {
+        // This is a directory or something, not a binary we can run.
+        continue;
+      }
+      try {
+        // Pretend we will execute it
+        fs.accessSync(vg_filename, fs.constants.X_OK)
+      } catch (e) {
+        // Not executable
+        continue;
+      }
+      // If we get here it is executable.
+      find_vg.found_vg = vg_filename;
+      console.log("Found vg at:", find_vg.found_vg);
+      return find_vg.found_vg;
+    }
+  }
+  // If we get here we don't see vg at all.
+  throw new InternalServerError("The vg command was not found. Install vg to use the Sequence Tube Map: https://github.com/vgteam/vg?tab=readme-ov-file#installation");
+}
+find_vg.found_vg = null;
