Skip to content

Commit b6cb4c8

Browse files
adamnovakadamnovak
committed
Put prepare_vg.sh under CI
1 parent 7d5d5c3 commit b6cb4c8

File tree

5 files changed

+165
-97
lines changed

5 files changed

+165
-97
lines changed

‎scripts/prepare_vg.sh

+6-7
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,10 @@ fi
1010
echo "${1}"
1111
echo "${1%.vg}"
1212

13-
if [[ -e "${1%.vg}.vcf.gz" && -e "${1%.vg}.vcf.gz.tbi" ]]
14-
then
15-
echo "Generating xg index and gbwt index from vg file and VCF"
16-
vg index "${1}" -v "${1%.vg}.vcf.gz" -x "${1}.xg" --gbwt-name "${1}.gbwt"
17-
else
18-
echo "Generating xg index from vg file"
19-
vg index "${1}" -x "${1}.xg"
13+
echo "Generating xg index ${1}.xg from vg file"
14+
vg convert "${1}" -x >"${1}.xg"
15+
16+
if [[ -e "${1%.vg}.vcf.gz" && -e "${1%.vg}.vcf.gz.tbi" ]] ; then
17+
echo "Generating gbwt index ${1}.gbwt from vg file and VCF"
18+
vg gbwt -x "${1}" -v "${1%.vg}.vcf.gz" -o "${1}.gbwt"
2019
fi

‎src/errors.mjs

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/// errors.mjs: Error type definitions for the tube map server
2+
3+
// We can throw this error to trigger our error handling code instead of
4+
// Express's default. It covers input validation failures, and vaguely-expected
5+
// server-side errors we want to report in a controlled way (because they could
6+
// be caused by bad user input to vg).
7+
export class TubeMapError extends Error {
8+
constructor(message) {
9+
super(message);
10+
}
11+
}
12+
13+
// We can throw this error to make Express respond with a bad request error
14+
// message. We should throw it whenever we detect that user input is
15+
// unacceptable.
16+
export class BadRequestError extends TubeMapError {
17+
constructor(message) {
18+
super(message);
19+
this.status = 400;
20+
}
21+
}
22+
23+
// We can throw this error to make Express respond with an internal server
24+
// error message
25+
export class InternalServerError extends TubeMapError {
26+
constructor(message) {
27+
super(message);
28+
this.status = 500;
29+
}
30+
}
31+
32+
// We can throw this error to make Express respond with an internal server
33+
// error message about vg.
34+
export class VgExecutionError extends InternalServerError {
35+
constructor(message) {
36+
super(message);
37+
}
38+
}

‎src/scripts.test.js

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Tests for data imp[ort scripts, to make sure vg still supports them.
2+
3+
import "./config-server.mjs";
4+
5+
import { find_vg } from "./vg.mjs";
6+
7+
import { mkdtemp, rm, cp, open, access } from 'node:fs/promises';
8+
import { join } from 'node:path';
9+
import { tmpdir } from 'node:os';
10+
import child_process from 'node:child_process';
11+
import { promisify } from 'node:util';
12+
13+
// This runs a command string and returns a promise for {stdout, stderr} that
14+
// rejects if the command fails.
15+
const exec = promisify(child_process.exec);
16+
17+
// This takes a command file and an array of arguments and returns a promise
18+
// for {stdout, stderr} that rejects if the command fails.
19+
const execFile = promisify(child_process.execFile);
20+
21+
const EXAMPLE_DATA = join(__dirname, "..", "exampleData");
22+
const SCRIPTS = join(__dirname, "..", "scripts");
23+
24+
// We set this to a fresh empty directory for each test.
25+
let workDir = null;
26+
27+
beforeEach(async () => {
28+
// Each test gets a fresh directory
29+
workDir = await mkdtemp(join(tmpdir(), 'test-'));
30+
});
31+
32+
afterEach(async () => {
33+
if (workDir) {
34+
rm(workDir, {force: true, recursive: true});
35+
}
36+
});
37+
38+
it("can run prepare_vg.sh", async () => {
39+
for (let filename of ["x.fa", "x.vcf.gz", "x.vcf.gz.tbi"]) {
40+
// Get all the input data
41+
await cp(join(EXAMPLE_DATA, filename), join(workDir, filename));
42+
}
43+
44+
// Build the graph
45+
const vgBuffer = (await execFile(find_vg(), ["construct", "-r", join(workDir, "x.fa"), "-v", join(workDir, "x.vcf.gz"), "-a"], {encoding: "buffer"})).stdout
46+
const graphPath = join(workDir, "x.vg");
47+
console.log("Save graph to " + graphPath);
48+
let file = await open(graphPath, "w");
49+
await file.writeFile(vgBuffer);
50+
await file.close();
51+
52+
// Do the call under test
53+
// We can't use expect here because await expect(...).resolves doesn't actually detect rejections.
54+
console.log("Call script");
55+
let {stdout, stderr} = await execFile(join(SCRIPTS, "prepare_vg.sh"), [join(workDir, "x.vg")]);
56+
console.log("stdout:", stdout);
57+
console.log("stderr:", stderr);
58+
await access(join(workDir, "x.vg.xg"));
59+
await access(join(workDir, "x.vg.gbwt"));
60+
});
61+
62+

‎src/server.mjs

+3-90
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66

77
import "./config-server.mjs";
88
import { config } from "./config-global.mjs";
9+
import { find_vg } from "./vg.mjs"
10+
import { TubeMapError, BadRequestError, InternalServerError, VgExecutionError } from "./errors.mjs"
11+
912
import assert from "assert";
1013
import { spawn } from "child_process";
1114
import express from "express";
@@ -35,65 +38,12 @@ import sanitize from "sanitize-filename";
3538
import { createHash } from "node:crypto";
3639
import cron from "node-cron";
3740
import { RWLock, combine } from "readers-writer-lock";
38-
import which from "which";
3941

4042
if (process.env.NODE_ENV !== "production") {
4143
// Load any .env file config
4244
dotenv.config();
4345
}
4446

45-
/// Return the command string to execute to run vg.
46-
/// Checks config.vgPath.
47-
/// An entry of "" in config.vgPath means to check PATH.
48-
function find_vg() {
49-
if (find_vg.found_vg !== null) {
50-
// Cache the answer and don't re-check all the time.
51-
// Nobody shoudl be deleting vg.
52-
return find_vg.found_vg;
53-
}
54-
for (let prefix of config.vgPath) {
55-
if (prefix === "") {
56-
// Empty string has special meaning of "use PATH".
57-
console.log("Check for vg on PATH");
58-
try {
59-
find_vg.found_vg = which.sync("vg");
60-
console.log("Found vg at:", find_vg.found_vg);
61-
return find_vg.found_vg;
62-
} catch (e) {
63-
// vg is not on PATH
64-
continue;
65-
}
66-
}
67-
if (prefix.length > 0 && prefix[prefix.length - 1] !== "/") {
68-
// Add trailing slash
69-
prefix = prefix + "/";
70-
}
71-
let vg_filename = prefix + "vg";
72-
console.log("Check for vg at:", vg_filename);
73-
if (fs.existsSync(vg_filename)) {
74-
if (!fs.statSync(vg_filename).isFile()) {
75-
// This is a directory or something, not a binary we can run.
76-
continue;
77-
}
78-
try {
79-
// Pretend we will execute it
80-
fs.accessSync(vg_filename, fs.constants.X_OK)
81-
} catch (e) {
82-
// Not executable
83-
continue;
84-
}
85-
// If we get here it is executable.
86-
find_vg.found_vg = vg_filename;
87-
console.log("Found vg at:", find_vg.found_vg);
88-
return find_vg.found_vg;
89-
}
90-
}
91-
// If we get here we don't see vg at all.
92-
throw new InternalServerError("The vg command was not found. Install vg to use the Sequence Tube Map: https://github.com/vgteam/vg?tab=readme-ov-file#installation");
93-
}
94-
find_vg.found_vg = null;
95-
96-
9747
const MOUNTED_DATA_PATH = config.dataPath;
9848
const INTERNAL_DATA_PATH = config.internalDataPath;
9949
// THis is where we will store uploaded files
@@ -1086,43 +1036,6 @@ function organizePathsTargetFirst(region, pathList) {
10861036
}
10871037
}
10881038

1089-
// We can throw this error to trigger our error handling code instead of
1090-
// Express's default. It covers input validation failures, and vaguely-expected
1091-
// server-side errors we want to report in a controlled way (because they could
1092-
// be caused by bad user input to vg).
1093-
class TubeMapError extends Error {
1094-
constructor(message) {
1095-
super(message);
1096-
}
1097-
}
1098-
1099-
// We can throw this error to make Express respond with a bad request error
1100-
// message. We should throw it whenever we detect that user input is
1101-
// unacceptable.
1102-
class BadRequestError extends TubeMapError {
1103-
constructor(message) {
1104-
super(message);
1105-
this.status = 400;
1106-
}
1107-
}
1108-
1109-
// We can throw this error to make Express respond with an internal server
1110-
// error message
1111-
class InternalServerError extends TubeMapError {
1112-
constructor(message) {
1113-
super(message);
1114-
this.status = 500;
1115-
}
1116-
}
1117-
1118-
// We can throw this error to make Express respond with an internal server
1119-
// error message about vg.
1120-
class VgExecutionError extends InternalServerError {
1121-
constructor(message) {
1122-
super(message);
1123-
}
1124-
}
1125-
11261039
// We can use this middleware to ensure that errors we synchronously throw or
11271040
// next(err) will be sent along to the user. It does *not* happen on API
11281041
// endpoint promise rejections until Express 5.

‎src/vg.mjs

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import { config } from "./config-global.mjs";
2+
import { InternalServerError } from "./errors.mjs";
3+
4+
import fs from "fs-extra";
5+
import which from "which";
6+
7+
/// Return the command string to execute to run vg.
8+
/// Checks config.vgPath.
9+
/// An entry of "" in config.vgPath means to check PATH.
10+
export function find_vg() {
11+
if (find_vg.found_vg !== null) {
12+
// Cache the answer and don't re-check all the time.
13+
// Nobody should be deleting vg.
14+
return find_vg.found_vg;
15+
}
16+
for (let prefix of config.vgPath) {
17+
if (prefix === "") {
18+
// Empty string has special meaning of "use PATH".
19+
console.log("Check for vg on PATH");
20+
try {
21+
find_vg.found_vg = which.sync("vg");
22+
console.log("Found vg at:", find_vg.found_vg);
23+
return find_vg.found_vg;
24+
} catch (e) {
25+
// vg is not on PATH
26+
continue;
27+
}
28+
}
29+
if (prefix.length > 0 && prefix[prefix.length - 1] !== "/") {
30+
// Add trailing slash
31+
prefix = prefix + "/";
32+
}
33+
let vg_filename = prefix + "vg";
34+
console.log("Check for vg at:", vg_filename);
35+
if (fs.existsSync(vg_filename)) {
36+
if (!fs.statSync(vg_filename).isFile()) {
37+
// This is a directory or something, not a binary we can run.
38+
continue;
39+
}
40+
try {
41+
// Pretend we will execute it
42+
fs.accessSync(vg_filename, fs.constants.X_OK)
43+
} catch (e) {
44+
// Not executable
45+
continue;
46+
}
47+
// If we get here it is executable.
48+
find_vg.found_vg = vg_filename;
49+
console.log("Found vg at:", find_vg.found_vg);
50+
return find_vg.found_vg;
51+
}
52+
}
53+
// If we get here we don't see vg at all.
54+
throw new InternalServerError("The vg command was not found. Install vg to use the Sequence Tube Map: https://github.com/vgteam/vg?tab=readme-ov-file#installation");
55+
}
56+
find_vg.found_vg = null;

0 commit comments

Comments
 (0)