Make server contemplate but not do GAF indexing, ban GAF uploads as unsupported, and allow upload errors to reach the user in an ugly way

Author: adamnovak

docker/config.json

@@ -84,7 +84,7 @@
   "fileTypeToExtensions": {
     "graph": ".xg,.vg,.hg,.gbz,.pg,.db",
     "haplotype": ".gbwt,.gbz",
-    "read": ".gam,gaf.gz"
+    "read": ".gam"
   },
 
   "MAXUPLOADSIZE": 5242880,

exampleData/cactus-NA12879-small.gaf

@@ -111,7 +111,7 @@ export class ServerAPI extends APIInterface {
     // Make sure server can identify a Read file
     formData.append("fileType", fileType);
 
-    return new Promise((resolve, reject) => {
+    return await new Promise((resolve, reject) => {
       const xhr = new XMLHttpRequest();
       xhr.responseType = "json";
       xhr.onreadystatechange = () => {
@@ -121,21 +121,28 @@ export class ServerAPI extends APIInterface {
           reject(new Error("Upload aborted"));
           return;
         }
-
+        
         if (xhr.readyState === 4) {
           if (xhr.status === 200 && xhr.response.path) {
             // Every thing ok, file uploaded, and we got a path.
             resolve(xhr.response.path);
           } else {
             // Something weird happened.
-            reject(
-              new Error(
-                "Failed to upload file: status " +
-                  xhr.status +
-                  " and response: " +
-                  xhr.response
-              )
-            );
+            
+            if (xhr.response.error) {
+              // The server sent us a particular message.
+              reject(new Error(xhr.response.error))
+            } else {
+              // The server did not help us. Compose a message.
+              reject( 
+                new Error(
+                  "Failed to upload file: status " +
+                    xhr.status +
+                    " and response: " +
+                    JSON.stringify(xhr.response)
+                )
+              );
+            }
           }
         }
       };

src/api/ServerAPI.mjs

@@ -912,7 +912,8 @@ class HeaderForm extends Component {
     this.setState({ uploadInProgress: val });
   };
 
-  // Sends uploaded file to server and returns a path to the file
+  // Sends uploaded file to server and returns a path to the file, or raises an exception if the upload fails or is rejected.
+  // If the file upload is canceled, returns nothing.
   handleFileUpload = async (fileType, file) => {
     if (!(this.props.APIInterface instanceof LocalAPI) && file.size > config.MAXUPLOADSIZE) {
       this.showFileSizeAlert();
@@ -927,11 +928,14 @@ class HeaderForm extends Component {
         // Refresh the graphs right away
         this.getMountedFilenames();
       }
+      // TODO: Is only one upload actually ever in progress at a time? Probably
+      // it's possible to have several!!!
       this.setUploadInProgress(false);
       return fileName;
     } catch (e) {
       if (!this.cancelSignal.aborted) {
         // Only pass along errors if we haven't canceled our fetches.
+        this.setUploadInProgress(false);
         throw e;
       }
     }

src/components/HeaderForm.js

@@ -7,10 +7,9 @@ import { Input } from "reactstrap";
 
 /*
  * A selection dropdown component that select files.
- * Expects a file type object in the form of {"name": string, "type": string}
+ * Expects a collection of track objects in the form of {"name": string, "type": string}
  *
- * The handleInputChange function expects to be passed an option type object in the form of
- * {"label": string, "value": file}
+ * The handleInputChange function expects to be passed a bare value.
  *
  * See demo and test file for examples of this component.
  */
@@ -30,11 +29,23 @@ export const TrackFilePicker = ({
   let acceptedExtensions = config.fileTypeToExtensions[fileType];
 
   async function uploadOnChange() {
-    const file = uploadFileInput.current.files[0];
-
-    const completePath = await handleFileUpload(fileType, file);
-    console.log("TrackFilePicker got an upload result:", completePath);
-    handleInputChange(completePath);
+    // TODO: we have a ref here for a reason, but the ref's value can be null
+    // when the upload await finishes. So we capture it to a local.
+    const uploadingInput = uploadFileInput.current;
+    const file = uploadingInput.files[0];
+    
+    try {
+      const completePath = await handleFileUpload(fileType, file);
+      console.log("TrackFilePicker got an upload result:", completePath);
+      // This will be nothing if the upoload was canceled.
+      handleInputChange(completePath);
+    } catch (e) {
+      console.error("TrackFilePicker could not upload: ", e);
+      // TODO: Display error to user in a better way
+      alert(e.toString());
+      // Clear out the uploaded file to show it didn't work.
+      uploadingInput.value = null; 
+    }
   }
 
   function mountedOnChange(option) {

src/components/TrackFilePicker.js

@@ -114,7 +114,7 @@
   "fileTypeToExtensions": {
     "graph": ".xg,.vg,.hg,.gbz,.pg,.db",
     "haplotype": ".gbwt,.gbz",
-    "read": ".gam,.gaf.gz"
+    "read": ".gam"
   },
 
   "MAXUPLOADSIZE": 5242880,

src/config.json

@@ -321,29 +321,66 @@ api.post("/trackFileSubmission", upload.single("trackFile"), (req, res) => {
   }
 });
 
-function indexGamSorted(req, res) {
-  const prefix = req.file.path.substring(0, req.file.path.lastIndexOf("."));
-  const sortedGamFile = fs.createWriteStream(prefix + ".sorted.gam", {
+function indexGamSorted(req, res, next) {
+  let readsPath = req.file.path;
+  let prefix;
+  let sortedSuffix;
+  let indexSuffix;
+  if (readsPath.endsWith(".gaf") || readsPath.endsWith(".gaf.gz")) {
+    throw new BadRequestError(`Server-side sorting and indexing not yet implemented for GAF: ${readsPath}`);
+  } else if (readsPath.endsWith(".gam")) {
+    prefix = readsPath.substring(0, req.file.path.lastIndexOf(".gam"));
+    sortedSuffix = ".sorted.gam";
+    indexSuffix = ".sorted.gam.gai";
+  } else {
+    throw new BadRequestError(`Read file is not a GAF or GAM: ${readsPath}`);
+  }
+
+  const sortedReadsFile = fs.createWriteStream(prefix + sortedSuffix, {
     encoding: "binary",
   });
-  const vgIndexChild = spawn(find_vg(), [
+
+  let vgGamsortParams = [
     "gamsort",
     "-i",
-    prefix + ".sorted.gam.gai",
+    prefix + indexSuffix,
     req.file.path,
-  ]);
+  ];
+  const vgGamsortChild = spawn(find_vg(), vgGamsortParams);
+
+  req.error = Buffer.alloc(0);
+
+  let sentResponse = false;
+
+  vgGamsortChild.on("error", function (err) {
+    console.log(
+      "Error executing " +
+        find_vg() + " " +
+        vgGamsortParams.join(" ") +
+        ": " +
+        err
+    );
+    if (!sentResponse) {
+      sentResponse = true;
+      return next(new VgExecutionError("vg gamsort failed"));
+    }
+  });
 
-  vgIndexChild.stderr.on("data", (data) => {
+  vgGamsortChild.stderr.on("data", (data) => {
     console.log(`err data: ${data}`);
+    req.error += data;
   });
 
-  vgIndexChild.stdout.on("data", function (data) {
-    sortedGamFile.write(data);
+  vgGamsortChild.stdout.on("data", function (data) {
+    sortedReadsFile.write(data);
   });
 
-  vgIndexChild.on("close", () => {
-    sortedGamFile.end();
-    res.json({ path: path.relative(".", prefix + ".sorted.gam") });
+  vgGamsortChild.on("close", () => {
+    sortedReadsFile.end();
+    if (!sentResponse) {
+      sentResponse = true;
+      res.json({ path: path.relative(".", prefix + sortedSuffix) });
+    }
   });
 }
 
@@ -673,20 +710,25 @@ async function getChunkedData(req, res, next) {
     let anyGam = false;
     let anyGaf = false;
     for (const gamFile of gamFiles) {
-      if (!gamFile.endsWith(".gam") && !gamFile.endsWith(".gaf.gz")) {
-        throw new BadRequestError("GAM/GAF file doesn't end in .gam or .gaf.gz: " + gamFile);
+      if (!gamFile.endsWith(".gam") && !gamFile.endsWith(".gaf") && !gamFile.endsWith(".gaf.gz")) {
+        throw new BadRequestError("GAM/GAF file doesn't end in .gam, .gaf, or .gaf.gz: " + gamFile);
       }
       if (!isAllowedPath(gamFile)) {
         throw new BadRequestError("GAM/GAF file path not allowed: " + gamFile);
       }
       if (gamFile.endsWith(".gam")) {
-        // Use a GAM index
+        // Use a GAM
         console.log("pushing gam file", gamFile);
         anyGam = true;
       }
+      if (gamFile.endsWith(".gaf")) {
+        // Use a small GAF without an index
+        console.log("pushing gaf file", gamFile);
+        anyGaf = true;
+      }
       if (gamFile.endsWith(".gaf.gz")) {
         // Use a GAF with index
-        console.log("pushing gaf file", gamFile);
+        console.log("pushing hopefully indexed gaf file", gamFile);
         anyGaf = true;
       }
       vgChunkParams.push("-a", gamFile);
@@ -1615,6 +1657,7 @@ api.get("/getFilenames", (req, res) => {
       if (file.endsWith(".sorted.gam")) {
         result.files.push({ trackFile: clientPath, trackType: "read" });
       }
+      // We don't allow un-sorted-and-indexed plain GAF files here
       if (file.endsWith(".gaf.gz")) {
         result.files.push({"trackFile": file, "trackType": "read"});
       }