Listing files with valid HTTP header (#326)

* HTTP headers for rootdir()
* iterateOnFiles() handles both counting and listing
* Counting files is non-blocking (10 files per loop)
* Add serial log for rootdir()
* Remove extra file
* Formatting changes
* Update version number

Author: je-foster

library.properties

@@ -1,5 +1,5 @@
 name=TankController
-version=22.7.2
+version=22.8.1
 author=Kirt Onthank <[email protected]>, Preston Carman <[email protected]>, James Foster <[email protected]>
 maintainer=James Foster <[email protected]>
 sentence=Software for the Arduino that controls pH and temperature in the Open-Acidification project.

src/Devices/EthernetServer_TC.cpp

@@ -99,7 +99,6 @@ void EthernetServer_TC::apiHandler() {
     } else if (memcmp_P(buffer + 11, F("display"), 7) == 0) {
       display();
     } else if (memcmp_P(buffer + 11, F("rootdir"), 7) == 0) {
-      state = IN_PROGRESS;
       rootdir();
     } else if (memcmp_P(buffer + 11, F("testRead"), 8) == 0) {
       testReadSpeed();
@@ -173,11 +172,32 @@ void writeToClientBufferCallback(char* buffer, bool isFinished) {
   EthernetServer_TC::instance()->writeToClientBuffer(buffer, isFinished);
 }
 
+// Non-member callback wrapper for singleton
+void countFilesCallback(int fileCount) {
+  // Called when the count is complete
+  EthernetServer_TC::instance()->sendHeadersForRootdir(fileCount);
+}
+
 // List the root directory to the client
 void EthernetServer_TC::rootdir() {
   // Call function on SD Card
   // Provide callback to call when writing to the client buffer
-  SD_TC::instance()->listRootToBuffer(writeToClientBufferCallback);
+  if (state != LISTING_FILES) {
+    state = LISTING_FILES;
+    isDoneCountingFiles = false;
+    startTime = millis();
+    serial(F("Preparing list of files in root directory..."));
+  } else {
+    if (isDoneCountingFiles) {
+      SD_TC::instance()->listRootToBuffer(writeToClientBufferCallback);
+    } else {
+#ifndef MOCK_PINS_COUNT
+      SD_TC::instance()->countFiles(countFilesCallback);
+#else
+      countFilesCallback(0);
+#endif
+    }
+  }
 }
 
 // Write to the client buffer
@@ -187,10 +207,25 @@ void EthernetServer_TC::writeToClientBuffer(char* buffer, bool isFinished) {
   if (isFinished) {
     client.write('\r');
     client.write('\n');
+    int endTime = millis();
+    serial(F("...Done sending, time = %i ms"), endTime - startTime);
     state = FINISHED;
   }
 }
 
+void EthernetServer_TC::sendHeadersForRootdir(int fileCount) {
+#ifndef MOCK_PINS_COUNT
+  isDoneCountingFiles = true;
+  serial(F("...%i files..."), fileCount);
+  sendHeadersWithSize((uint32_t)fileCount * 24);  // 24 characters per line
+  state = LISTING_FILES;  // TODO: This is here only because sendHeadersWithSize() changes the state prematurely.
+#else
+  isDoneCountingFiles = true;
+  sendHeadersWithSize((uint32_t)49);
+  state = LISTING_FILES;
+#endif
+}
+
 // Tests speed for reading a file from the SD Card
 // Empirical results show about 1 ms per 512 B
 void EthernetServer_TC::testReadSpeed() {
@@ -255,15 +290,14 @@ void EthernetServer_TC::fileSetup() {
   file = SD_TC::instance()->open(buffer + 4);
   uint32_t size = file.size();
   serial(F("file \"%s\" has a size of %lu"), buffer + 4, size);
-  sendFileHeadersWithSize(size);
+  sendHeadersWithSize(size);
   state = IN_TRANSFER;
   startTime = millis();
   fileContinue();
 }
 
 // Continue file transfer (return value is whether we are finished)
 bool EthernetServer_TC::fileContinue() {
-  client.write(boundary);
   if (file.available()) {
     int readSize = file.read(buffer, sizeof(buffer));  // Flawfinder: ignore
     int writeSize = client.write(buffer, readSize);
@@ -297,8 +331,8 @@ void EthernetServer_TC::loop() {
           state = FINISHED;
         }
         break;
-      case IN_PROGRESS:
-        // In progress (so far only for SD Card)
+      case LISTING_FILES:
+        // Listing files from SD Card
         rootdir();
         break;
       case NOT_CONNECTED:
@@ -370,22 +404,7 @@ void EthernetServer_TC::sendHeadersWithSize(uint32_t size) {
   // blank line indicates end of headers
   client.write('\r');
   client.write('\n');
-  state = FINISHED;
-}
-
-void EthernetServer_TC::sendFileHeadersWithSize(uint32_t size) {
-  snprintf_P(buffer, sizeof(buffer),
-             (PGM_P)F("HTTP/1.1 200 OK\r\n"
-                      "Content-Type: multipart/form-data; boundary=%s\r\n"
-                      "Access-Control-Allow-Origin: *\r\n"
-                      "Content-Length: %lu\r\n"),
-             boundary, (unsigned long)size);
-  client.write(buffer);
-
-  // blank line indicates end of headers
-  client.write('\r');
-  client.write('\n');
-  state = FINISHED;
+  state = FINISHED;  // TODO: Why?! This is awkward when we want to send a body next.
 }
 
 // 303 response

src/Devices/EthernetServer_TC.h

@@ -9,7 +9,7 @@
 #include <Ethernet.h>
 #endif
 
-enum serverState_t { NOT_CONNECTED, READ_REQUEST, GET_REQUEST, POST_REQUEST, IN_PROGRESS, IN_TRANSFER, FINISHED };
+enum serverState_t { NOT_CONNECTED, READ_REQUEST, GET_REQUEST, POST_REQUEST, LISTING_FILES, IN_TRANSFER, FINISHED };
 
 /**
  * EthernetServer_TC provides wrapper for web server for TankController
@@ -29,6 +29,7 @@ class EthernetServer_TC : public EthernetServer {
   }
   void loop();
   void writeToClientBuffer(char*, bool);
+  void sendHeadersForRootdir(int);
 
 private:
   // class variables
@@ -43,12 +44,12 @@ class EthernetServer_TC : public EthernetServer {
   unsigned long connectedAt = 0;
   File file;
   int startTime;
+  bool isDoneCountingFiles = true;  // TODO: Perhaps replace this with a new COUNTING_FILES state
 
   // instance methods: constructor
   EthernetServer_TC(uint16_t port);
   // instance methods: utility
   void sendHeadersWithSize(uint32_t size);
-  void sendFileHeadersWithSize(uint32_t size);
   void sendRedirectHeaders();
   void sendBadRequestHeaders();
   int weekday(int year, int month, int day);

src/Devices/SD_TC.cpp

@@ -84,91 +84,108 @@ bool SD_TC::format() {
   return sd.format();
 }
 
-void SD_TC::listFiles(void (*callWhenFull)(char*, bool), byte tabulation) {
-  // Only called on real device
+bool SD_TC::iterateOnFiles(doOnFile functionName, void* userData) {
 #ifndef MOCK_PINS_COUNT
-  File* parent = &hierarchy[hierarchySize - 1];
-  File current;
-  char fileName[15];
-  char buffer[300];  // Each line shouldn't be more than 30 characters long
-  int linePos = 0;
-  int filesWritten = 0;  // When we hit 10 here we will call buffer and suspend
-
-  while (filesWritten < 10) {
-    if (current.openNext(parent, O_READ)) {
-      if (!current.isHidden()) {
-        current.getName(fileName, sizeof(fileName));
-        for (int i = 0; i < tabulation; i++) {
-          buffer[i] = '\t';
-          ++linePos;
-        }
-        if (current.isDir()) {
-          int bytesWritten = snprintf_P(buffer + linePos, sizeof(buffer) - linePos, (PGM_P)F("%s/\n"), fileName);
-          // "Overwrite" null terminator
-          linePos += bytesWritten;
-          ++filesWritten;
-          // Create a new array with bigger size
-          File* newHierarchy = new File[hierarchySize + 1];
-          memcpy(newHierarchy, hierarchy, sizeof(File) * hierarchySize);
-          delete[] hierarchy;
-          hierarchy = newHierarchy;
-          ++hierarchySize;
-          // Add the current file to the new array
-          hierarchy[hierarchySize - 1] = current;
-          // Now we change parent directory
-          parent = &hierarchy[hierarchySize - 1];
+  // Only called on real device
+  // Returns false only when all files have been iterated on
+  bool flag = true;
+  while (flag) {
+    if (fileStack[fileStackSize].openNext(&fileStack[fileStackSize - 1], O_READ)) {
+      if (!fileStack[fileStackSize].isHidden()) {
+        flag = functionName(&fileStack[fileStackSize], userData);
+        if (fileStack[fileStackSize].isDir()) {
+          // maxDepth was set to 2 in SD_TC.h
+          // So this code is untested
+          if (fileStackSize < maxDepth - 1) {
+            ++fileStackSize;
+          };
         } else {
-          int bytesWritten = snprintf_P(buffer + linePos, sizeof(buffer) - linePos, (PGM_P)F("%s\t%6u bytes\n"),
-                                        fileName, current.fileSize());
-          // "Overwrite" null terminator
-          linePos += bytesWritten;
-          ++filesWritten;
-          // Close current (if it's a file); close current (if it's a directory) as a parent later
-          current.close();
+          // Close current file; directories are closed later
+          fileStack[fileStackSize].close();
         }
       }
     } else {
-      if (hierarchySize == 1) {
-        // We're done listing root
-        parent->close();
-        --hierarchySize;
-        delete[] hierarchy;
-        hierarchy = nullptr;
-        buffer[linePos] = '\0';
-        inProgress = false;
-        callWhenFull(buffer, true);
-        return;
+      // We're done with a directory
+      fileStack[--fileStackSize].close();
+      if (fileStackSize == 0) {
+        return false;  // Done with root---there are no more files
       }
-      // All done with parent, remove directory from hierarchy
-      parent->close();
-      --hierarchySize;
-      parent = &hierarchy[hierarchySize - 1];
     }
   }
-  // We have 10 lines written, so add null terminator
-  buffer[linePos] = '\0';
-  callWhenFull(buffer, false);
+  return true;  // There are (probably) more files remaining
+#else
+  return false;
+#endif
+}
+
+bool SD_TC::incrementFileCount(File* myFile, void* pFileCount) {
+  return ++(*(int*)pFileCount) % 10 != 0;  // Pause after counting 10 files
+}
+
+void SD_TC::countFiles(void (*callWhenFinished)(int)) {
+  if (!inProgress) {
+    const char path[] PROGMEM = "/";
+    fileStack[0] = SD_TC::instance()->open(path);
+    if (!fileStack[0]) {
+      serial(F("SD_TC open() failed"));
+      return;
+    }
+    fileStack[0].rewind();
+    fileStackSize = 1;
+    fileCount = 0;
+    inProgress = true;
+  }
+  inProgress = iterateOnFiles(incrementFileCount, (void*)&fileCount);
+  if (!inProgress) {
+    callWhenFinished(fileCount);
+  }
+}
+
+// Issue: This function does not visually display depth for items in subfolders
+// With maxDepth set to 2, no subfolders are traversed
+bool SD_TC::listFile(File* myFile, void* userData) {
+#ifndef MOCK_PINS_COUNT
+  listFilesData_t* pListFileData = static_cast<listFilesData_t*>(userData);
+  char fileName[15];
+  myFile->getName(fileName, sizeof(fileName));
+  int bytesWritten;
+  if (myFile->isDir()) {
+    bytesWritten = snprintf_P(pListFileData->buffer + pListFileData->linePos,
+                              sizeof(pListFileData->buffer) - pListFileData->linePos,
+                              (PGM_P)F("%11.11s/          \r\n"), fileName);
+  } else {
+    bytesWritten = snprintf_P(pListFileData->buffer + pListFileData->linePos,
+                              sizeof(pListFileData->buffer) - pListFileData->linePos, (PGM_P)F("%s\t%6u KB\r\n"),
+                              fileName, myFile->fileSize() / 1024 + 1);
+  }
+  // "Overwrite" null terminator
+  pListFileData->linePos += bytesWritten;
+  return (++(pListFileData->filesWritten)) % 10 != 0;  // Stop iterating after 10 files
+#else
+  return false;
 #endif
 }
 
 void SD_TC::listRootToBuffer(void (*callWhenFull)(char*, bool)) {
 #ifndef MOCK_PINS_COUNT
   if (!inProgress) {
-    // Initialize hierarchy
-    hierarchy = new File;
-    ++hierarchySize;
     const char path[] PROGMEM = "/";
-    File root = SD_TC::instance()->open(path);
-    if (!root) {
+    fileStack[0] = SD_TC::instance()->open(path);
+    if (!fileStack[0]) {
       serial(F("SD_TC open() failed"));
       return;
     }
-    // Add root to hierarchy if successful
-    root.rewind();
-    hierarchy[0] = root;
+    fileStack[0].rewind();
+    fileStackSize = 1;
     inProgress = true;
   }
-  listFiles(callWhenFull);
+  listFilesData_t listFileData;
+  listFileData.linePos = 0;
+  listFileData.filesWritten = 0;
+  inProgress = iterateOnFiles(listFile, (void*)&listFileData);
+  // Terminate the buffer
+  listFileData.buffer[listFileData.linePos] = '\0';
+  callWhenFull(listFileData.buffer, !inProgress);
 #else
   static const char notImplemented[] PROGMEM = "Root directory not supported by CI framework.\r\n";
   char buffer[sizeof(notImplemented)];

src/Devices/SD_TC.h

@@ -2,9 +2,18 @@
 
 #include <Arduino.h>
 
+#define maxDepth 2
 #define SS 4
 #include <SdFat.h>
 
+typedef bool (*doOnFile)(File*, void*);
+
+struct listFilesData_t {
+  char buffer[250];  // Each line should be 24 characters long; 10 lines
+  int linePos;
+  int filesWritten;
+};
+
 class SD_TC {
 public:
   // class methods
@@ -16,6 +25,7 @@ class SD_TC {
   bool exists(const char* path);
   bool format();
   void listRootToBuffer(void (*callWhenFull)(char*, bool));
+  void countFiles(void (*callWhenFinished)(int));
   bool mkdir(const char* path);
   File open(const char* path, oflag_t oflag = 0x00);
   void printRootDirectory();
@@ -30,12 +40,19 @@ class SD_TC {
   const uint8_t SD_SELECT_PIN = SS;
   bool hasHadError = false;
   SdFat sd;
-  File* hierarchy = nullptr;
-  int hierarchySize = 0;
+
+  // Max depth of file system search for rootdir()
+  // Two is minimum: First for root, second for files
+  // Each is 64 bytes
+  File fileStack[maxDepth];
+  int fileStackSize;
+  int fileCount;
   bool inProgress = false;
 
   // instance methods
   SD_TC();
   void appendDataToPath(const char* data, const char* path);
-  void listFiles(void (*callWhenFull)(char*, bool), byte tabulation = 0);
+  bool iterateOnFiles(doOnFile functionName, void* userData);
+  static bool incrementFileCount(File* myFile, void* pFileCount);
+  static bool listFile(File* myFile, void* userData);
 };

src/TankController.cpp

@@ -21,7 +21,7 @@
 #include "UIState/MainMenu.h"
 #include "UIState/UIState.h"
 
-const char TANK_CONTROLLER_VERSION[] = "22.07.2";
+const char TANK_CONTROLLER_VERSION[] = "22.08.1";
 
 // ------------ Class Methods ------------
 /**