Merge pull request #19 from DannyArends/development

Development

Author: DannyArends

danode/cgi.d

@@ -38,6 +38,8 @@ class CGI : Payload {
       return -1; 
     }
 
+    @property void notifyovertime() { external.notifyovertime(); }
+
     // Last modified time (not interesting for scripts)
     final @property SysTime mtime() { return Clock.currTime(); }
 

danode/client.d

@@ -68,10 +68,10 @@ class Client : Thread, ClientInterface {
           Thread.sleep(dur!"msecs"(2));
         }
       } catch(Exception e) { 
-        warning("unknown client exception: %s", e.msg);
+        warning("unknown client exception: %s", e);
         stop();
       }
-      custom(1, "CLIENT", "connection %s:%s (%s) closed after %d requests %s (%s msecs)", ip, port, (driver.isSecure() ? "⚐" : "⚑"), 
+      custom(1, "CLIENT", "connection %s:%s (%s) closed after %d requests %s (%s msecs)", ip, port, (driver.isSecure() ? "SSL" : "HTTP"), 
                                                                                           driver.requests, driver.senddata, starttime);
       driver.destroy();                                               // Clear the response structure
     }

danode/filesystem.d

@@ -3,142 +3,32 @@ module danode.filesystem;
 import danode.imports;
 import danode.statuscode : StatusCode;
 import danode.mimetypes : mime;
-import danode.payload : Payload, PayloadType;
+import danode.payload : Payload, FilePayload, PayloadType;
 import danode.functions : has, isCGI;
 import danode.log : custom, info, Log, warning, trace, cverbose, NOTSET, NORMAL, DEBUG;
 
-class FileInfo : Payload {
-  public:
-    bool      deflate = false;
-  private:
-    string    path;
-    SysTime   btime;
-    bool      buffered = false;
-    char[]    buf = null;
-    char[]    encbuf = null;
-    File*     fp = null;
-
-  public:
-    this(string path) { this.path = path; }
-
-    // Does the file needs updating
-    final bool needsupdate(size_t buffersize = 4096) {
-      if( fitsInBuffer(buffersize) && needsBuffer() ) {
-        if (!buffered) {
-          info("need to buffer file record: %s", path);
-          return true;
-        }
-        if (mtime > btime) {
-          info("re-buffer stale file record: %s", path);
-          return true;
-        }
-      }
-      return false;
-    }
-
-    // Does the file fit in the buffer
-    final bool fitsInBuffer(size_t buffersize = 4096) {
-      if(fileSize() > 0 && fileSize() < buffersize){ return(true); }
-      return(false);
-    }
-
-    // Buffer the file
-    final void buffer() { synchronized {
-      if(buf is null) buf = new char[](fileSize());
-      buf.length = fileSize();
-      try {
-        if(fp is null) fp = new File(path, "rb");
-        fp.open(path, "rb");
-        fp.rawRead(buf);
-        fp.close();
-      } catch (Exception e) {
-        warning("exception during buffering '%s': %s", path, e.msg);
-        return;
-      }
-      try {
-        encbuf = cast(char[])( compress(buf, 9) );
-      } catch (Exception e) {
-        warning("exception during compressing '%s': %s", path, e.msg);
-      }
-      btime = Clock.currTime();
-      trace("buffered %s: %d|%d bytes", path, fileSize(), encbuf.length);
-      buffered = true;
-    } }
-
-    final @property string content(){ return( to!string(bytes(0, length)) ); }
-    final @property bool realfile() const { return(path.exists()); }
-    final @property bool hasEncodedVersion() const { return(encbuf !is null); }
-    final @property bool needsBuffer() { return(!path.isCGI()); }
-    final @property SysTime mtime() const { if(!realfile){ return btime; } return path.timeLastModified(); }
-    final @property long ready() { return(true); }
-    final @property PayloadType type() const { return(PayloadType.Message); }
-    final @property ptrdiff_t fileSize() const { if(!realfile){ return -1; } return to!ptrdiff_t(path.getSize()); }
-    final @property long buffersize() const { return cast(long)(buf.length); }
-    final @property string mimetype() const { return mime(path); }
-    final @property StatusCode statuscode() const { return StatusCode.Ok; }
-
-    final @property ptrdiff_t length() const {
-      if(hasEncodedVersion && deflate) return(encbuf.length);
-      return(fileSize());
-    }
-
-    // Send the file as a stream
-    final char[] asStream(ptrdiff_t from, ptrdiff_t maxsize = 1024) {
-      if(buf is null) buf = new char[](maxsize);
-      char[] slice = [];
-      if (cverbose >= DEBUG && from == 0) write("[STREAM] .");
-      if (from >= fileSize()) {
-        trace("from >= filesize, are we still trying to send?");
-        return([]);
-      }
-      try {
-        if(fp is null) fp = new File(path, "rb");
-        fp.open(path, "rb");
-        if(fp.isOpen()) {
-          fp.seek(from);
-          slice = fp.rawRead!char(buf);
-          fp.close();
-          if (cverbose >= DEBUG) write(".");
-          if (cverbose >= DEBUG && (from + slice.length) >= fileSize()) write("\n");
-        }
-      } catch(Exception e) { 
-        warning("exception %s while streaming file: %s", e.msg, path);
-      }
-      return(slice);
-    }
-
-    final char[] bytes(ptrdiff_t from, ptrdiff_t maxsize = 1024){ synchronized {
-      if (!realfile) { return []; }
-      trace("file provided is a real file");
-      if (needsupdate) { buffer(); }
-      if (!buffered) {
-        return(asStream(from, maxsize));
-      } else {
-        if(hasEncodedVersion && deflate) {
-          if(from < encbuf.length) return( encbuf[from .. to!ptrdiff_t(min(from+maxsize, $))] );
-        } else {
-          if(from < buf.length) return( buf[from .. to!ptrdiff_t(min(from+maxsize, $))] );
-        }
-      }
-      return([]);
-    } }
-}
-
+/* Domain name structure containing files in that domain
+   Domains are loaded by the FileSystem from the -wwwRoot variable (set to www/ by default)
+   Note 1: Domains are named as requested by the HTTP client so SSL keynames must match domainnames (e.g.: localhost / 127.0.0.1 / XX.XX.XX.XX or xxx.xx)
+   Note 2: ./www/localhost existing is required for unit testing */
 struct Domain {
-  FileInfo[string] files;
+  FilePayload[string] files;
   long entries;
   long buffered;
 
   @property long buffersize() const { long sum = 0; foreach(ref f; files.byKey){ sum += files[f].buffersize(); } return sum; }
   @property long size() const { long sum = 0; foreach(ref f; files.byKey){ sum += files[f].length(); } return sum; }
 }
 
+/* File system class that manages the underlying domains
+   Note: Should this really be thread synchronized access ?
+ */
 class FileSystem {
   private:
-    string            root;
-    Domain[string]    domains;
-    Log               logger;
-    size_t              maxsize;
+    string         root;
+    Domain[string] domains;
+    Log            logger;
+    size_t         maxsize;
 
   public:
     this(Log logger, string root = "./www/", size_t maxsize = 1024 * 512){
@@ -148,24 +38,24 @@ class FileSystem {
       scan();
     }
 
-    // Scan the whole filesystem
+    /* Scan the whole filesystem for changes */
     final void scan(){ synchronized {
       foreach (DirEntry d; dirEntries(root, SpanMode.shallow)){ if(d.isDir()){
         domains[d.name] = scan(d.name);
       } }
     } }
 
-    // Scan a single directory
+    /* Scan a single folder */
     final Domain scan(string dname){ synchronized {
       Domain domain;
       foreach (DirEntry f; dirEntries(dname, SpanMode.depth)) {
         if (f.isFile()) {
           string shortname = replace(f.name[dname.length .. $], "\\", "/");
           custom(1, "SCAN", "file: %s -> %s", f.name, shortname);
           if (!domain.files.has(shortname)) {
-            domain.files[shortname] = new FileInfo(f.name);
+            domain.files[shortname] = new FilePayload(f.name, maxsize);
             domain.entries++;
-            if (domain.files[shortname].needsupdate(maxsize)) {
+            if (domain.files[shortname].needsupdate()) {
               domain.files[shortname].buffer();
               domain.buffered++;
             }
@@ -177,27 +67,35 @@ class FileSystem {
       return(domain);
     } }
 
-    // Get the localroot of the domain
-    final string localroot(string hostname) const { return(format("%s%s",this.root, hostname)); }
+    /* Get the localroot of the domain (TODO: Is there a bug, did I require that this.root should always end in a '/' ?) */
+    final string localroot(string hostname) const { return(format("%s%s", this.root, hostname)); }
 
-    // Get the file at path from the localroot
-    final FileInfo file(string localroot, string path){ synchronized {
-      if(!domains[localroot].files.has(path) && exists(format("%s%s", localroot, path))){
-        custom(1, "SCAN", "new file %s, rescanning index: %s", path, localroot);
+    /* Get the FilePayload at path from the localroot, with update check on buffers */
+    final FilePayload file(string localroot, string path){ synchronized {
+      // New file created after last scan ? -> Scan the whole folder for changes
+      if (!domains[localroot].files.has(path) && exists(format("%s%s", localroot, path))) {
+        custom(1, "SCAN", "New file %s, rescanning index: %s", path, localroot);
         domains[localroot] = scan(localroot);
       }
-      if(domains[localroot].files.has(path)) return(domains[localroot].files[path]);
-      return new FileInfo("");
+      // File exists, buffer the individual file if modified after buffer date
+      if (domains[localroot].files.has(path)) {
+        if (domains[localroot].files[path].needsupdate) domains[localroot].files[path].buffer();
+        return(domains[localroot].files[path]);
+      }
+      custom(1, "SCAN", "should not be here not in index, but exists %s, %s", path, localroot);
+      return new FilePayload("", maxsize);
     } }
 
-    // Rebuffer all files
+    /* Rebuffer all file domains from disk, 
+       By reusing domain keys so, we don't buffer new domains. This is ok since we would need to load SSL */
     final void rebuffer() {
       foreach(ref d; domains.byKey){ foreach(ref f; domains[d].files.byKey){
         domains[d].files[f].buffer();
       } }
     }
 }
 
+/* Basic unit-tests should be extended */
 unittest {
   custom(0, "FILE", "%s", __FILE__);
   Log logger = new Log(NORMAL);

danode/imports.d

@@ -12,13 +12,14 @@ public import std.conv : to;
 public import std.datetime : dur, msecs;
 public import std.getopt : getopt;
 public import std.path : baseName, extension;
-public import std.process : pipe, spawnShell, tryWait, wait, kill;
+public import std.process : pipe, spawnShell, executeShell, tryWait, wait, kill;
 public import std.file : dirEntries, exists, remove, isFile, isDir, timeLastModified, getSize;
 public import std.format : format, formatValue;
 public import std.regex : regex, match;
 public import std.stdio : fgetc, fflush, ftell, stderr, stdin, stdout, writef, writefln, write, writeln;
 public import std.string : chomp, endsWith, empty, format, indexOf, join, replace, split, startsWith, strip, toLower, toStringz;
 public import std.uuid : md5UUID;
+public import std.uri : decodeComponent;
 public import std.zlib : compress;
 
 // Public imported structures and enums from core
@@ -29,7 +30,7 @@ public import std.array : Appender;
 public import std.datetime : Clock, DateTime, Duration, SysTime;
 public import std.format : FormatSpec;
 public import std.file : DirEntry, SpanMode;
-public import std.process : Config, Pipe;
+public import std.process : Pid, Config, Pipe;
 public import std.stdio : EOF, File;
 public import std.socket : Address, AddressFamily, InternetAddress, ProtocolType, Socket, SocketOption, SocketOptionLevel, SocketSet, SocketShutdown, SocketType;
 public import std.traits: SetFunctionAttributes, functionAttributes, EnumMembers;

danode/log.d

@@ -111,8 +111,8 @@ class Log {
     // Log the responses to the request
     void logRequest(in ClientInterface cl, in Request rq, in Response rs) {
       if (cverbose >= NOTSET) {
-        string s = format("[%d]    %s %s:%s %s%s %s %s", rs.statuscode, htmltime(), cl.ip, cl.port, rq.shorthost, rq.uri, Msecs(rq.starttime), rs.payload.length);
-        RequestLogFp.writefln(s);
+        string s = format("[%d]    %s %s:%s %s%s %s %s", rs.statuscode, htmltime(), cl.ip, cl.port, rq.shorthost, decodeComponent(rq.uri), Msecs(rq.starttime), rs.payload.length);
+        RequestLogFp.writeln(s);
         custom(-1, "REQ", s);
         RequestLogFp.flush();
       }