1 files changed, 537 insertions, 97 deletions

diff --git a/src/ext_depends/arsd/cgi.d b/src/ext_depends/arsd/cgi.d
index 0497eb2..0af9f25 100644
--- a/src/ext_depends/arsd/cgi.d
+++ b/src/ext_depends/arsd/cgi.d
@@ -134,7 +134,7 @@ void main() {
 		* `embedded_httpd` for the embedded httpd version (built-in web server). This is the default for dub builds. You can run the program then connect directly to it from your browser. Note: prior to version 11, this would be embedded_httpd_processes on Linux and embedded_httpd_threads everywhere else. It now means embedded_httpd_hybrid everywhere supported and embedded_httpd_threads everywhere else.
 		* `cgi` for traditional cgi binaries. These are run by an outside web server as-needed to handle requests.
 		* `fastcgi` for FastCGI builds. FastCGI is managed from an outside helper, there's one built into Microsoft IIS, Apache httpd, and Lighttpd, and a generic program you can use with nginx called `spawn-fcgi`. If you don't already know how to use it, I suggest you use one of the other modes.
-		* `scgi` for SCGI builds. SCGI is a simplified form of FastCGI, where you run the server as an application service which is proxied by your outside webserver.
+		* `scgi` for SCGI builds. SCGI is a simplified form of FastCGI, where you run the server as an application service which is proxied by your outside webserver. Please note: on nginx make sure you add `scgi_param  PATH_INFO          $document_uri;` to the config!
 		* `stdio_http` for speaking raw http over stdin and stdout. This is made for systemd services. See [RequestServer.serveSingleHttpConnectionOnStdio] for more information.
 	)
 
@@ -188,6 +188,12 @@ void main() {
 
 	For an embedded HTTP server, run `dmd yourfile.d cgi.d -version=embedded_httpd` and run the generated program. It listens on port 8085 by default. You can change this on the command line with the --port option when running your program.
 
+	Command_line_interface:
+
+	If using [GenericMain] or [DispatcherMain], an application using arsd.cgi will offer a command line interface out of the box.
+
+	See [RequestServer.listenSpec] for more information.
+
 	Simulating_requests:
 
 	If you are using one of the [GenericMain] or [DispatcherMain] mixins, or main with your own call to [RequestServer.trySimulatedRequest], you can simulate requests from your command-ine shell. Call the program like this:
@@ -486,10 +492,6 @@ void main() {
 +/
 module arsd.cgi;
 
-static import arsd.core;
-version(Posix)
-import arsd.core : makeNonBlocking;
-
 // FIXME: Nullable!T can be a checkbox that enables/disables the T on the automatic form
 // and a SumType!(T, R) can be a radio box to pick between T and R to disclose the extra boxes on the automatic form
 
@@ -572,8 +574,44 @@ unittest {
 	}
 }
 
+/++
+	The session system works via a built-in spawnable server.
+
+	Bugs:
+		Requires addon servers, which are not implemented yet on Windows.
++/
+version(Posix)
+version(Demo)
+unittest {
+	import arsd.cgi;
+
+	struct SessionData {
+		string userId;
+	}
+
+	void handler(Cgi cgi) {
+		auto session = cgi.getSessionObject!SessionData;
+
+		if(cgi.pathInfo == "/login") {
+			session.userId = cgi.queryString;
+			cgi.setResponseLocation("view");
+		} else {
+			cgi.write(session.userId);
+		}
+	}
+
+	mixin GenericMain!handler;
+}
+
 static import std.file;
 
+static import arsd.core;
+version(Posix)
+import arsd.core : makeNonBlocking;
+
+import arsd.core : encodeUriComponent, decodeUriComponent;
+
+
 // for a single thread, linear request thing, use:
 // -version=embedded_httpd_threads -version=cgi_no_threads
 
@@ -584,6 +622,8 @@ version(Posix) {
 
 	} else {
 		version(FreeBSD) {
+			// not implemented on bsds
+		} else version(OpenBSD) {
 			// I never implemented the fancy stuff there either
 		} else {
 			version=with_breaking_cgi_features;
@@ -622,7 +662,10 @@ version(with_addon_servers)
 	version=with_addon_servers_connections;
 
 version(embedded_httpd) {
-	version=embedded_httpd_hybrid;
+	version(OSX)
+		version = embedded_httpd_threads;
+	else
+		version=embedded_httpd_hybrid;
 	/*
 	version(with_openssl) {
 		pragma(lib, "crypto");
@@ -689,7 +732,6 @@ enum long defaultMaxContentLength = 5_000_000;
 public import std.string;
 public import std.stdio;
 public import std.conv;
-import std.uri;
 import std.uni;
 import std.algorithm.comparison;
 import std.algorithm.searching;
@@ -841,6 +883,35 @@ class Cgi {
 
 
 	/+
+
+	ubyte[] perRequestMemoryPool;
+	void[] perRequestMemoryPoolWithPointers;
+	// might want to just slice the buffer itself too when we happened to have gotten a full request inside it and don't need to decode
+	// then the buffer also can be recycled if it is set.
+
+	// we might also be able to set memory recyclable true by default, but then the property getters set it to false. but not all the things are property getters. but realistically anything except benchmarks are gonna get something lol so meh.
+
+	/+
+	struct VariableCollection {
+		string[] opIndex(string name) {
+
+		}
+	}
+
+	/++
+		Call this to indicate that you've not retained any reference to the request-local memory (including all strings returned from the Cgi object) outside the request (you can .idup anything you need to store) and it is thus free to be freed or reused by another request.
+
+		Most handlers should be able to call this; retaining memory is the exception in any cgi program, but since I can't prove it from inside the library, it plays it safe and lets the GC manage it unless you opt into this behavior. All cgi.d functions will duplicate strings if needed (e.g. session ids from cookies) so unless you're doing something yourself, this should be ok.
+
+		History:
+			Added
+	+/
+	public void recycleMemory() {
+
+	}
+	+/
+
+
 	/++
 		Cgi provides a per-request memory pool
 
@@ -897,6 +968,7 @@ class Cgi {
 		--accept 'content' // FIXME: better example
 		--last-event-id 'something'
 		--host 'something.com'
+		--session name=value (these are added to a mock session, changes to the session are printed out as dummy response headers)
 
 	  Non-simulation arguments:
 	  	--port xxx listening port for non-cgi things (valid for the cgi interfaces)
@@ -962,8 +1034,13 @@ class Cgi {
 					auto info = breakUp(arg);
 					if(_cookie.length)
 						_cookie ~= "; ";
-					_cookie ~= std.uri.encodeComponent(info[0]) ~ "=" ~ std.uri.encodeComponent(info[1]);
+					_cookie ~= encodeUriComponent(info[0]) ~ "=" ~ encodeUriComponent(info[1]);
 				}
+				if (nextArgIs == "session") {
+					auto info = breakUp(arg);
+					_commandLineSession[info[0]] = info[1];
+				}
+
 				else if (nextArgIs == "port") {
 					port = to!int(arg);
 				}
@@ -1042,7 +1119,7 @@ class Cgi {
 					if(_queryString.length)
 						_queryString ~= "&";
 					auto parts = breakUp(arg);
-					_queryString ~= std.uri.encodeComponent(parts[0]) ~ "=" ~ std.uri.encodeComponent(parts[1]);
+					_queryString ~= encodeUriComponent(parts[0]) ~ "=" ~ encodeUriComponent(parts[1]);
 				}
 			}
 		}
@@ -1083,6 +1160,7 @@ class Cgi {
 		this.pathInfo = pathInfo;
 		this.queryString = queryString;
 		this.postBody = null;
+		this.requestContentType = null;
 	}
 
 	private {
@@ -1306,6 +1384,7 @@ class Cgi {
 				files = keepLastOf(filesArray);
 				post = keepLastOf(postArray);
 				this.postBody = pps.postBody;
+				this.requestContentType = contentType;
 				cleanUpPostDataState();
 			}
 
@@ -2133,6 +2212,8 @@ class Cgi {
 			files = keepLastOf(filesArray);
 			post = keepLastOf(postArray);
 			postBody = pps.postBody;
+			this.requestContentType = contentType;
+
 			cleanUpPostDataState();
 		}
 
@@ -2187,28 +2268,31 @@ class Cgi {
 		return assumeUnique(forTheLoveOfGod);
 	}
 
-	/// Very simple method to require a basic auth username and password.
-	/// If the http request doesn't include the required credentials, it throws a
-	/// HTTP 401 error, and an exception.
-	///
-	/// Note: basic auth does not provide great security, especially over unencrypted HTTP;
-	/// the user's credentials are sent in plain text on every request.
-	///
-	/// If you are using Apache, the HTTP_AUTHORIZATION variable may not be sent to the
-	/// application. Either use Apache's built in methods for basic authentication, or add
-	/// something along these lines to your server configuration:
-	///
-	///      RewriteEngine On
-	///      RewriteCond %{HTTP:Authorization} ^(.*)
-	///      RewriteRule ^(.*) - [E=HTTP_AUTHORIZATION:%1]
-	///
-	/// To ensure the necessary data is available to cgi.d.
-	void requireBasicAuth(string user, string pass, string message = null) {
+	/++
+		Very simple method to require a basic auth username and password.
+		If the http request doesn't include the required credentials, it throws a
+		HTTP 401 error, and an exception to cancel your handler. Do NOT catch the
+		`AuthorizationRequiredException` exception thrown by this if you want the
+		http basic auth prompt to work for the user!
+
+		Note: basic auth does not provide great security, especially over unencrypted HTTP;
+		the user's credentials are sent in plain text on every request.
+
+		If you are using Apache, the HTTP_AUTHORIZATION variable may not be sent to the
+		application. Either use Apache's built in methods for basic authentication, or add
+		something along these lines to your server configuration:
+
+		     ```
+		     RewriteEngine On
+		     RewriteCond %{HTTP:Authorization} ^(.*)
+		     RewriteRule ^(.*) - [E=HTTP_AUTHORIZATION:%1]
+		     ```
+
+		To ensure the necessary data is available to cgi.d.
+	+/
+	void requireBasicAuth(string user, string pass, string message = null, string file = __FILE__, size_t line = __LINE__) {
 		if(authorization != "Basic " ~ Base64.encode(cast(immutable(ubyte)[]) (user ~ ":" ~ pass))) {
-			setResponseStatus("401 Authorization Required");
-			header ("WWW-Authenticate: Basic realm=\""~message~"\"");
-			close();
-			throw new Exception("Not authorized; got " ~ authorization);
+			throw new AuthorizationRequiredException("Basic", message, file, line);
 		}
 	}
 
@@ -2370,8 +2454,8 @@ class Cgi {
 	+/
 	void setCookie(string name, string data, long expiresIn = 0, string path = null, string domain = null, bool httpOnly = false, bool secure = false, SameSitePolicy sameSitePolicy = SameSitePolicy.Lax) {
 		assert(!outputtedResponseData);
-		string cookie = std.uri.encodeComponent(name) ~ "=";
-		cookie ~= std.uri.encodeComponent(data);
+		string cookie = encodeUriComponent(name) ~ "=";
+		cookie ~= encodeUriComponent(data);
 		if(path !is null)
 			cookie ~= "; path=" ~ path;
 		// FIXME: should I just be using max-age here? (also in cache below)
@@ -2728,6 +2812,8 @@ class Cgi {
 		return closed;
 	}
 
+	private SessionObject commandLineSessionObject;
+
 	/++
 		Gets a session object associated with the `cgi` request. You can use different type throughout your application.
 	+/
@@ -2743,6 +2829,22 @@ class Cgi {
 				return o;
 			}
 		} else {
+			// FIXME: the changes are not printed out at the end!
+			if(_commandLineSession !is null) {
+				if(commandLineSessionObject is null) {
+					auto clso = new MockSession!Data();
+					commandLineSessionObject = clso;
+
+
+					foreach(memberName; __traits(allMembers, Data)) {
+						if(auto str = memberName in _commandLineSession)
+							__traits(getMember, clso.store_, memberName) = to!(typeof(__traits(getMember, Data, memberName)))(*str);
+					}
+				}
+
+				return cast(typeof(return)) commandLineSessionObject;
+			}
+
 			// normal operation
 			return new BasicDataServerSession!Data(this);
 		}
@@ -2785,6 +2887,14 @@ class Cgi {
 	public immutable string postBody;
 	alias postJson = postBody; // old name
 
+	/++
+		The content type header of the request. The [postBody] member may hold the actual data (see [postBody] for details).
+
+		History:
+			Added January 26, 2024 (dub v11.4)
+	+/
+	public immutable string requestContentType;
+
 	/* Internal state flags */
 	private bool outputtedResponseData;
 	private bool noCache = true;
@@ -2831,6 +2941,9 @@ class Cgi {
 	immutable(string[string]) post; /// The data from the request's body, on POST requests. It parses application/x-www-form-urlencoded data (used by most web requests, including typical forms), and multipart/form-data requests (used by file uploads on web forms) into the same container, so you can always access them the same way. It makes no attempt to parse other content types. If you want to accept an XML Post body (for a web api perhaps), you'll need to handle the raw data yourself.
 	immutable(string[string]) cookies; /// Separates out the cookie header into individual name/value pairs (which is how you set them!)
 
+	/// added later
+	alias query = get;
+
 	/**
 		Represents user uploaded files.
 
@@ -2845,6 +2958,8 @@ class Cgi {
 	immutable(string[][string]) postArray; /// ditto for post
 	immutable(string[][string]) cookiesArray; /// ditto for cookies
 
+	private string[string] _commandLineSession;
+
 	// convenience function for appending to a uri without extra ?
 	// matches the name and effect of javascript's location.search property
 	string search() const {
@@ -3025,7 +3140,7 @@ struct Uri {
 
 	// idk if i want to keep these, since the functions they wrap are used many, many, many times in existing code, so this is either an unnecessary alias or a gratuitous break of compatibility
 	// the decode ones need to keep different names anyway because we can't overload on return values...
-	static string encode(string s) { return std.uri.encodeComponent(s); }
+	static string encode(string s) { return encodeUriComponent(s); }
 	static string encode(string[string] s) { return encodeVariables(s); }
 	static string encode(string[][string] s) { return encodeVariables(s); }
 
@@ -3409,13 +3524,13 @@ string[][string] decodeVariables(string data, string separator = "&", string[]*
 		string name;
 		string value;
 		if(equal == -1) {
-			name = decodeComponent(var);
+			name = decodeUriComponent(var);
 			value = "";
 		} else {
-			//_get[decodeComponent(var[0..equal])] ~= decodeComponent(var[equal + 1 .. $].replace("+", " "));
+			//_get[decodeUriComponent(var[0..equal])] ~= decodeUriComponent(var[equal + 1 .. $].replace("+", " "));
 			// stupid + -> space conversion.
-			name = decodeComponent(var[0..equal].replace("+", " "));
-			value = decodeComponent(var[equal + 1 .. $].replace("+", " "));
+			name = decodeUriComponent(var[0..equal].replace("+", " "));
+			value = decodeUriComponent(var[equal + 1 .. $].replace("+", " "));
 		}
 
 		_get[name] ~= value;
@@ -3448,7 +3563,7 @@ string encodeVariables(in string[string] data) {
 		else
 			outputted = true;
 
-		ret ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v);
+		ret ~= encodeUriComponent(k) ~ "=" ~ encodeUriComponent(v);
 	}
 
 	return ret;
@@ -3465,7 +3580,7 @@ string encodeVariables(in string[][string] data) {
 				ret ~= "&";
 			else
 				outputted = true;
-			ret ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v);
+			ret ~= encodeUriComponent(k) ~ "=" ~ encodeUriComponent(v);
 		}
 	}
 
@@ -3593,6 +3708,11 @@ mixin template CustomCgiDispatcherMain(CustomCgi, size_t maxContentLength, Prese
 		activePresenter = presenter;
 		scope(exit) activePresenter = null;
 
+		if(cgi.pathInfo.length == 0) {
+			cgi.setResponseLocation(cgi.scriptName ~ "/");
+			return;
+		}
+
 		if(cgi.dispatcher!DispatcherArgs(presenter))
 			return;
 
@@ -3746,8 +3866,17 @@ bool trySimulatedRequest(alias fun, CustomCgi = Cgi)(string[] args) if(is(Custom
 	if(args.length >= 3 && isCgiRequestMethod(args[1])) {
 		Cgi cgi = new CustomCgi(args);
 		scope(exit) cgi.dispose();
-		fun(cgi);
-		cgi.close();
+		try {
+			fun(cgi);
+			cgi.close();
+		} catch(AuthorizationRequiredException are) {
+			cgi.setResponseStatus("401 Authorization Required");
+			cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\"");
+			cgi.close();
+		}
+		writeln(); // just to put a blank line before the prompt cuz it annoys me
+		// FIXME: put in some footers to show what changes happened in the session
+		// could make the MockSession be some kind of ReflectableSessionObject or something
 		return true;
 	}
 	return false;
@@ -4046,9 +4175,34 @@ struct RequestServer {
 		} else
 		version(stdio_http) {
 			serveSingleHttpConnectionOnStdio!(fun, CustomCgi, maxContentLength)();
-		} else {
-			//version=plain_cgi;
+		} else
+		version(plain_cgi) {
 			handleCgiRequest!(fun, CustomCgi, maxContentLength)();
+		} else {
+			if(this.listenSpec.length) {
+				// FIXME: what about heterogeneous listen specs?
+				if(this.listenSpec[0].startsWith("scgi:"))
+					serveScgi!(fun, CustomCgi, maxContentLength)();
+				else
+					serveEmbeddedHttp!(fun, CustomCgi, maxContentLength)();
+			} else {
+				import std.process;
+				if("REQUEST_METHOD" in environment) {
+					// GATEWAY_INTERFACE must be set according to the spec for it to be a cgi request
+					// REQUEST_METHOD must also be set
+					handleCgiRequest!(fun, CustomCgi, maxContentLength)();
+				} else {
+					import std.stdio;
+					writeln("To start a local-only http server, use `thisprogram --listen http://localhost:PORT_NUMBER`");
+					writeln("To start a externally-accessible http server, use `thisprogram --listen http://:PORT_NUMBER`");
+					writeln("To start a scgi server, use `thisprogram --listen scgi://localhost:PORT_NUMBER`");
+					writeln("To test a request on the command line, use `thisprogram REQUEST /path arg=value`");
+					writeln("Or copy this program to your web server's cgi-bin folder to run it that way.");
+					writeln("If you need FastCGI, recompile this program with -version=fastcgi");
+					writeln();
+					writeln("Learn more at https://opendlang.org/library/arsd.cgi.html#Command-line-interface");
+				}
+			}
 		}
 	}
 
@@ -4155,6 +4309,17 @@ struct RequestServer {
 	}
 }
 
+class AuthorizationRequiredException : Exception {
+	string type;
+	string realm;
+	this(string type, string realm, string file, size_t line) {
+		this.type = type;
+		this.realm = realm;
+
+		super("Authorization Required", file, line);
+	}
+}
+
 private alias AliasSeq(T...) = T;
 
 version(with_breaking_cgi_features)
@@ -4316,6 +4481,11 @@ void serveEmbeddedHttpdProcesses(alias fun, CustomCgi = Cgi)(RequestServer param
 							cgi.close();
 							if(cgi.websocketMode)
 								closeConnection = true;
+
+						} catch(AuthorizationRequiredException are) {
+							cgi.setResponseStatus("401 Authorization Required");
+							cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\"");
+							cgi.close();
 						} catch(ConnectionException ce) {
 							closeConnection = true;
 						} catch(Throwable t) {
@@ -4478,6 +4648,10 @@ void serveFastCgi(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMax
 		try {
 			fun(cgi);
 			cgi.close();
+		} catch(AuthorizationRequiredException are) {
+			cgi.setResponseStatus("401 Authorization Required");
+			cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\"");
+			cgi.close();
 		} catch(Throwable t) {
 			// log it to the error stream
 			FCGX_PutStr(cast(ubyte*) t.msg.ptr, t.msg.length, error);
@@ -4526,7 +4700,7 @@ void serveFastCgi(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMax
 }
 
 /// Returns the default listening port for the current cgi configuration. 8085 for embedded httpd, 4000 for scgi, irrelevant for others.
-ushort defaultListeningPort() {
+ushort defaultListeningPort() @safe {
 	version(netman_httpd)
 		return 8080;
 	else version(embedded_httpd_processes)
@@ -4540,7 +4714,7 @@ ushort defaultListeningPort() {
 }
 
 /// Default host for listening. 127.0.0.1 for scgi, null (aka all interfaces) for all others. If you want the server directly accessible from other computers on the network, normally use null. If not, 127.0.0.1 is a bit better. Settable with default handlers with --listening-host command line argument.
-string defaultListeningHost() {
+string defaultListeningHost() @safe {
 	version(netman_httpd)
 		return null;
 	else version(embedded_httpd_processes)
@@ -4630,6 +4804,10 @@ void handleCgiRequest(alias fun, CustomCgi = Cgi, long maxContentLength = defaul
 	try {
 		fun(cgi);
 		cgi.close();
+	} catch(AuthorizationRequiredException are) {
+		cgi.setResponseStatus("401 Authorization Required");
+		cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\"");
+		cgi.close();
 	} catch (Throwable t) {
 		version(CRuntime_Musl) {
 			// LockingTextWriter fails here
@@ -4758,6 +4936,11 @@ extern(Windows) private {
 	alias GROUP=uint;
 	alias LPWSAPROTOCOL_INFOW = void*;
 	SOCKET WSASocketW(int af, int type, int protocol, LPWSAPROTOCOL_INFOW lpProtocolInfo, GROUP g, DWORD dwFlags);
+	alias WSASend = arsd.core.WSASend;
+	alias WSARecv = arsd.core.WSARecv;
+	alias WSABUF = arsd.core.WSABUF;
+
+	/+
 	int WSASend(SOCKET s, LPWSABUF lpBuffers, DWORD dwBufferCount, LPDWORD lpNumberOfBytesSent, DWORD dwFlags, LPWSAOVERLAPPED lpOverlapped, LPWSAOVERLAPPED_COMPLETION_ROUTINE lpCompletionRoutine);
 	int WSARecv(SOCKET s, LPWSABUF lpBuffers, DWORD dwBufferCount, LPDWORD lpNumberOfBytesRecvd, LPDWORD lpFlags, LPWSAOVERLAPPED lpOverlapped, LPWSAOVERLAPPED_COMPLETION_ROUTINE lpCompletionRoutine);
 
@@ -4765,6 +4948,7 @@ extern(Windows) private {
 		ULONG len;
 		CHAR  *buf;
 	}
+	+/
 	alias LPWSABUF = WSABUF*;
 
 	alias WSAOVERLAPPED = OVERLAPPED;
@@ -4917,7 +5101,7 @@ private class PseudoblockingOverlappedSocket : Socket {
 	override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) @trusted {
 		overlapped = overlapped.init;
 		buffer[0].len = cast(DWORD) buf.length;
-		buffer[0].buf = cast(CHAR*) buf.ptr;
+		buffer[0].buf = cast(ubyte*) buf.ptr;
 		fiber.setPostYield( () {
 			if(!WSASend(handle, buffer.ptr, cast(DWORD) buffer.length, null, 0, &overlapped, null)) {
 				if(GetLastError() != 997) {
@@ -4932,7 +5116,7 @@ private class PseudoblockingOverlappedSocket : Socket {
 	override ptrdiff_t receive(scope void[] buf, SocketFlags flags) @trusted {
 		overlapped = overlapped.init;
 		buffer[0].len = cast(DWORD) buf.length;
-		buffer[0].buf = cast(CHAR*) buf.ptr;
+		buffer[0].buf = cast(ubyte*) buf.ptr;
 
 		DWORD flags2 = 0;
 
@@ -4990,6 +5174,124 @@ void doThreadHttpConnection(CustomCgi, alias fun)(Socket connection) {
 	}
 }
 
+/+
+
+/+
+	The represents a recyclable per-task arena allocator. The default is to let the GC manage the whole block as a large array, meaning if a reference into it is escaped, it waste memory but is not dangerous. If you don't escape any references to it and don't do anything special, the GC collects it.
+
+	But, if you call `cgi.recyclable = true`, the memory is retained for the next request on the thread. If a reference is escaped, it is the user's problem; it can be modified (and break the `immutable` guarantees!) and thus be memory unsafe. They're taking responsibility for doing it right when they call `escape`. But if they do it right and opt into recycling, the memory is all reused to give a potential boost without requiring the GC's involvement.
+
+	What if one request used an abnormally large amount of memory though? Will recycling it keep that pinned forever? No, that's why it keeps track of some stats. If a working set was significantly above average and not fully utilized for a while, it will just let the GC have it again despite your suggestion to recycle it.
+
+	Be warned that growing the memory block may release the old, smaller block for garbage collection. If you retained references to it, it may not be collectable and lead to some unnecessary memory growth. It is probably best to try to keep the things sized in a continuous block that doesn't have to grow often.
+
+	Internally, it is broken up into a few blocks:
+		* the request block. This holds the incoming request and associated data (parsed headers, variables, etc).
+		* the scannable block. this holds pointers arrays, classes, etc. associated with this request, so named because the GC scans it.
+		* the response block. This holds the output buffer.
+
+	And I may add more later if I decide to open this up to outside user code.
+
+	The scannable block is separate to limit the amount of work the GC has to do; no point asking it to scan that which need not be scanned.
+
+	The request and response blocks are separated because they will have different typical sizes, with the request likely being less predictable. Being able to release one to the GC while recycling the other might help, and having them grow independently (if needed) may also prevent some pain.
+
+	All of this are internal implementation details subject to change at any time without notice. It is valid for my recycle method to do absolutely nothing; the GC also eventually recycles memory!
+
+	Each active task can have its own recyclable memory object. When you recycle it, it is added to a thread-local freelist. If the list is excessively large, entries maybe discarded at random and left for the GC to prevent a temporary burst of activity from leading to a permanent waste of memory.
++/
+struct RecyclableMemory {
+	private ubyte[] inputBuffer;
+	private ubyte[] processedRequestBlock;
+	private void[] scannableBlock;
+	private ubyte[] outputBuffer;
+
+	RecyclableMemory* next;
+}
+
+/++
+	This emulates the D associative array interface with a different internal implementation.
+
+	string s = cgi.get["foo"]; // just does cgi.getArray[x][$-1];
+	string[] arr = cgi.getArray["foo"];
+
+	"foo" in cgi.get
+
+	foreach(k, v; cgi.get)
+
+	cgi.get.toAA // for compatibility
+
+	// and this can urldecode lazily tbh... in-place even, since %xx is always longer than a single char thing it turns into...
+		... but how does it mark that it has already been processed in-place? it'd have to just add it to the index then.
+
+	deprecated alias toAA this;
++/
+struct VariableCollection {
+	private VariableArrayCollection* vac;
+
+	const(char[]) opIndex(scope const char[] key) {
+		return (*vac)[key][$-1];
+	}
+
+	const(char[]*) opBinaryRight(string op : "in")(scope const char[] key) {
+		return key in (*vac);
+	}
+
+	int opApply(int delegate(scope const(char)[] key, scope const(char)[] value) dg) {
+		foreach(k, v; *vac) {
+			if(auto res = dg(k, v[$-1]))
+				return res;
+		}
+		return 0;
+	}
+
+	immutable(string[string]) toAA() {
+		string[string] aa;
+		foreach(k, v; *vac)
+			aa[k.idup] = v[$-1].idup;
+		return aa;
+	}
+
+	deprecated alias toAA this;
+}
+
+struct VariableArrayCollection {
+	/+
+		This needs the actual implementation of looking it up. As it pulls data, it should
+		decode and index for later.
+
+		The index will go into a block attached to the cgi object and it should prolly be sorted
+		something like
+
+		[count of names]
+		[slice to name][count of values][slice to value, decoded in-place, ...]
+		...
+	+/
+	private Cgi cgi;
+
+	const(char[][]) opIndex(scope const char[] key) {
+		return null;
+	}
+
+	const(char[][]*) opBinaryRight(string op : "in")(scope const char[] key) {
+		return null;
+	}
+
+	// int opApply(int delegate(scope const(char)[] key, scope const(char)[][] value) dg)
+
+	immutable(string[string]) toAA() {
+		return null;
+	}
+
+	deprecated alias toAA this;
+
+}
+
+struct HeaderCollection {
+
+}
++/
+
 void doThreadHttpConnectionGuts(CustomCgi, alias fun, bool alwaysCloseConnection = false)(Socket connection) {
 	scope(failure) {
 		// catch all for other errors
@@ -5065,6 +5367,10 @@ void doThreadHttpConnectionGuts(CustomCgi, alias fun, bool alwaysCloseConnection
 			cgi.close();
 			if(cgi.websocketMode)
 				closeConnection = true;
+		} catch(AuthorizationRequiredException are) {
+			cgi.setResponseStatus("401 Authorization Required");
+			cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\"");
+			cgi.close();
 		} catch(ConnectionException ce) {
 			// broken pipe or something, just abort the connection
 			closeConnection = true;
@@ -5201,6 +5507,11 @@ void doThreadScgiConnection(CustomCgi, alias fun, long maxContentLength)(Socket
 		fun(cgi);
 		cgi.close();
 		connection.close();
+
+	} catch(AuthorizationRequiredException are) {
+		cgi.setResponseStatus("401 Authorization Required");
+		cgi.header ("WWW-Authenticate: "~are.type~" realm=\""~are.realm~"\"");
+		cgi.close();
 	} catch(Throwable t) {
 		// no std err
 		if(!handleException(cgi, t)) {
@@ -5336,7 +5647,7 @@ version(fastcgi) {
 		int FCGX_HasSeenEOF(FCGX_Stream* stream);
 		c_int FCGX_FFlush(FCGX_Stream *stream);
 
-		int FCGX_OpenSocket(in char*, int);
+		int FCGX_OpenSocket(const char*, int);
 	}
 }
 
@@ -5567,7 +5878,7 @@ class BufferedInputRange {
 		return view;
 	}
 
-	invariant() {
+	@system invariant() {
 		assert(view.ptr >= underlyingBuffer.ptr);
 		// it should never be equal, since if that happens view ought to be empty, and thus reusing the buffer
 		assert(view.ptr < underlyingBuffer.ptr + underlyingBuffer.length);
@@ -6584,6 +6895,8 @@ version(cgi_with_websocket) {
 	class WebSocket {
 		Cgi cgi;
 
+		private bool isClient = false;
+
 		private this(Cgi cgi) {
 			this.cgi = cgi;
 
@@ -6616,12 +6929,14 @@ version(cgi_with_websocket) {
 				return true;
 			}
 
-			if(bfr.sourceClosed)
+			if(bfr.sourceClosed) {
 				return false;
+			}
 
 			bfr.popFront(0);
-			if(bfr.sourceClosed)
+			if(bfr.sourceClosed) {
 				return false;
+			}
 			goto top;
 		}
 
@@ -6701,7 +7016,48 @@ version(cgi_with_websocket) {
 			string origin; /// Origin URL to send with the handshake, if desired.
 			string protocol; /// the protocol header, if desired.
 
-			int pingFrequency = 5000; /// Amount of time (in msecs) of idleness after which to send an automatic ping
+			/++
+				Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example:
+
+				---
+				Config config;
+				config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here";
+				---
+
+				History:
+					Added February 19, 2021 (included in dub version 9.2)
+			+/
+			string[] additionalHeaders;
+
+			/++
+				Amount of time (in msecs) of idleness after which to send an automatic ping
+
+				Please note how this interacts with [timeoutFromInactivity] - a ping counts as activity that
+				keeps the socket alive.
+			+/
+			int pingFrequency = 5000;
+
+			/++
+				Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead.
+
+				The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though!
+
+				History:
+					Added March 31, 2021 (included in dub version 9.4)
+			+/
+			Duration timeoutFromInactivity = 1.minutes;
+
+			/++
+				For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be
+				verified. Setting this to `false` will skip this check and allow the connection to continue anyway.
+
+				History:
+					Added April 5, 2022 (dub v10.8)
+
+					Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes
+					even if it was true, it would skip the verification. Now, it always respects this local setting.
+			+/
+			bool verifyPeer = true;
 		}
 
 		/++
@@ -6713,9 +7069,15 @@ version(cgi_with_websocket) {
 
 		/++
 			Closes the connection, sending a graceful teardown message to the other side.
+
+			Code 1000 is the normal closure code.
+
+			History:
+				The default `code` was changed to 1000 on January 9, 2023. Previously it was 0,
+				but also ignored anyway.
 		+/
 		/// Group: foundational
-		void close(int code = 0, string reason = null)
+		void close(int code = 1000, string reason = null)
 			//in (reason.length < 123)
 			in { assert(reason.length < 123); } do
 		{
@@ -6723,31 +7085,43 @@ version(cgi_with_websocket) {
 				return; // it cool, we done
 			WebSocketFrame wss;
 			wss.fin = true;
+			wss.masked = this.isClient;
 			wss.opcode = WebSocketOpcode.close;
-			wss.data = cast(ubyte[]) reason.dup;
+			wss.data = [ubyte((code >> 8) & 0xff), ubyte(code & 0xff)] ~ cast(ubyte[]) reason.dup;
 			wss.send(&llsend);
 
 			readyState_ = CLOSING;
 
+			closeCalled = true;
+
 			llclose();
 		}
 
+		private bool closeCalled;
+
 		/++
 			Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function.
 		+/
 		/// Group: foundational
-		void ping() {
+		void ping(in ubyte[] data = null) {
 			WebSocketFrame wss;
 			wss.fin = true;
+			wss.masked = this.isClient;
 			wss.opcode = WebSocketOpcode.ping;
+			if(data !is null) wss.data = data.dup;
 			wss.send(&llsend);
 		}
 
-		// automatically handled....
-		void pong() {
+		/++
+			Sends a pong message to the server. This is normally done automatically in response to pings.
+		+/
+		/// Group: foundational
+		void pong(in ubyte[] data = null) {
 			WebSocketFrame wss;
 			wss.fin = true;
+			wss.masked = this.isClient;
 			wss.opcode = WebSocketOpcode.pong;
+			if(data !is null) wss.data = data.dup;
 			wss.send(&llsend);
 		}
 
@@ -6758,6 +7132,7 @@ version(cgi_with_websocket) {
 		void send(in char[] textData) {
 			WebSocketFrame wss;
 			wss.fin = true;
+			wss.masked = this.isClient;
 			wss.opcode = WebSocketOpcode.text;
 			wss.data = cast(ubyte[]) textData.dup;
 			wss.send(&llsend);
@@ -6769,6 +7144,7 @@ version(cgi_with_websocket) {
 		/// Group: foundational
 		void send(in ubyte[] binaryData) {
 			WebSocketFrame wss;
+			wss.masked = this.isClient;
 			wss.fin = true;
 			wss.opcode = WebSocketOpcode.binary;
 			wss.data = cast(ubyte[]) binaryData.dup;
@@ -6803,8 +7179,12 @@ version(cgi_with_websocket) {
 				return false;
 			if(!isDataPending())
 				return true;
-			while(isDataPending())
-				lowLevelReceive();
+
+			while(isDataPending()) {
+				if(lowLevelReceive() == false)
+					throw new ConnectionClosedException("Connection closed in middle of message");
+			}
+
 			goto checkAgain;
 		}
 
@@ -6882,23 +7262,40 @@ version(cgi_with_websocket) {
 						}
 					break;
 					case WebSocketOpcode.close:
-						readyState_ = CLOSED;
+
+						//import std.stdio; writeln("closed ", cast(string) m.data);
+
+						ushort code = CloseEvent.StandardCloseCodes.noStatusCodePresent;
+						const(char)[] reason;
+
+						if(m.data.length >= 2) {
+							code = (m.data[0] << 8) | m.data[1];
+							reason = (cast(char[]) m.data[2 .. $]);
+						}
+
 						if(onclose)
-							onclose();
+							onclose(CloseEvent(code, reason, true));
+
+						// if we receive one and haven't sent one back we're supposed to echo it back and close.
+						if(!closeCalled)
+							close(code, reason.idup);
+
+						readyState_ = CLOSED;
 
 						unregisterActiveSocket(this);
 					break;
 					case WebSocketOpcode.ping:
-						pong();
+						// import std.stdio; writeln("ping received ", m.data);
+						pong(m.data);
 					break;
 					case WebSocketOpcode.pong:
+						// import std.stdio; writeln("pong received ", m.data);
 						// just really references it is still alive, nbd.
 					break;
 					default: // ignore though i could and perhaps should throw too
 				}
 			}
 
-			// the recv thing can be invalidated so gotta copy it over ugh
 			if(d.length) {
 				m.data = m.data.dup();
 			}
@@ -6917,8 +7314,52 @@ version(cgi_with_websocket) {
 			} while(lowLevelReceive());
 		}
 
+		/++
+			Arguments for the close event. The `code` and `reason` are provided from the close message on the websocket, if they are present. The spec says code 1000 indicates a normal, default reason close, but reserves the code range from 3000-5000 for future definition; the 3000s can be registered with IANA and the 4000's are application private use. The `reason` should be user readable, but not displayed to the end user. `wasClean` is true if the server actually sent a close event, false if it just disconnected.
+
+			$(PITFALL
+				The `reason` argument references a temporary buffer and there's no guarantee it will remain valid once your callback returns. It may be freed and will very likely be overwritten. If you want to keep the reason beyond the callback, make sure you `.idup` it.
+			)
+
+			History:
+				Added March 19, 2023 (dub v11.0).
+		+/
+		static struct CloseEvent {
+			ushort code;
+			const(char)[] reason;
+			bool wasClean;
 
-		void delegate() onclose; ///
+			string extendedErrorInformationUnstable;
+
+			/++
+				See https://www.rfc-editor.org/rfc/rfc6455#section-7.4.1 for details.
+			+/
+			enum StandardCloseCodes {
+				purposeFulfilled = 1000,
+				goingAway = 1001,
+				protocolError = 1002,
+				unacceptableData = 1003, // e.g. got text message when you can only handle binary
+				Reserved = 1004,
+				noStatusCodePresent = 1005, // not set by endpoint.
+				abnormalClosure = 1006, // not set by endpoint. closed without a Close control. FIXME: maybe keep a copy of errno around for these
+				inconsistentData = 1007, // e.g. utf8 validation failed
+				genericPolicyViolation = 1008,
+				messageTooBig = 1009,
+				clientRequiredExtensionMissing = 1010, // only the client should send this
+				unnexpectedCondition = 1011,
+				unverifiedCertificate = 1015, // not set by client
+			}
+		}
+
+		/++
+			The `CloseEvent` you get references a temporary buffer that may be overwritten after your handler returns. If you want to keep it or the `event.reason` member, remember to `.idup` it.
+
+			History:
+				The `CloseEvent` was changed to a [arsd.core.FlexibleDelegate] on March 19, 2023 (dub v11.0). Before that, `onclose` was a public member of type `void delegate()`. This change means setters still work with or without the [CloseEvent] argument.
+
+				Your onclose method is now also called on abnormal terminations. Check the `wasClean` member of the `CloseEvent` to know if it came from a close frame or other cause.
+		+/
+		arsd.core.FlexibleDelegate!(void delegate(CloseEvent event)) onclose;
 		void delegate() onerror; ///
 		void delegate(in char[]) ontextmessage; ///
 		void delegate(in ubyte[]) onbinarymessage; ///
@@ -6937,7 +7378,8 @@ version(cgi_with_websocket) {
 			onbinarymessage = dg;
 		}
 
-		/* } end copy/paste */
+	/* } end copy/paste */
+
 
 
 	}
@@ -6983,7 +7425,9 @@ version(cgi_with_websocket) {
 
 		cgi.flush();
 
-		return new WebSocket(cgi);
+		auto ws = new WebSocket(cgi);
+		ws.readyState_ = WebSocket.OPEN;
+		return ws;
 	}
 
 	// FIXME get websocket to work on other modes, not just embedded_httpd
@@ -8352,16 +8796,23 @@ final class ScheduledJobServerImplementation : ScheduledJobServer, EventIoServer
 	int epoll_fd() { return epoll_fd_; }
 }
 
-///
+/++
+	History:
+		Added January 6, 2019
++/
 version(with_addon_servers_connections)
 interface EventSourceServer {
 	/++
 		sends this cgi request to the event server so it will be fed events. You should not do anything else with the cgi object after this.
 
-		$(WARNING This API is extremely unstable. I might change it or remove it without notice.)
-
 		See_Also:
 			[sendEvent]
+
+		Bugs:
+			Not implemented on Windows!
+
+		History:
+			Officially stabilised on November 23, 2023 (dub v11.4). It actually worked pretty well in its original design.
 	+/
 	public static void adoptConnection(Cgi cgi, in char[] eventUrl) {
 		/*
@@ -8415,16 +8866,17 @@ interface EventSourceServer {
 	/++
 		Sends an event to the event server, starting it if necessary. The event server will distribute it to any listening clients, and store it for `lifetime` seconds for any later listening clients to catch up later.
 
-		$(WARNING This API is extremely unstable. I might change it or remove it without notice.)
-
 		Params:
 			url = A string identifying this event "bucket". Listening clients must also connect to this same string. I called it `url` because I envision it being just passed as the url of the request.
 			event = the event type string, which is used in the Javascript addEventListener API on EventSource
 			data = the event data. Available in JS as `event.data`.
 			lifetime = the amount of time to keep this event for replaying on the event server.
 
-		See_Also:
-			[sendEventToEventServer]
+		Bugs:
+			Not implemented on Windows!
+
+		History:
+			Officially stabilised on November 23, 2023 (dub v11.4). It actually worked pretty well in its original design.
 	+/
 	public static void sendEvent(string url, string event, string data, int lifetime) {
 		auto s = openLocalServerConnection("/tmp/arsd_cgi_event_server", "--event-server");
@@ -9713,8 +10165,10 @@ q"css
 
 			ol.automatic-data-display {
 				margin: 0px;
+				/*
 				list-style-position: inside;
 				padding: 0px;
+				*/
 			}
 
 			dl.automatic-data-display {
@@ -9764,11 +10218,13 @@ q"css
 			}
 			#site-container {
 				display: flex;
+				flex-wrap: wrap;
 			}
 			main {
 				flex: 1 1 auto;
 				order: 2;
 				min-height: calc(100vh - 64px - 64px);
+				min-width: 80ch;
 				padding: 4px;
 				padding-left: 1em;
 			}
@@ -10290,7 +10746,7 @@ html", true, true);
 			return dl;
 		} else static if(is(T == bool)) {
 			return Element.make("span", t ? "true" : "false", "automatic-data-display");
-		} else static if(is(T == E[], E)) {
+		} else static if(is(T == E[], E) || is(T == E[N], E, size_t N)) {
 			static if(is(E : RestObject!Proxy, Proxy)) {
 				// treat RestObject similar to struct
 				auto table = cast(Table) Element.make("table");
@@ -11518,7 +11974,7 @@ auto serveStaticFile(string urlPrefix, string filename = null, string contentTyp
 // man 2 sendfile
 	assert(urlPrefix[0] == '/');
 	if(filename is null)
-		filename = decodeComponent(urlPrefix[1 .. $]); // FIXME is this actually correct?
+		filename = decodeUriComponent(urlPrefix[1 .. $]); // FIXME is this actually correct?
 	if(contentType is null) {
 		contentType = contentTypeFromFileExtension(filename);
 	}
@@ -11566,27 +12022,8 @@ auto serveStaticData(string urlPrefix, immutable(void)[] data, string contentTyp
 }
 
 string contentTypeFromFileExtension(string filename) {
-		if(filename.endsWith(".png"))
-			return "image/png";
-		if(filename.endsWith(".apng"))
-			return "image/apng";
-		if(filename.endsWith(".svg"))
-			return "image/svg+xml";
-		if(filename.endsWith(".jpg"))
-			return "image/jpeg";
-		if(filename.endsWith(".html"))
-			return "text/html";
-		if(filename.endsWith(".css"))
-			return "text/css";
-		if(filename.endsWith(".js"))
-			return "application/javascript";
-		if(filename.endsWith(".wasm"))
-			return "application/wasm";
-		if(filename.endsWith(".mp3"))
-			return "audio/mpeg";
-		if(filename.endsWith(".pdf"))
-			return "application/pdf";
-		return null;
+	import arsd.core;
+	return FilePath(filename).contentTypeFromFileExtension();
 }
 
 /// This serves a directory full of static files, figuring out the content-types from file extensions.
@@ -11609,7 +12046,7 @@ auto serveStaticFileDirectory(string urlPrefix, string directory = null, bool re
 	assert(directory[$-1] == '/');
 
 	static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) {
-		auto file = decodeComponent(cgi.pathInfo[urlPrefix.length .. $]); // FIXME: is this actually correct
+		auto file = decodeUriComponent(cgi.pathInfo[urlPrefix.length .. $]); // FIXME: is this actually correct
 
 		if(details.recursive) {
 			// never allow a backslash since it isn't in a typical url anyway and makes the following checks easier
@@ -11635,6 +12072,9 @@ auto serveStaticFileDirectory(string urlPrefix, string directory = null, bool re
 				return false;
 		}
 
+		if(file.length == 0)
+			return false;
+
 		auto contentType = contentTypeFromFileExtension(file);
 
 		auto fn = details.directory ~ file;