diff --git a/http/vibe/http/fileserver.d b/http/vibe/http/fileserver.d index 786e3f4e08..2fd2d2c787 100644 --- a/http/vibe/http/fileserver.d +++ b/http/vibe/http/fileserver.d @@ -16,11 +16,15 @@ import vibe.inet.mimetypes; import vibe.inet.url; import vibe.internal.interfaceproxy; +import std.ascii : isWhite; +import std.algorithm; import std.conv; import std.datetime; import std.digest.md; +import std.exception; +import std.range : popFront, empty, drop; import std.string; -import std.algorithm; +import std.typecons : Flag, Yes, No; @safe: @@ -164,12 +168,12 @@ HTTPServerRequestDelegateS serveStaticFile(string local_path, HTTPFileServerSett When serving a file, certain request headers are supported to avoid sending the file if the client has it already cached. These headers are `"If-Modified-Since"` and `"If-None-Match"`. The client will be delivered - with the necessary `"Etag"` (generated from the path, size and last - modification time of the file) and `"Last-Modified"` headers. + with the necessary `"Etag"` (generated from size and last modification time + of the file) and `"Last-Modified"` headers. - The cache control directives `"Expires"` and `"Cache-Control"` will also be + The cache control directives `"Expires"` and/or `"Cache-Control"` will also be emitted if the `HTTPFileServerSettings.maxAge` field is set to a positive - duration. + duration and/or `HTTPFileServerSettings.cacheControl` has been set. Finally, HEAD requests will automatically be handled without reading the actual file contents. Am empty response body is written instead. @@ -203,6 +207,20 @@ class HTTPFileServerSettings { /// Maximum cache age to report to the client (zero by default) Duration maxAge = 0.seconds; + /** Cache control to control where cache can be saved, if at all, such as + proxies, the storage, etc. + + Leave null or empty to not emit any cache control directives other than + max-age if maxAge is set. + + Common values include: public for making a shared resource cachable across + multiple users or private for a response that should only be cached for a + single user. + + See https://developer.mozilla.org/de/docs/Web/HTTP/Headers/Cache-Control + */ + string cacheControl = null; + /// General options HTTPFileServerOption options = HTTPFileServerOption.defaults; /// additional options @@ -282,32 +300,8 @@ private void sendFileImpl(scope HTTPServerRequest req, scope HTTPServerResponse return; } - auto lastModified = toRFC822DateTimeString(dirent.timeModified.toUTC()); - // simple etag generation - auto etag = "\"" ~ hexDigest!MD5(pathstr ~ ":" ~ lastModified ~ ":" ~ to!string(dirent.size)).idup ~ "\""; - - res.headers["Last-Modified"] = lastModified; - res.headers["Etag"] = etag; - if (settings.maxAge > seconds(0)) { - auto expireTime = Clock.currTime(UTC()) + settings.maxAge; - res.headers["Expires"] = toRFC822DateTimeString(expireTime); - res.headers["Cache-Control"] = "max-age="~to!string(settings.maxAge.total!"seconds"); - } - - if( auto pv = "If-Modified-Since" in req.headers ) { - if( *pv == lastModified ) { - res.statusCode = HTTPStatus.NotModified; - res.writeVoidBody(); - return; - } - } - - if( auto pv = "If-None-Match" in req.headers ) { - if ( *pv == etag ) { - res.statusCode = HTTPStatus.NotModified; - res.writeVoidBody(); - return; - } + if (handleCacheFile(req, res, dirent, settings.cacheControl, settings.maxAge)) { + return; } auto mimetype = res.headers.get("Content-Type", getMimeTypeForFile(pathstr)); @@ -427,3 +421,292 @@ private void sendFileImpl(scope HTTPServerRequest req, scope HTTPServerResponse logTrace("sent file %d, %s!", fil.size, res.headers["Content-Type"]); } } + +/** + Calls $(D handleCache) with prefilled etag and lastModified value based on a file. + + See_Also: handleCache + + Returns: $(D true) if the cache was already handled and no further response must be sent or $(D false) if a response must be sent. +*/ +bool handleCacheFile(scope HTTPServerRequest req, scope HTTPServerResponse res, + string file, string cache_control = null, Duration max_age = Duration.zero) +{ + return handleCacheFile(req, res, NativePath(file), cache_control, max_age); +} + +/// ditto +bool handleCacheFile(scope HTTPServerRequest req, scope HTTPServerResponse res, + NativePath file, string cache_control = null, Duration max_age = Duration.zero) +{ + if (!existsFile(file)) { + return false; + } + + FileInfo ent; + try { + ent = getFileInfo(file); + } catch (Exception) { + throw new HTTPStatusException(HTTPStatus.internalServerError, + "Failed to get information for the file due to a file system error."); + } + + return handleCacheFile(req, res, ent, cache_control, max_age); +} + +/// ditto +bool handleCacheFile(scope HTTPServerRequest req, scope HTTPServerResponse res, + FileInfo dirent, string cache_control = null, Duration max_age = Duration.zero) +{ + import std.bitmanip : nativeToLittleEndian; + import std.digest.md : MD5, toHexString; + + SysTime lastModified = dirent.timeModified; + const weak = cast(Flag!"weak") dirent.isDirectory; + auto etag = ETag.md5(weak, lastModified.stdTime.nativeToLittleEndian, dirent.size.nativeToLittleEndian); + + return handleCache(req, res, etag, lastModified, cache_control, max_age); +} + +/** + Processes header tags in a request and writes responses given on requested cache status. + + Params: + req = the client request used to determine cache control flow. + res = the response to write cache headers to. + etag = if set to anything except .init, adds a Etag header to the response and enables handling of If-Match and If-None-Match cache control request headers. + last_modified = if set to anything except .init, adds a Last-Modified header to the response and enables handling of If-Modified-Since and If-Unmodified-Since cache control request headers. + cache_control = if set, adds or modifies the Cache-Control header in the response to this string. Might get an additional max-age value appended if max_age is set. + max_age = optional duration to set the Expires header and Cache-Control max-age part to. (if no existing `max-age=` part is given in the cache_control parameter) + + Returns: $(D true) if the cache was already handled and no further response must be sent or $(D false) if a response must be sent. +*/ +bool handleCache(scope HTTPServerRequest req, scope HTTPServerResponse res, ETag etag, + SysTime last_modified, string cache_control = null, Duration max_age = Duration.zero) +{ + // https://tools.ietf.org/html/rfc7232#section-4.1 + // and + // https://tools.ietf.org/html/rfc7232#section-6 + string lastModifiedString; + if (last_modified != SysTime.init) { + lastModifiedString = toRFC822DateTimeString(last_modified); + res.headers["Last-Modified"] = lastModifiedString; + } + + if (etag != ETag.init) { + res.headers["Etag"] = etag.toString; + } + + if (max_age > Duration.zero) { + res.headers["Expires"] = toRFC822DateTimeString(Clock.currTime(UTC()) + max_age); + } + + if (cache_control.length) { + if (max_age > Duration.zero && !cache_control.canFind("max-age=")) { + res.headers["Cache-Control"] = cache_control + ~ ", max-age=" ~ to!string(max_age.total!"seconds"); + } else { + res.headers["Cache-Control"] = cache_control; + } + } else if (max_age > Duration.zero) { + res.headers["Cache-Control"] = text("max-age=", max_age.total!"seconds"); + } + + // https://tools.ietf.org/html/rfc7232#section-3.1 + string ifMatch = req.headers.get("If-Match"); + if (ifMatch.length) { + if (!cacheMatch(ifMatch, etag, No.allowWeak)) { + res.statusCode = HTTPStatus.preconditionFailed; + res.writeVoidBody(); + return true; + } + } + else if (last_modified != SysTime.init) { + // https://tools.ietf.org/html/rfc7232#section-3.4 + string ifUnmodifiedSince = req.headers.get("If-Unmodified-Since"); + if (ifUnmodifiedSince.length) { + const check = lastModifiedString != ifUnmodifiedSince + || last_modified > parseRFC822DateTimeString(ifUnmodifiedSince); + if (check) { + res.statusCode = HTTPStatus.preconditionFailed; + res.writeVoidBody(); + return true; + } + } + } + + // https://tools.ietf.org/html/rfc7232#section-3.2 + string ifNoneMatch = req.headers.get("If-None-Match"); + if (ifNoneMatch.length) { + if (cacheMatch(ifNoneMatch, etag, Yes.allowWeak)) { + if (req.method.among!(HTTPMethod.GET, HTTPMethod.HEAD)) + res.statusCode = HTTPStatus.notModified; + else + res.statusCode = HTTPStatus.preconditionFailed; + res.writeVoidBody(); + return true; + } + } + else if (last_modified != SysTime.init && req.method.among!(HTTPMethod.GET, HTTPMethod.HEAD)) { + // https://tools.ietf.org/html/rfc7232#section-3.3 + string ifModifiedSince = req.headers.get("If-Modified-Since"); + if (ifModifiedSince.length) { + const check = lastModifiedString == ifModifiedSince || + last_modified <= parseRFC822DateTimeString(ifModifiedSince); + if (check) { + res.statusCode = HTTPStatus.notModified; + res.writeVoidBody(); + return true; + } + } + } + + // TODO: support If-Range here + + return false; +} + +/** + Represents an Entity-Tag value for use inside HTTP Cache headers. + + Standards: https://tools.ietf.org/html/rfc7232#section-2.3 +*/ +struct ETag +{ + bool weak; + string tag; + + static ETag parse(string s) + { + enforce!ConvException(s.endsWith('"')); + + if (s.startsWith(`W/"`)) { + ETag ret = { weak: true, tag: s[3 .. $ - 1] }; + return ret; + } else if (s.startsWith('"')) { + ETag ret; + ret.tag = s[1 .. $ - 1]; + return ret; + } else { + throw new ConvException(`ETag didn't start with W/" nor with " !`); + } + } + + string toString() const @property + { + return text(weak ? `W/"` : `"`, tag, '"'); + } + + /** + Encodes the bytes with URL Base64 to a human readable string and returns an ETag struct wrapping it. + */ + static ETag fromBytesBase64URLNoPadding(scope const(ubyte)[] bytes, Flag!"weak" weak = No.weak) + { + import std.base64 : Base64URLNoPadding; + + return ETag(weak, Base64URLNoPadding.encode(bytes).idup); + } + + /** + Hashes the input bytes with md5 and returns an URL Base64 encoded representation as ETag. + */ + static ETag md5(T...)(Flag!"weak" weak, T data) + { + import std.digest.md : md5Of; + + return fromBytesBase64URLNoPadding(md5Of(data), weak); + } +} + +/** + Matches a given match expression with a specific ETag. Can allow or disallow weak ETags and supports multiple tags. + + Standards: https://tools.ietf.org/html/rfc7232#section-2.3.2 +*/ +bool cacheMatch(string match, ETag etag, Flag!"allowWeak" allow_weak) +{ + if (match == "*") { + return true; + } + + if ((etag.weak && !allow_weak) || !match.length) { + return false; + } + + auto allBytes = match.representation; + auto range = allBytes; + + while (!range.empty) + { + range = range.stripLeft!isWhite; + bool isWeak = range.skipOver("W/"); + if (!range.skipOver('"')) + return false; // malformed + + auto end = range.countUntil('"'); + if (end == -1) + return false; // malformed + + const check = range[0 .. end]; + range = range[end .. $]; + + if (allow_weak || !isWeak) { + if (check == etag.tag) { + return true; + } + } + + range.skipOver('"'); + range = range.stripLeft!isWhite; + + if (!range.skipOver(",")) + return false; // malformed + } + + return false; +} + +unittest +{ + // from RFC 7232 Section 2.3.2 + // +--------+--------+-------------------+-----------------+ + // | ETag 1 | ETag 2 | Strong Comparison | Weak Comparison | + // +--------+--------+-------------------+-----------------+ + // | W/"1" | W/"1" | no match | match | + // | W/"1" | W/"2" | no match | no match | + // | W/"1" | "1" | no match | match | + // | "1" | "1" | match | match | + // +--------+--------+-------------------+-----------------+ + + assert(!cacheMatch(`W/"1"`, ETag(Yes.weak, "1"), No.allowWeak)); + assert( cacheMatch(`W/"1"`, ETag(Yes.weak, "1"), Yes.allowWeak)); + + assert(!cacheMatch(`W/"1"`, ETag(Yes.weak, "2"), No.allowWeak)); + assert(!cacheMatch(`W/"1"`, ETag(Yes.weak, "2"), Yes.allowWeak)); + + assert(!cacheMatch(`W/"1"`, ETag(No.weak, "1"), No.allowWeak)); + assert( cacheMatch(`W/"1"`, ETag(No.weak, "1"), Yes.allowWeak)); + + assert(cacheMatch(`"1"`, ETag(No.weak, "1"), No.allowWeak)); + assert(cacheMatch(`"1"`, ETag(No.weak, "1"), Yes.allowWeak)); + + assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "xyzzy"), No.allowWeak)); + assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "xyzzy"), Yes.allowWeak)); + + assert(!cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "xyzzz"), No.allowWeak)); + assert(!cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "xyzzz"), Yes.allowWeak)); + + assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "r2d2xxxx"), No.allowWeak)); + assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "r2d2xxxx"), Yes.allowWeak)); + + assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "c3piozzzz"), No.allowWeak)); + assert(cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "c3piozzzz"), Yes.allowWeak)); + + assert(!cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, ""), No.allowWeak)); + assert(!cacheMatch(`"xyzzy","r2d2xxxx", "c3piozzzz"`, ETag(No.weak, ""), Yes.allowWeak)); + + assert(!cacheMatch(`"xyzzy",W/"r2d2xxxx", "c3piozzzz"`, ETag(Yes.weak, "r2d2xxxx"), No.allowWeak)); + assert( cacheMatch(`"xyzzy",W/"r2d2xxxx", "c3piozzzz"`, ETag(Yes.weak, "r2d2xxxx"), Yes.allowWeak)); + assert(!cacheMatch(`"xyzzy",W/"r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "r2d2xxxx"), No.allowWeak)); + assert( cacheMatch(`"xyzzy",W/"r2d2xxxx", "c3piozzzz"`, ETag(No.weak, "r2d2xxxx"), Yes.allowWeak)); +} diff --git a/web/vibe/web/rest.d b/web/vibe/web/rest.d index 3c354ab36d..56bb84b1f5 100644 --- a/web/vibe/web/rest.d +++ b/web/vibe/web/rest.d @@ -244,7 +244,7 @@ import vibe.web.auth : AuthInfo, handleAuthentication, handleAuthorization, isAu import std.algorithm : startsWith, endsWith; import std.range : isOutputRange; -import std.typecons : Nullable; +import std.typecons : No, Nullable, Yes; import std.typetuple : anySatisfy, Filter; import std.traits; @@ -469,23 +469,20 @@ URLRouter registerRestInterface(TImpl)(URLRouter router, TImpl instance, string HTTPServerRequestDelegate serveRestJSClient(I)(RestInterfaceSettings settings) if (is(I == interface)) { - import std.digest.md : md5Of; - import std.digest : toHexString; + import std.datetime.systime : SysTime; import std.array : appender; + import vibe.http.fileserver : ETag, handleCache; + auto app = appender!string(); generateRestJSClient!I(app, settings); - auto hash = app.data.md5Of.toHexString.idup; + ETag tag = ETag.md5(No.weak, app.data); void serve(HTTPServerRequest req, HTTPServerResponse res) { - if (auto pv = "If-None-Match" in res.headers) { - res.statusCode = HTTPStatus.notModified; - res.writeVoidBody(); + if (handleCache(req, res, tag, SysTime.init, "public")) return; - } - res.headers["Etag"] = hash; res.writeBody(app.data, "application/javascript; charset=UTF-8"); }