Precompress middleware for FileSystem Writer

Changes

5 changed files (+217/-1)

• deno.lock
  • @@ -1,6 +1,10 @@
    { "version": "3", + "redirects": { + "https://deno.land/std/encoding/base64.ts": "https://deno.land/std@0.221.0/encoding/base64.ts" + }, "remote": { + "https://deno.land/std@0.159.0/encoding/ascii85.ts": "f2b9cb8da1a55b3f120d3de2e78ac993183a4fd00dfa9cb03b51cf3a75bc0baa", "https://deno.land/std@0.221.0/assert/_constants.ts": "a271e8ef5a573f1df8e822a6eb9d09df064ad66a4390f21b3e31f820a38e0975", "https://deno.land/std@0.221.0/assert/_diff.ts": "4bf42969aa8b1a33aaf23eb8e478b011bfaa31b82d85d2ff4b5c4662d8780d2b", "https://deno.land/std@0.221.0/assert/_format.ts": "0ba808961bf678437fb486b56405b6fefad2cf87b5809667c781ddee8c32aff4",
  • @@ -33,6 +37,7 @@ "https://deno.land/std@0.221.0/assert/mod.ts": "7e41449e77a31fef91534379716971bebcfc12686e143d38ada5438e04d4a90e",
    "https://deno.land/std@0.221.0/assert/unimplemented.ts": "47ca67d1c6dc53abd0bd729b71a31e0825fc452dbcd4fde4ca06789d5644e7fd", "https://deno.land/std@0.221.0/assert/unreachable.ts": "3670816a4ab3214349acb6730e3e6f5299021234657eefe05b48092f3848c270", "https://deno.land/std@0.221.0/encoding/_util.ts": "beacef316c1255da9bc8e95afb1fa56ed69baef919c88dc06ae6cb7a6103d376", + "https://deno.land/std@0.221.0/encoding/base64.ts": "8ccae67a1227b875340a8582ff707f37b131df435b07080d3bb58e07f5f97807", "https://deno.land/std@0.221.0/encoding/hex.ts": "e939f50d55be48a1fe42fecaaecdb54353df38e831c47f374be7e6fdbe61510e", "https://deno.land/std@0.221.0/fmt/colors.ts": "d239d84620b921ea520125d778947881f62c50e78deef2657073840b8af9559a", "https://deno.land/std@0.221.0/front_matter/_formats.ts": "9a8ac1524f93b3ae093bd66864a49fc0088037920c6d60863da136d10f92e04d",
  • @@ -146,6 +151,13 @@ "https://deno.land/std@0.221.0/yaml/schema/extended.ts": "996da59626409047b5c1a2d68bdbeead43914cedede47c5923e80ae4febe7d24",
    "https://deno.land/std@0.221.0/yaml/schema/failsafe.ts": "24b2b630cef6fcce7de6d29db651523b0f49e5691d690931c42ecf4823837fdb", "https://deno.land/std@0.221.0/yaml/schema/json.ts": "0fb9268282d266c24d963e75ef77f51accbbb74f40713a99e83ad621a81bc9ae", "https://deno.land/std@0.221.0/yaml/schema/mod.ts": "9bf7ff80c2a246f781bdcab979211d0389760831a974cf5883bf2016567e3507", - "https://deno.land/std@0.221.0/yaml/type.ts": "708dde5f20b01cc1096489b7155b6af79a217d585afb841128e78c3c2391eb5c" + "https://deno.land/std@0.221.0/yaml/type.ts": "708dde5f20b01cc1096489b7155b6af79a217d585afb841128e78c3c2391eb5c", + "https://deno.land/x/brotli@0.1.7/mod.ts": "08b913e51488b6e7fa181f2814b9ad087fdb5520041db0368f8156bfa45fd73e", + "https://deno.land/x/brotli@0.1.7/wasm.js": "77771b89e89ec7ff6e3e0939a7fb4f9b166abec3504cec0532ad5c127d6f35d2", + "https://deno.land/x/lz4@v0.1.2/mod.ts": "4decfc1a3569d03fd1813bd39128b71c8f082850fe98ecfdde20025772916582", + "https://deno.land/x/lz4@v0.1.2/wasm.js": "b9c65605327ba273f0c76a6dc596ec534d4cda0f0225d7a94ebc606782319e46", + "https://deno.land/x/zstd_wasm@0.0.21/deno/zstd.deno.js": "189187a43a4bdc8d695ff2f6525c8d3461cbd715333123e502a6a3bd84c272b4", + "https://deno.land/x/zstd_wasm@0.0.21/deno/zstd.encoded.wasm.ts": "87e04064657bf5082b47a709931f8eb3f0d2fe537a8c650cfb5c997b7b460d07", + "https://deno.land/x/zstd_wasm@0.0.21/deno/zstd.ts": "ba3889ee2b11f73ae7c6192da2851ec5f037b5ae7e3afa4c9129f33e856616ec" } }
• deps/deno.land/x/brotli/mod.ts (new)
  • @@ -0,0 +1,5 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + export * from "https://deno.land/x/brotli@0.1.7/mod.ts";
• deps/deno.land/x/zstd_wasm/deno/zstd.ts (new)
  • @@ -0,0 +1,5 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + export * from "https://deno.land/x/zstd_wasm@0.0.21/deno/zstd.ts";
• filesystem_writer/precompress.test.ts (new)
  • @@ -0,0 +1,55 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + import { assertExists, assertLess } from "../deps/deno.land/std/assert/mod.ts"; + + import { MemoryFsWriter } from "./memory_fs.ts"; + import { precompress } from "./precompress.ts"; + + Deno.test("Should save precompress files along with the original file", async () => { + const mw = new MemoryFsWriter(); + + const writer = precompress()(mw); + + const content = new TextEncoder().encode(` + const loop = () => { + const delta = getDelta(); + someMainThreadFunction(); + + const input = getUserInput() + + if (input.leftButton.pressed) { + player.x -= delta * player.speed; + } + + if (input.rightButton.pressed) { + player.x += delta * player.speed; + } + + requestAnimationFrame(loop); + } + + requestAnimationFrame(() => { + loop() + }) + `); + + await writer.write(["foo.js"], content); + + const original = mw.get(["foo.js"]); + assertExists(original); + + const gzip = mw.get(["foo.js.gz"]); + assertExists(gzip); + + const brotli = mw.get(["foo.js.br"]); + assertExists(brotli); + + const zstd = mw.get(["foo.js.zst"]); + assertExists(zstd); + + assertLess(gzip.byteLength, original.byteLength); + assertLess(brotli.byteLength, original.byteLength); + assertLess(zstd.byteLength, original.byteLength); + });
• filesystem_writer/precompress.ts (new)
  • @@ -0,0 +1,139 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + import * as brotli from "../deps/deno.land/x/brotli/mod.ts"; + import * as zstd from "../deps/deno.land/x/zstd_wasm/deno/zstd.ts"; + + import type { FileSystemWriter } from "./interface.ts"; + + type WriterFactory = (childWriter: FileSystemWriter) => FileSystemWriter; + + function DEFAULT_IGNORE_FN(path: readonly string[]): boolean { + const filename = path[path.length - 1]; + if (!filename) { + // Something went wrong, but this middleware is not responsible for this case. + return false; + } + + return !(/\.(js|css|html)$/.test(filename)); + } + + /** + * @param ext - File extension, *including* leading dot. + */ + function appendExt(path: readonly string[], ext: string): readonly string[] { + const filename = path[path.length - 1]; + const dirname = path.slice(0, -1); + + if (!filename) { + // Something went wrong, but this middleware is not responsible for this case. + return path; + } + + return [...dirname, filename + ext]; + } + + // TODO: Support compression options. + // TODO: Enable users to discard compressed file depends on compression ratio. + // `discard?(compressed: Uint8Array, original: Uint8Array): boolean + interface PrecompressOptions { + /** + * Whether to enable gzip compression. + * @default true + */ + gzip?: boolean; + + /** + * Whether to enable brotli compression. + * @default true + */ + brotli?: boolean; + + /** + * Whether to enable Zstandard compression. + * @default true + */ + zstd?: boolean; + + /** + * Minimum byte length to create compressed files. + * If the source file size is below this threshold, this middleware does not compress + * the file. + * By default, this middleware compresses regardless of the file size. + * @default 0 + */ + minBytes?: number; + + /** + * If this callback function returned `true`, this middleware skips compression + * for the file. + * By default, this middleware skips files whose filename does not end with `.js`, + * `.html`, or `.css`. + */ + ignore?(path: readonly string[], content: Uint8Array): boolean; + } + + /** + * Wraps the given FileSystem Writer and returns a new FileSystem Writer + * that produces additional gzip/brotli/Zstandard compressed files along with + * the original file. + * + * Each compressed files have ".gz", ".br", ".zst" suffix, respectively. + * This works well with Caddy's `precompressed` [0] directive. + * + * [0]: https://caddyserver.com/docs/caddyfile/directives/file_server#precompressed + */ + export function precompress( + { + gzip: enableGzip = true, + brotli: enableBrotli = true, + zstd: enableZstd = true, + minBytes = 0, + ignore = DEFAULT_IGNORE_FN, + }: PrecompressOptions = {}, + ): WriterFactory { + let isZstdInitCompleted = false; + + return (childWriter) => { + return { + async write(path, content) { + if (ignore(path, content) || content.byteLength < minBytes) { + return childWriter.write(path, content); + } + + if (enableGzip) { + const blob = new Blob([content]); + const stream = blob.stream().pipeThrough( + new CompressionStream("gzip"), + ); + await childWriter.write( + appendExt(path, ".gz"), + new Uint8Array(await new Response(stream).arrayBuffer()), + ); + } + + if (enableBrotli) { + await childWriter.write( + appendExt(path, ".br"), + brotli.compress(content), + ); + } + + if (enableZstd) { + if (!isZstdInitCompleted) { + await zstd.init(); + isZstdInitCompleted = true; + } + + await childWriter.write( + appendExt(path, ".zst"), + zstd.compress(content, 10), + ); + } + + return childWriter.write(path, content); + }, + }; + }; + }