From f89f0152a913cec14c6ee688a4f1b2ac7c4758ff Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Mon, 21 Sep 2026 08:58:06 -0400 Subject: pod: read when zstd-wrapped sisupod using zstd compression recognised by first bytes. .sisupod: spine reads one before it writes one, so when the name changes (from .zip to .ssiupod) every pod already published stays readable. A .sisupod is one zstd frame wrapping the archive spine already builds. The reader unwraps the bytes and hands the same archive to the same parser, so every guard downstream is untouched: entry names, per-entry and total size, path depth, escape and symlinks. Which container it is comes from the first four bytes (rather than the name). A pod published as a plain .zip reads as it always did, a .sisupod reads, and either one renamed reads too. The suffix is still recognised, both spellings, for the argument and for a url. libzstd is declared rather than bound: provides the whole surface of fifteen extern C prototypes (there is nothing to generate and no upstream tree to track, which is the arrangement sqlite3 already has). dub links it with "libs": [ "zstd" ]; nix needs zstd.out rather than zstd, whose default output is the binaries and carries no library at all. A frame declares its uncompressed size in its own header, and for a pod fetched over https that number is attacker controlled. The declared size is checked against a ceiling before a buffer is asked for, a frame that will not declare one is refused, and what comes out is checked against what was promised. The ceiling is the extraction limit the archive reader already applies, so the two bounds agree. Measured: output built from a .sisupod is byte identical to output built from the same pod's .zip, 359 files over text, html, epub, odt and .ssp for three documents, live-manual's ten languages included. A truncated frame is refused and the document skipped. free_culture 2863504 -> 1099736 2.60x the_wealth_of_networks 4294022 -> 1172649 3.66x live-manual 6972418 -> 675970 10.31x (assisted by Claude-Code) --- org/compression_zstd.org | 200 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 org/compression_zstd.org (limited to 'org/compression_zstd.org') diff --git a/org/compression_zstd.org b/org/compression_zstd.org new file mode 100644 index 0000000..bcb67f3 --- /dev/null +++ b/org/compression_zstd.org @@ -0,0 +1,200 @@ +-*- mode: org -*- +#+TITLE: sisudoc spine (doc_reform) output zip +#+DESCRIPTION: documents - structuring, publishing in multiple formats & search +#+FILETAGS: :spine:zip: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +- [[./doc-reform.org][doc-reform.org]] [[./][org/]] +- [[./output_hub.org][output_hub]] + +* _zstd_ :module:spine:compression: + +#+HEADER: :tangle "../src/sisudoc/ocda/zstd.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<> +/+ + module zstd;
+ - the few libzstd entry points spine uses, and a buffer in / buffer out + wrapper over each direction
+ - one frame per call: what is compressed is a whole pod archive, not a + stream ++/ +module sisudoc.ocda.zstd; +@safe: +/+ ↓ libzstd, declared rather than bound by a generated header + Provides the fifteen declarations that spine needs (there is nothing + here to generate and no vendored tree to track): the C library is linked + (dub "libs": ["zstd"], nix buildInputs pkgs.zstd) and these + prototypes say what is being called. That is the same arrangement as + sqlite3, which is declared in the vendored d2sqlite3 and linked from the + system. + . + It sits under ocda/ rather than outputs/ because the pod *reader* needs it, + and the reader (ocda/io_in/read_zip_pod.d) is in the abstraction part, + which the outputs depend on and not the other way round. + A compression primitive is not an abstraction concern; the dependency + direction is what puts it here. ++/ +extern (C) @nogc nothrow private { + size_t ZSTD_compressBound(size_t srcSize); + size_t ZSTD_compress(void* dst, size_t dstCapacity, + const(void)* src, size_t srcSize, int compressionLevel); + size_t ZSTD_decompress(void* dst, size_t dstCapacity, + const(void)* src, size_t compressedSize); + ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize); + uint ZSTD_isError(size_t code); + const(char)* ZSTD_getErrorName(size_t code); + int ZSTD_minCLevel(); + int ZSTD_maxCLevel(); + uint ZSTD_versionNumber(); +} +/+ ↓ what ZSTD_getFrameContentSize says when it cannot say +/ +private enum ulong ZSTD_CONTENTSIZE_UNKNOWN = ulong.max; /+ (0ULL - 1) +/ +private enum ulong ZSTD_CONTENTSIZE_ERROR = ulong.max - 1; /+ (0ULL - 2) +/ +/+ ↓ the four bytes that open every zstd frame, 0xFD2FB528 little endian. + A pod is recognised by this rather than by its name, so that a pod + already published as a plain zip keeps being read whatever it is called. ++/ +enum ubyte[4] zstd_frame_magic = [0x28, 0xB5, 0x2F, 0xFD]; +/+ ↓ the ceiling on what one frame may be allowed to become. + A frame declares its uncompressed size in its header and that number is + attacker-controlled: a .sisupod fetched over https could claim a size no + machine can allocate. The declared size is checked against this before a + buffer is asked for, so a hostile declaration is refused rather than + attempted. It matches MAX_TOTAL_SIZE in read_zip_pod.d, which bounds what + the archive inside may extract to. ++/ +enum size_t ZSTD_MAX_FRAME_CONTENT = 500 * 1024 * 1024; +/+ ↓ thrown rather than returned: every caller here wants the whole buffer or + nothing, and there is no partial result worth handing back ++/ +class ZstdException : Exception { + this(string _msg, string _file = __FILE__, size_t _line = __LINE__) @safe pure nothrow { + super(_msg, _file, _line); + } +} +/+ ↓ does this begin a zstd frame? +/ +bool zstdIsFrame(const(ubyte)[] _bytes) { + if (_bytes.length < zstd_frame_magic.length) { + return false; + } + foreach (_i, _b; zstd_frame_magic) { + if (_bytes[_i] != _b) { + return false; + } + } + return true; +} +/+ ↓ the library's version, for a run that wants to record what compressed a + pod. The bytes of a frame are a function of the library and the level, so + two zstd versions may compress the same input differently; nothing is + promised about archive bytes ++/ +@trusted uint zstdVersion() { + return ZSTD_versionNumber(); +} +/+ ↓ the level clamped to what this library actually supports +/ +@trusted int zstdLevelClamped(int _level) { + int _min = ZSTD_minCLevel(); + int _max = ZSTD_maxCLevel(); + if (_level < _min) { return _min; } + if (_level > _max) { return _max; } + return _level; +} +private @trusted string _zstdError(size_t _code) { + import std.string : fromStringz; + return ZSTD_getErrorName(_code).fromStringz.idup; +} +/+ ↓ one buffer to one frame +/ +@trusted ubyte[] zstdCompress(const(ubyte)[] _src, int _level = 19) { + size_t _bound = ZSTD_compressBound(_src.length); + auto _dst = new ubyte[_bound]; + size_t _got = ZSTD_compress( + _dst.ptr, _dst.length, + _src.ptr, _src.length, + zstdLevelClamped(_level), + ); + if (ZSTD_isError(_got)) { + throw new ZstdException("zstd could not compress: " ~ _zstdError(_got)); + } + return _dst[0 .. _got]; +} +/+ ↓ one frame back to one buffer, refusing a frame that will not say how + large it is or that says something absurd ++/ +@trusted ubyte[] zstdDecompress(const(ubyte)[] _src, + size_t _max = ZSTD_MAX_FRAME_CONTENT +) { + import std.conv : to; + if (!(zstdIsFrame(_src))) { + throw new ZstdException("not a zstd frame"); + } + ulong _declared = ZSTD_getFrameContentSize(_src.ptr, _src.length); + if (_declared == ZSTD_CONTENTSIZE_ERROR) { + throw new ZstdException("zstd frame header will not read"); + } + if (_declared == ZSTD_CONTENTSIZE_UNKNOWN) { + /+ every frame spine writes is one whole buffer, so its size is always + declared; a frame without one was made by something else + +/ + throw new ZstdException("zstd frame does not declare its size"); + } + if (_declared > _max) { + throw new ZstdException("zstd frame declares " + ~ _declared.to!string ~ " bytes, over the " + ~ _max.to!string ~ " allowed"); + } + auto _dst = new ubyte[_declared.to!size_t]; + size_t _got = ZSTD_decompress( + _dst.ptr, _dst.length, + _src.ptr, _src.length, + ); + if (ZSTD_isError(_got)) { + throw new ZstdException("zstd could not decompress: " ~ _zstdError(_got)); + } + if (_got != _dst.length) { + throw new ZstdException("zstd frame gave " ~ _got.to!string + ~ " bytes where its header declared " ~ _dst.length.to!string); + } + return _dst; +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ -- cgit v1.2.3