pod: read when zstd-wrapped

sisupod using zstd compression recognised by first bytes.
<doc>.sisupod: spine reads one before it writes one, so when the name
changes (from .zip to .ssiupod) every pod already published stays
readable.

A .sisupod is one zstd frame wrapping the archive spine already builds.
The reader unwraps the bytes and hands the same archive to the same
parser, so every guard downstream is untouched: entry names, per-entry
and total size, path depth, escape and symlinks.

Which container it is comes from the first four bytes (rather than the
name). A pod published as a plain .zip reads as it always did, a
.sisupod reads, and either one renamed reads too. The suffix is still
recognised, both spellings, for the argument and for a url.

libzstd is declared rather than bound: provides the whole surface of
fifteen extern C prototypes (there is nothing to generate and no
upstream tree to track, which is the arrangement sqlite3 already has).
dub links it with "libs": [ "zstd" ]; nix needs zstd.out rather than
zstd, whose default output is the binaries and carries no library at
all.

A frame declares its uncompressed size in its own header, and for a pod
fetched over https that number is attacker controlled. The declared size
is checked against a ceiling before a buffer is asked for, a frame that
will not declare one is refused, and what comes out is checked against
what was promised. The ceiling is the extraction limit the archive
reader already applies, so the two bounds agree.

Measured: output built from a .sisupod is byte identical to output built
from the same pod's .zip, 359 files over text, html, epub, odt and .ssp
for three documents, live-manual's ten languages included. A truncated
frame is refused and the document skipped.

  free_culture           2863504 -> 1099736   2.60x
  the_wealth_of_networks 4294022 -> 1172649   3.66x
  live-manual            6972418 ->  675970  10.31x

(assisted by Claude-Code)

6 files changed, 230 insertions, 10 deletions

diff --git a/src/sisudoc/ocda/abstraction/load.d b/src/sisudoc/ocda/abstraction/load.d
index ea387d1..2072aae 100644
--- a/src/sisudoc/ocda/abstraction/load.d
+++ b/src/sisudoc/ocda/abstraction/load.d
@@ -120,7 +120,12 @@ template spineAbstractionLoad() {
     if (_path.endsWith(".ocda.db"))                    { return AbstractionSource.ocda_db; }
     if (_path.endsWith(".ssp"))                        { return AbstractionSource.ssp; }
     if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; }
-    if (_path.endsWith(".zip"))                        { return AbstractionSource.pod_zip; }
+    /+ ↓ .sisupod is the pod archive spine writes; .zip is what it wrote before,
+         and every pod already published carries that name, so both are read.
+         What is inside decides how it is opened, not the name.
+    +/
+    if (_path.endsWith(".sisupod") || _path.endsWith(".zip"))
+                                                       { return AbstractionSource.pod_zip; }
     if (_path.endsWith(".db"))                         { return AbstractionSource.ocda_db; }
     return AbstractionSource.unknown;
   }
diff --git a/src/sisudoc/ocda/io_in/read_zip_pod.d b/src/sisudoc/ocda/io_in/read_zip_pod.d
index 49429be..d0eb65a 100644
--- a/src/sisudoc/ocda/io_in/read_zip_pod.d
+++ b/src/sisudoc/ocda/io_in/read_zip_pod.d
@@ -63,6 +63,7 @@ template spineExtractZipPod() {
   import std.regex;
   import std.stdio;
   import std.string : indexOf;
+  import sisudoc.ocda.zstd;
 
   /+ security limits for zip extraction +/
   enum size_t MAX_ENTRY_SIZE     = 50 * 1024 * 1024;   /+ 50 MB per entry +/
@@ -122,10 +123,29 @@ template spineExtractZipPod() {
     }
     /+ ↓ derive pod name from zip filename +/
     string zip_basename = zip_path.baseName.stripExtension;
-    /+ ↓ read and parse zip archive +/
+    /+ ↓ read and parse the pod archive.
+         a .sisupod is one zstd frame wrapping the archive spine writes, so
+         the bytes are unwrapped before the archive is parsed. Determined from
+         the first four bytes and not from the filename: a pod published as a
+         plain .zip, or one a reader has renamed, is read either way. Everything
+         after this point sees the same archive it always did, guards included.
+    +/
+    ubyte[] _archive_bytes;
+    try {
+      _archive_bytes = cast(ubyte[]) read(zip_path);
+      if (zstdIsFrame(_archive_bytes)) {
+        _archive_bytes = zstdDecompress(_archive_bytes, MAX_TOTAL_SIZE);
+      }
+    } catch (ZstdException ex) {
+      result.error_msg = "failed to decompress pod archive: " ~ zip_path ~ " - " ~ ex.msg;
+      return result;
+    } catch (Exception ex) {
+      result.error_msg = "error reading pod archive: " ~ zip_path ~ " - " ~ ex.msg;
+      return result;
+    }
     ZipArchive zip;
     try {
-      zip = new ZipArchive(read(zip_path));
+      zip = new ZipArchive(_archive_bytes);
     } catch (ZipException ex) {
       result.error_msg = "failed to read zip archive: " ~ zip_path ~ " - " ~ ex.msg;
       return result;
@@ -274,7 +294,7 @@ template spineExtractZipPod() {
        its own, carrying its images. A .ssp is not here: it describes its
        images without carrying them, so it would arrive without them.
   +/
-  static auto rgx_url_source = ctRegex!(`^https?://[a-zA-Z0-9._:/-]+([.]zip|[.]ocda[.]db)$`);
+  static auto rgx_url_source = ctRegex!(`^https?://[a-zA-Z0-9._:/-]+([.]sisupod|[.]zip|[.]ocda[.]db)$`);
 
   struct DownloadResult {
     string local_path;       /+ path to downloaded temp file +/
diff --git a/src/sisudoc/ocda/meta/rgx.d b/src/sisudoc/ocda/meta/rgx.d
index 4cc40ae..d3c9aa1 100644
--- a/src/sisudoc/ocda/meta/rgx.d
+++ b/src/sisudoc/ocda/meta/rgx.d
@@ -211,8 +211,8 @@ static template spineRgxIn() {
     static src_pth_sst_or_ssm                       = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`);
     static src_pth_pod_sst_or_ssm                   = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*?[.]ss[tm])$`);
     static src_pth_contents                         = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*)/pod[.]manifest$`);
-    static src_pth_zip                              = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]zip)$`);
-    static src_pth_types                            = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+[.]zip))$`);
+    static src_pth_zip                              = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod))$`);
+    static src_pth_types                            = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod)))$`);
     static src_fn                                   = ctRegex!(`^([/]?(?:[a-zA-Z0-9._-]+/)*)(?P<fn_src>(?P<fn_base>[a-zA-Z0-9._-]+)[.](?P<fn_src_suffix>ss[tm]))$`);
     static src_fn_master                            = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ssm)$`);
     static src_fn_find_inserts                      = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[im])$`);
diff --git a/src/sisudoc/ocda/meta/rgx_files.d b/src/sisudoc/ocda/meta/rgx_files.d
index b26422f..13c7e46 100644
--- a/src/sisudoc/ocda/meta/rgx_files.d
+++ b/src/sisudoc/ocda/meta/rgx_files.d
@@ -57,8 +57,8 @@ static template spineRgxFiles() {
     static src_pth_sst_or_ssm                       = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`);
     static src_pth_pod_sst_or_ssm                   = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*?[.]ss[tm])$`);
     static src_pth_contents                         = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*)/pod[.]manifest$`);
-    static src_pth_zip                              = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]zip)$`);
-    static src_pth_types                            = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+[.]zip))$`);
+    static src_pth_zip                              = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod))$`);
+    static src_pth_types                            = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod)))$`);
     static src_fn                                   = ctRegex!(`^([/]?(?:[a-zA-Z0-9._-]+/)*)(?P<fn_src>(?P<fn_base>[a-zA-Z0-9._-]+)[.](?P<fn_src_suffix>ss[tm]))$`);
     static src_fn_master                            = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ssm)$`);
     static src_fn_find_inserts                      = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[im])$`);
diff --git a/src/sisudoc/ocda/zstd.d b/src/sisudoc/ocda/zstd.d
new file mode 100644
index 0000000..2c8186f
--- /dev/null
+++ b/src/sisudoc/ocda/zstd.d
@@ -0,0 +1,195 @@
+/+
+- Name: SisuDoc Spine, Doc Reform [a part of]
+  - Description: documents, structuring, processing, publishing, search
+    - static content generator
+
+  - Author: Ralph Amissah
+    [ralph.amissah@gmail.com]
+
+  - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved.
+
+  - License: AGPL 3 or later:
+
+    Spine (SiSU), a framework for document structuring, publishing and
+    search
+
+    Copyright (C) Ralph Amissah
+
+    This program is free software: you can redistribute it and/or modify it
+    under the terms of the GNU AFERO General Public License as published by the
+    Free Software Foundation, either version 3 of the License, or (at your
+    option) any later version.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+    more details.
+
+    You should have received a copy of the GNU General Public License along with
+    this program. If not, see [https://www.gnu.org/licenses/].
+
+    If you have Internet connection, the latest version of the AGPL should be
+    available at these locations:
+    [https://www.fsf.org/licensing/licenses/agpl.html]
+    [https://www.gnu.org/licenses/agpl.html]
+
+  - Spine (by Doc Reform, related to SiSU) uses standard:
+    - docReform markup syntax
+      - standard SiSU markup syntax with modified headers and minor modifications
+    - docReform object numbering
+      - standard SiSU object citation numbering & system
+
+  - Homepages:
+    [https://www.sisudoc.org]
+    [https://www.doc-reform.org]
+
+  - Git
+    [https://git.sisudoc.org/]
+
++/
+/+
+  module zstd;<BR>
+  - the few libzstd entry points spine uses, and a buffer in / buffer out
+    wrapper over each direction<BR>
+  - one frame per call: what is compressed is a whole pod archive, not a
+    stream
++/
+module sisudoc.ocda.zstd;
+@safe:
+/+ ↓ libzstd, declared rather than bound by a generated header
+     Provides the fifteen declarations that spine needs (there is nothing
+     here to generate and no vendored tree to track): the C library is linked
+     (dub "libs": ["zstd"], nix buildInputs pkgs.zstd) and these
+     prototypes say what is being called. That is the same arrangement as
+     sqlite3, which is declared in the vendored d2sqlite3 and linked from the
+     system.
+     .
+     It sits under ocda/ rather than outputs/ because the pod *reader* needs it,
+     and the reader (ocda/io_in/read_zip_pod.d) is in the abstraction part,
+     which the outputs depend on and not the other way round.
+     A compression primitive is not an abstraction concern; the dependency
+     direction is what puts it here.
++/
+extern (C) @nogc nothrow private {
+  size_t ZSTD_compressBound(size_t srcSize);
+  size_t ZSTD_compress(void* dst, size_t dstCapacity,
+    const(void)* src, size_t srcSize, int compressionLevel);
+  size_t ZSTD_decompress(void* dst, size_t dstCapacity,
+    const(void)* src, size_t compressedSize);
+  ulong  ZSTD_getFrameContentSize(const(void)* src, size_t srcSize);
+  uint   ZSTD_isError(size_t code);
+  const(char)* ZSTD_getErrorName(size_t code);
+  int    ZSTD_minCLevel();
+  int    ZSTD_maxCLevel();
+  uint   ZSTD_versionNumber();
+}
+/+ ↓ what ZSTD_getFrameContentSize says when it cannot say +/
+private enum ulong ZSTD_CONTENTSIZE_UNKNOWN = ulong.max;       /+ (0ULL - 1) +/
+private enum ulong ZSTD_CONTENTSIZE_ERROR   = ulong.max - 1;   /+ (0ULL - 2) +/
+/+ ↓ the four bytes that open every zstd frame, 0xFD2FB528 little endian.
+     A pod is recognised by this rather than by its name, so that a pod
+     already published as a plain zip keeps being read whatever it is called.
++/
+enum ubyte[4] zstd_frame_magic = [0x28, 0xB5, 0x2F, 0xFD];
+/+ ↓ the ceiling on what one frame may be allowed to become.
+     A frame declares its uncompressed size in its header and that number is
+     attacker-controlled: a .sisupod fetched over https could claim a size no
+     machine can allocate. The declared size is checked against this before a
+     buffer is asked for, so a hostile declaration is refused rather than
+     attempted. It matches MAX_TOTAL_SIZE in read_zip_pod.d, which bounds what
+     the archive inside may extract to.
++/
+enum size_t ZSTD_MAX_FRAME_CONTENT = 500 * 1024 * 1024;
+/+ ↓ thrown rather than returned: every caller here wants the whole buffer or
+     nothing, and there is no partial result worth handing back
++/
+class ZstdException : Exception {
+  this(string _msg, string _file = __FILE__, size_t _line = __LINE__) @safe pure nothrow {
+    super(_msg, _file, _line);
+  }
+}
+/+ ↓ does this begin a zstd frame? +/
+bool zstdIsFrame(const(ubyte)[] _bytes) {
+  if (_bytes.length < zstd_frame_magic.length) {
+    return false;
+  }
+  foreach (_i, _b; zstd_frame_magic) {
+    if (_bytes[_i] != _b) {
+      return false;
+    }
+  }
+  return true;
+}
+/+ ↓ the library's version, for a run that wants to record what compressed a
+     pod. The bytes of a frame are a function of the library and the level, so
+     two zstd versions may compress the same input differently; nothing is
+     promised about archive bytes
++/
+@trusted uint zstdVersion() {
+  return ZSTD_versionNumber();
+}
+/+ ↓ the level clamped to what this library actually supports +/
+@trusted int zstdLevelClamped(int _level) {
+  int _min = ZSTD_minCLevel();
+  int _max = ZSTD_maxCLevel();
+  if (_level < _min) { return _min; }
+  if (_level > _max) { return _max; }
+  return _level;
+}
+private @trusted string _zstdError(size_t _code) {
+  import std.string : fromStringz;
+  return ZSTD_getErrorName(_code).fromStringz.idup;
+}
+/+ ↓ one buffer to one frame +/
+@trusted ubyte[] zstdCompress(const(ubyte)[] _src, int _level = 19) {
+  size_t _bound = ZSTD_compressBound(_src.length);
+  auto _dst = new ubyte[_bound];
+  size_t _got = ZSTD_compress(
+    _dst.ptr, _dst.length,
+    _src.ptr, _src.length,
+    zstdLevelClamped(_level),
+  );
+  if (ZSTD_isError(_got)) {
+    throw new ZstdException("zstd could not compress: " ~ _zstdError(_got));
+  }
+  return _dst[0 .. _got];
+}
+/+ ↓ one frame back to one buffer, refusing a frame that will not say how
+     large it is or that says something absurd
++/
+@trusted ubyte[] zstdDecompress(const(ubyte)[] _src,
+  size_t _max = ZSTD_MAX_FRAME_CONTENT
+) {
+  import std.conv : to;
+  if (!(zstdIsFrame(_src))) {
+    throw new ZstdException("not a zstd frame");
+  }
+  ulong _declared = ZSTD_getFrameContentSize(_src.ptr, _src.length);
+  if (_declared == ZSTD_CONTENTSIZE_ERROR) {
+    throw new ZstdException("zstd frame header will not read");
+  }
+  if (_declared == ZSTD_CONTENTSIZE_UNKNOWN) {
+    /+ every frame spine writes is one whole buffer, so its size is always
+       declared; a frame without one was made by something else
+    +/
+    throw new ZstdException("zstd frame does not declare its size");
+  }
+  if (_declared > _max) {
+    throw new ZstdException("zstd frame declares "
+      ~ _declared.to!string ~ " bytes, over the "
+      ~ _max.to!string ~ " allowed");
+  }
+  auto _dst = new ubyte[_declared.to!size_t];
+  size_t _got = ZSTD_decompress(
+    _dst.ptr, _dst.length,
+    _src.ptr, _src.length,
+  );
+  if (ZSTD_isError(_got)) {
+    throw new ZstdException("zstd could not decompress: " ~ _zstdError(_got));
+  }
+  if (_got != _dst.length) {
+    throw new ZstdException("zstd frame gave " ~ _got.to!string
+      ~ " bytes where its header declared " ~ _dst.length.to!string);
+  }
+  return _dst;
+}
diff --git a/src/sisudoc/outputs/io_out/rgx.d b/src/sisudoc/outputs/io_out/rgx.d
index 7ce87e0..cd7a8da 100644
--- a/src/sisudoc/outputs/io_out/rgx.d
+++ b/src/sisudoc/outputs/io_out/rgx.d
@@ -66,8 +66,8 @@ static template spineRgxOut() {
     static src_pth_sst_or_ssm                       = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`);
     static src_pth_pod_sst_or_ssm                   = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*?[.]ss[tm])$`);
     static src_pth_contents                         = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*)/pod[.]manifest$`);
-    static src_pth_zip                              = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]zip)$`);
-    static src_pth_types                            = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+[.]zip))$`);
+    static src_pth_zip                              = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod))$`);
+    static src_pth_types                            = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod)))$`);
     static src_fn                                   = ctRegex!(`^([/]?(?:[a-zA-Z0-9._-]+/)*)(?P<fn_src>(?P<fn_base>[a-zA-Z0-9._-]+)[.](?P<fn_src_suffix>ss[tm]))$`);
     static src_fn_master                            = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ssm)$`);
     static src_fn_find_inserts                      = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[im])$`);