/+ - Name: SisuDoc Spine - Description: part of SiSU spine (Doc Reform): for document structuring, processing, publishing and search - Author: Ralph Amissah [ralph.amissah@gmail.com] - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later, for everything in this repository unless a file or a directory specifies otherwise. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License (AGPLv3) as published by the Free Software Foundation (FSF), either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see [https://www.gnu.org/licenses/]. If you have Internet connection, the latest version of the AGPL should be available at: [https://www.gnu.org/licenses/agpl-3.0.html] - Spine (by Doc Reform, related to SiSU) uses standard: - docReform markup syntax - standard SiSU markup syntax with modified headers and minor modifications - docReform object numbering - standard SiSU object citation numbering & system - Homepages: [https://www.sisudoc.org] [https://www.doc-reform.org] - Git [https://git.sisudoc.org/] +/ /+ module zstd;
- the few libzstd entry points spine uses, and a buffer in / buffer out wrapper over each direction
- one frame per call: what is compressed is a whole pod archive, not a stream +/ module sisudoc.ocda.zstd; @safe: /+ ↓ libzstd, declared rather than bound by a generated header Provides the fifteen declarations that spine needs (there is nothing here to generate and no vendored tree to track): the C library is linked (dub "libs": ["zstd"], nix buildInputs pkgs.zstd) and these prototypes say what is being called. That is the same arrangement as sqlite3, which is declared in the vendored d2sqlite3 and linked from the system. . It sits under ocda/ rather than outputs/ because the pod *reader* needs it, and the reader (ocda/io_in/read_zip_pod.d) is in the abstraction part, which the outputs depend on and not the other way round. A compression primitive is not an abstraction concern; the dependency direction is what puts it here. +/ extern (C) @nogc nothrow private { size_t ZSTD_compressBound(size_t srcSize); size_t ZSTD_compress(void* dst, size_t dstCapacity, const(void)* src, size_t srcSize, int compressionLevel); size_t ZSTD_decompress(void* dst, size_t dstCapacity, const(void)* src, size_t compressedSize); ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize); uint ZSTD_isError(size_t code); const(char)* ZSTD_getErrorName(size_t code); int ZSTD_minCLevel(); int ZSTD_maxCLevel(); uint ZSTD_versionNumber(); } /+ ↓ what ZSTD_getFrameContentSize says when it cannot say +/ private enum ulong ZSTD_CONTENTSIZE_UNKNOWN = ulong.max; /+ (0ULL - 1) +/ private enum ulong ZSTD_CONTENTSIZE_ERROR = ulong.max - 1; /+ (0ULL - 2) +/ /+ ↓ the four bytes that open every zstd frame, 0xFD2FB528 little endian. A pod is recognised by this rather than by its name, so that a pod already published as a plain zip keeps being read whatever it is called. +/ enum ubyte[4] zstd_frame_magic = [0x28, 0xB5, 0x2F, 0xFD]; /+ ↓ the ceiling on what one frame may be allowed to become. A frame declares its uncompressed size in its header and that number is attacker-controlled: a .sisupod fetched over https could claim a size no machine can allocate. The declared size is checked against this before a buffer is asked for, so a hostile declaration is refused rather than attempted. It matches MAX_TOTAL_SIZE in read_zip_pod.d, which bounds what the archive inside may extract to. +/ enum size_t ZSTD_MAX_FRAME_CONTENT = 500 * 1024 * 1024; /+ ↓ thrown rather than returned: every caller here wants the whole buffer or nothing, and there is no partial result worth handing back +/ class ZstdException : Exception { this(string _msg, string _file = __FILE__, size_t _line = __LINE__) @safe pure nothrow { super(_msg, _file, _line); } } /+ ↓ does this begin a zstd frame? +/ bool zstdIsFrame(const(ubyte)[] _bytes) { if (_bytes.length < zstd_frame_magic.length) { return false; } foreach (_i, _b; zstd_frame_magic) { if (_bytes[_i] != _b) { return false; } } return true; } /+ ↓ the library's version, for a run that wants to record what compressed a pod. The bytes of a frame are a function of the library and the level, so two zstd versions may compress the same input differently; nothing is promised about archive bytes +/ @trusted uint zstdVersion() { return ZSTD_versionNumber(); } /+ ↓ the level clamped to what this library actually supports +/ @trusted int zstdLevelClamped(int _level) { int _min = ZSTD_minCLevel(); int _max = ZSTD_maxCLevel(); if (_level < _min) { return _min; } if (_level > _max) { return _max; } return _level; } private @trusted string _zstdError(size_t _code) { import std.string : fromStringz; return ZSTD_getErrorName(_code).fromStringz.idup; } /+ ↓ one buffer to one frame +/ @trusted ubyte[] zstdCompress(const(ubyte)[] _src, int _level = 19) { size_t _bound = ZSTD_compressBound(_src.length); auto _dst = new ubyte[_bound]; size_t _got = ZSTD_compress( _dst.ptr, _dst.length, _src.ptr, _src.length, zstdLevelClamped(_level), ); if (ZSTD_isError(_got)) { throw new ZstdException("zstd could not compress: " ~ _zstdError(_got)); } return _dst[0 .. _got]; } /+ ↓ one frame back to one buffer, refusing a frame that will not say how large it is or that says something absurd +/ @trusted ubyte[] zstdDecompress(const(ubyte)[] _src, size_t _max = ZSTD_MAX_FRAME_CONTENT ) { import std.conv : to; if (!(zstdIsFrame(_src))) { throw new ZstdException("not a zstd frame"); } ulong _declared = ZSTD_getFrameContentSize(_src.ptr, _src.length); if (_declared == ZSTD_CONTENTSIZE_ERROR) { throw new ZstdException("zstd frame header will not read"); } if (_declared == ZSTD_CONTENTSIZE_UNKNOWN) { /+ every frame spine writes is one whole buffer, so its size is always declared; a frame without one was made by something else +/ throw new ZstdException("zstd frame does not declare its size"); } if (_declared > _max) { throw new ZstdException("zstd frame declares " ~ _declared.to!string ~ " bytes, over the " ~ _max.to!string ~ " allowed"); } auto _dst = new ubyte[_declared.to!size_t]; size_t _got = ZSTD_decompress( _dst.ptr, _dst.length, _src.ptr, _src.length, ); if (ZSTD_isError(_got)) { throw new ZstdException("zstd could not decompress: " ~ _zstdError(_got)); } if (_got != _dst.length) { throw new ZstdException("zstd frame gave " ~ _got.to!string ~ " bytes where its header declared " ~ _dst.length.to!string); } return _dst; }