/+
- Name: SisuDoc Spine
- Description: part of SiSU spine (Doc Reform):
for document structuring, processing, publishing and search
- Author: Ralph Amissah
[ralph.amissah@gmail.com]
- Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah,
All Rights Reserved.
- License: AGPL 3 or later, for everything in this repository unless a file or
a directory specifies otherwise.
This program is free software: you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License (AGPLv3) as
published by the Free Software Foundation (FSF), either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see [https://www.gnu.org/licenses/].
If you have Internet connection, the latest version of the AGPL should be
available at: [https://www.gnu.org/licenses/agpl-3.0.html]
- Spine (by Doc Reform, related to SiSU) uses standard:
- docReform markup syntax
- standard SiSU markup syntax with modified headers and minor modifications
- docReform object numbering
- standard SiSU object citation numbering & system
- Homepages:
[https://www.sisudoc.org]
[https://www.doc-reform.org]
- Git
[https://git.sisudoc.org/]
+/
/+
module zstd;
- the few libzstd entry points spine uses, and a buffer in / buffer out
wrapper over each direction
- one frame per call: what is compressed is a whole pod archive, not a
stream
+/
module sisudoc.ocda.zstd;
@safe:
/+ ↓ libzstd, declared rather than bound by a generated header
Provides the fifteen declarations that spine needs (there is nothing
here to generate and no vendored tree to track): the C library is linked
(dub "libs": ["zstd"], nix buildInputs pkgs.zstd) and these
prototypes say what is being called. That is the same arrangement as
sqlite3, which is declared in the vendored d2sqlite3 and linked from the
system.
.
It sits under ocda/ rather than outputs/ because the pod *reader* needs it,
and the reader (ocda/io_in/read_zip_pod.d) is in the abstraction part,
which the outputs depend on and not the other way round.
A compression primitive is not an abstraction concern; the dependency
direction is what puts it here.
+/
extern (C) @nogc nothrow private {
size_t ZSTD_compressBound(size_t srcSize);
size_t ZSTD_compress(void* dst, size_t dstCapacity,
const(void)* src, size_t srcSize, int compressionLevel);
size_t ZSTD_decompress(void* dst, size_t dstCapacity,
const(void)* src, size_t compressedSize);
ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize);
uint ZSTD_isError(size_t code);
const(char)* ZSTD_getErrorName(size_t code);
int ZSTD_minCLevel();
int ZSTD_maxCLevel();
uint ZSTD_versionNumber();
}
/+ ↓ what ZSTD_getFrameContentSize says when it cannot say +/
private enum ulong ZSTD_CONTENTSIZE_UNKNOWN = ulong.max; /+ (0ULL - 1) +/
private enum ulong ZSTD_CONTENTSIZE_ERROR = ulong.max - 1; /+ (0ULL - 2) +/
/+ ↓ the four bytes that open every zstd frame, 0xFD2FB528 little endian.
A pod is recognised by this rather than by its name, so that a pod
already published as a plain zip keeps being read whatever it is called.
+/
enum ubyte[4] zstd_frame_magic = [0x28, 0xB5, 0x2F, 0xFD];
/+ ↓ the ceiling on what one frame may be allowed to become.
A frame declares its uncompressed size in its header and that number is
attacker-controlled: a .sisupod fetched over https could claim a size no
machine can allocate. The declared size is checked against this before a
buffer is asked for, so a hostile declaration is refused rather than
attempted. It matches MAX_TOTAL_SIZE in read_zip_pod.d, which bounds what
the archive inside may extract to.
+/
enum size_t ZSTD_MAX_FRAME_CONTENT = 500 * 1024 * 1024;
/+ ↓ thrown rather than returned: every caller here wants the whole buffer or
nothing, and there is no partial result worth handing back
+/
class ZstdException : Exception {
this(string _msg, string _file = __FILE__, size_t _line = __LINE__) @safe pure nothrow {
super(_msg, _file, _line);
}
}
/+ ↓ does this begin a zstd frame? +/
bool zstdIsFrame(const(ubyte)[] _bytes) {
if (_bytes.length < zstd_frame_magic.length) {
return false;
}
foreach (_i, _b; zstd_frame_magic) {
if (_bytes[_i] != _b) {
return false;
}
}
return true;
}
/+ ↓ the library's version, for a run that wants to record what compressed a
pod. The bytes of a frame are a function of the library and the level, so
two zstd versions may compress the same input differently; nothing is
promised about archive bytes
+/
@trusted uint zstdVersion() {
return ZSTD_versionNumber();
}
/+ ↓ the level clamped to what this library actually supports +/
@trusted int zstdLevelClamped(int _level) {
int _min = ZSTD_minCLevel();
int _max = ZSTD_maxCLevel();
if (_level < _min) { return _min; }
if (_level > _max) { return _max; }
return _level;
}
private @trusted string _zstdError(size_t _code) {
import std.string : fromStringz;
return ZSTD_getErrorName(_code).fromStringz.idup;
}
/+ ↓ one buffer to one frame +/
@trusted ubyte[] zstdCompress(const(ubyte)[] _src, int _level = 19) {
size_t _bound = ZSTD_compressBound(_src.length);
auto _dst = new ubyte[_bound];
size_t _got = ZSTD_compress(
_dst.ptr, _dst.length,
_src.ptr, _src.length,
zstdLevelClamped(_level),
);
if (ZSTD_isError(_got)) {
throw new ZstdException("zstd could not compress: " ~ _zstdError(_got));
}
return _dst[0 .. _got];
}
/+ ↓ one frame back to one buffer, refusing a frame that will not say how
large it is or that says something absurd
+/
@trusted ubyte[] zstdDecompress(const(ubyte)[] _src,
size_t _max = ZSTD_MAX_FRAME_CONTENT
) {
import std.conv : to;
if (!(zstdIsFrame(_src))) {
throw new ZstdException("not a zstd frame");
}
ulong _declared = ZSTD_getFrameContentSize(_src.ptr, _src.length);
if (_declared == ZSTD_CONTENTSIZE_ERROR) {
throw new ZstdException("zstd frame header will not read");
}
if (_declared == ZSTD_CONTENTSIZE_UNKNOWN) {
/+ every frame spine writes is one whole buffer, so its size is always
declared; a frame without one was made by something else
+/
throw new ZstdException("zstd frame does not declare its size");
}
if (_declared > _max) {
throw new ZstdException("zstd frame declares "
~ _declared.to!string ~ " bytes, over the "
~ _max.to!string ~ " allowed");
}
auto _dst = new ubyte[_declared.to!size_t];
size_t _got = ZSTD_decompress(
_dst.ptr, _dst.length,
_src.ptr, _src.length,
);
if (ZSTD_isError(_got)) {
throw new ZstdException("zstd could not decompress: " ~ _zstdError(_got));
}
if (_got != _dst.length) {
throw new ZstdException("zstd frame gave " ~ _got.to!string
~ " bytes where its header declared " ~ _dst.length.to!string);
}
return _dst;
}