diff options
Diffstat (limited to 'org/compression_zstd.org')
| -rw-r--r-- | org/compression_zstd.org | 200 |
1 files changed, 200 insertions, 0 deletions
diff --git a/org/compression_zstd.org b/org/compression_zstd.org new file mode 100644 index 0000000..bcb67f3 --- /dev/null +++ b/org/compression_zstd.org @@ -0,0 +1,200 @@ +-*- mode: org -*- +#+TITLE: sisudoc spine (doc_reform) output zip +#+DESCRIPTION: documents - structuring, publishing in multiple formats & search +#+FILETAGS: :spine:zip: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +- [[./doc-reform.org][doc-reform.org]] [[./][org/]] +- [[./output_hub.org][output_hub]] + +* _zstd_ :module:spine:compression: + +#+HEADER: :tangle "../src/sisudoc/ocda/zstd.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/+ + module zstd;<BR> + - the few libzstd entry points spine uses, and a buffer in / buffer out + wrapper over each direction<BR> + - one frame per call: what is compressed is a whole pod archive, not a + stream ++/ +module sisudoc.ocda.zstd; +@safe: +/+ ↓ libzstd, declared rather than bound by a generated header + Provides the fifteen declarations that spine needs (there is nothing + here to generate and no vendored tree to track): the C library is linked + (dub "libs": ["zstd"], nix buildInputs pkgs.zstd) and these + prototypes say what is being called. That is the same arrangement as + sqlite3, which is declared in the vendored d2sqlite3 and linked from the + system. + . + It sits under ocda/ rather than outputs/ because the pod *reader* needs it, + and the reader (ocda/io_in/read_zip_pod.d) is in the abstraction part, + which the outputs depend on and not the other way round. + A compression primitive is not an abstraction concern; the dependency + direction is what puts it here. ++/ +extern (C) @nogc nothrow private { + size_t ZSTD_compressBound(size_t srcSize); + size_t ZSTD_compress(void* dst, size_t dstCapacity, + const(void)* src, size_t srcSize, int compressionLevel); + size_t ZSTD_decompress(void* dst, size_t dstCapacity, + const(void)* src, size_t compressedSize); + ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize); + uint ZSTD_isError(size_t code); + const(char)* ZSTD_getErrorName(size_t code); + int ZSTD_minCLevel(); + int ZSTD_maxCLevel(); + uint ZSTD_versionNumber(); +} +/+ ↓ what ZSTD_getFrameContentSize says when it cannot say +/ +private enum ulong ZSTD_CONTENTSIZE_UNKNOWN = ulong.max; /+ (0ULL - 1) +/ +private enum ulong ZSTD_CONTENTSIZE_ERROR = ulong.max - 1; /+ (0ULL - 2) +/ +/+ ↓ the four bytes that open every zstd frame, 0xFD2FB528 little endian. + A pod is recognised by this rather than by its name, so that a pod + already published as a plain zip keeps being read whatever it is called. ++/ +enum ubyte[4] zstd_frame_magic = [0x28, 0xB5, 0x2F, 0xFD]; +/+ ↓ the ceiling on what one frame may be allowed to become. + A frame declares its uncompressed size in its header and that number is + attacker-controlled: a .sisupod fetched over https could claim a size no + machine can allocate. The declared size is checked against this before a + buffer is asked for, so a hostile declaration is refused rather than + attempted. It matches MAX_TOTAL_SIZE in read_zip_pod.d, which bounds what + the archive inside may extract to. ++/ +enum size_t ZSTD_MAX_FRAME_CONTENT = 500 * 1024 * 1024; +/+ ↓ thrown rather than returned: every caller here wants the whole buffer or + nothing, and there is no partial result worth handing back ++/ +class ZstdException : Exception { + this(string _msg, string _file = __FILE__, size_t _line = __LINE__) @safe pure nothrow { + super(_msg, _file, _line); + } +} +/+ ↓ does this begin a zstd frame? +/ +bool zstdIsFrame(const(ubyte)[] _bytes) { + if (_bytes.length < zstd_frame_magic.length) { + return false; + } + foreach (_i, _b; zstd_frame_magic) { + if (_bytes[_i] != _b) { + return false; + } + } + return true; +} +/+ ↓ the library's version, for a run that wants to record what compressed a + pod. The bytes of a frame are a function of the library and the level, so + two zstd versions may compress the same input differently; nothing is + promised about archive bytes ++/ +@trusted uint zstdVersion() { + return ZSTD_versionNumber(); +} +/+ ↓ the level clamped to what this library actually supports +/ +@trusted int zstdLevelClamped(int _level) { + int _min = ZSTD_minCLevel(); + int _max = ZSTD_maxCLevel(); + if (_level < _min) { return _min; } + if (_level > _max) { return _max; } + return _level; +} +private @trusted string _zstdError(size_t _code) { + import std.string : fromStringz; + return ZSTD_getErrorName(_code).fromStringz.idup; +} +/+ ↓ one buffer to one frame +/ +@trusted ubyte[] zstdCompress(const(ubyte)[] _src, int _level = 19) { + size_t _bound = ZSTD_compressBound(_src.length); + auto _dst = new ubyte[_bound]; + size_t _got = ZSTD_compress( + _dst.ptr, _dst.length, + _src.ptr, _src.length, + zstdLevelClamped(_level), + ); + if (ZSTD_isError(_got)) { + throw new ZstdException("zstd could not compress: " ~ _zstdError(_got)); + } + return _dst[0 .. _got]; +} +/+ ↓ one frame back to one buffer, refusing a frame that will not say how + large it is or that says something absurd ++/ +@trusted ubyte[] zstdDecompress(const(ubyte)[] _src, + size_t _max = ZSTD_MAX_FRAME_CONTENT +) { + import std.conv : to; + if (!(zstdIsFrame(_src))) { + throw new ZstdException("not a zstd frame"); + } + ulong _declared = ZSTD_getFrameContentSize(_src.ptr, _src.length); + if (_declared == ZSTD_CONTENTSIZE_ERROR) { + throw new ZstdException("zstd frame header will not read"); + } + if (_declared == ZSTD_CONTENTSIZE_UNKNOWN) { + /+ every frame spine writes is one whole buffer, so its size is always + declared; a frame without one was made by something else + +/ + throw new ZstdException("zstd frame does not declare its size"); + } + if (_declared > _max) { + throw new ZstdException("zstd frame declares " + ~ _declared.to!string ~ " bytes, over the " + ~ _max.to!string ~ " allowed"); + } + auto _dst = new ubyte[_declared.to!size_t]; + size_t _got = ZSTD_decompress( + _dst.ptr, _dst.length, + _src.ptr, _src.length, + ); + if (ZSTD_isError(_got)) { + throw new ZstdException("zstd could not decompress: " ~ _zstdError(_got)); + } + if (_got != _dst.length) { + throw new ZstdException("zstd frame gave " ~ _got.to!string + ~ " bytes where its header declared " ~ _dst.length.to!string); + } + return _dst; +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ |
