diff options
Diffstat (limited to 'org')
| -rw-r--r-- | org/compression_zstd.org | 200 | ||||
| -rw-r--r-- | org/config_dub.org | 2 | ||||
| -rw-r--r-- | org/config_nix.org | 9 | ||||
| -rw-r--r-- | org/default_regex.org | 4 | ||||
| -rw-r--r-- | org/in_abstraction_artefacts.org | 7 | ||||
| -rw-r--r-- | org/in_zip_pod.org | 26 |
6 files changed, 239 insertions, 9 deletions
diff --git a/org/compression_zstd.org b/org/compression_zstd.org new file mode 100644 index 0000000..bcb67f3 --- /dev/null +++ b/org/compression_zstd.org @@ -0,0 +1,200 @@ +-*- mode: org -*- +#+TITLE: sisudoc spine (doc_reform) output zip +#+DESCRIPTION: documents - structuring, publishing in multiple formats & search +#+FILETAGS: :spine:zip: +#+AUTHOR: Ralph Amissah +#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] +#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah +#+LANGUAGE: en +#+STARTUP: content hideblocks hidestars noindent entitiespretty +#+PROPERTY: header-args+ :eval never-export :exports code +#+PROPERTY: header-args+ :noweb yes :padline no +#+PROPERTY: header-args+ :results silent :cache no +#+PROPERTY: header-args+ :mkdirp yes +#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t +- magic single double-quote → " ← FIX changes hilighting behavior (occuring + after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX + +- [[./doc-reform.org][doc-reform.org]] [[./][org/]] +- [[./output_hub.org][output_hub]] + +* _zstd_ :module:spine:compression: + +#+HEADER: :tangle "../src/sisudoc/ocda/zstd.d" +#+HEADER: :noweb yes +#+BEGIN_SRC d +<<doc_header_including_copyright_and_license>> +/+ + module zstd;<BR> + - the few libzstd entry points spine uses, and a buffer in / buffer out + wrapper over each direction<BR> + - one frame per call: what is compressed is a whole pod archive, not a + stream ++/ +module sisudoc.ocda.zstd; +@safe: +/+ ↓ libzstd, declared rather than bound by a generated header + Provides the fifteen declarations that spine needs (there is nothing + here to generate and no vendored tree to track): the C library is linked + (dub "libs": ["zstd"], nix buildInputs pkgs.zstd) and these + prototypes say what is being called. That is the same arrangement as + sqlite3, which is declared in the vendored d2sqlite3 and linked from the + system. + . + It sits under ocda/ rather than outputs/ because the pod *reader* needs it, + and the reader (ocda/io_in/read_zip_pod.d) is in the abstraction part, + which the outputs depend on and not the other way round. + A compression primitive is not an abstraction concern; the dependency + direction is what puts it here. ++/ +extern (C) @nogc nothrow private { + size_t ZSTD_compressBound(size_t srcSize); + size_t ZSTD_compress(void* dst, size_t dstCapacity, + const(void)* src, size_t srcSize, int compressionLevel); + size_t ZSTD_decompress(void* dst, size_t dstCapacity, + const(void)* src, size_t compressedSize); + ulong ZSTD_getFrameContentSize(const(void)* src, size_t srcSize); + uint ZSTD_isError(size_t code); + const(char)* ZSTD_getErrorName(size_t code); + int ZSTD_minCLevel(); + int ZSTD_maxCLevel(); + uint ZSTD_versionNumber(); +} +/+ ↓ what ZSTD_getFrameContentSize says when it cannot say +/ +private enum ulong ZSTD_CONTENTSIZE_UNKNOWN = ulong.max; /+ (0ULL - 1) +/ +private enum ulong ZSTD_CONTENTSIZE_ERROR = ulong.max - 1; /+ (0ULL - 2) +/ +/+ ↓ the four bytes that open every zstd frame, 0xFD2FB528 little endian. + A pod is recognised by this rather than by its name, so that a pod + already published as a plain zip keeps being read whatever it is called. ++/ +enum ubyte[4] zstd_frame_magic = [0x28, 0xB5, 0x2F, 0xFD]; +/+ ↓ the ceiling on what one frame may be allowed to become. + A frame declares its uncompressed size in its header and that number is + attacker-controlled: a .sisupod fetched over https could claim a size no + machine can allocate. The declared size is checked against this before a + buffer is asked for, so a hostile declaration is refused rather than + attempted. It matches MAX_TOTAL_SIZE in read_zip_pod.d, which bounds what + the archive inside may extract to. ++/ +enum size_t ZSTD_MAX_FRAME_CONTENT = 500 * 1024 * 1024; +/+ ↓ thrown rather than returned: every caller here wants the whole buffer or + nothing, and there is no partial result worth handing back ++/ +class ZstdException : Exception { + this(string _msg, string _file = __FILE__, size_t _line = __LINE__) @safe pure nothrow { + super(_msg, _file, _line); + } +} +/+ ↓ does this begin a zstd frame? +/ +bool zstdIsFrame(const(ubyte)[] _bytes) { + if (_bytes.length < zstd_frame_magic.length) { + return false; + } + foreach (_i, _b; zstd_frame_magic) { + if (_bytes[_i] != _b) { + return false; + } + } + return true; +} +/+ ↓ the library's version, for a run that wants to record what compressed a + pod. The bytes of a frame are a function of the library and the level, so + two zstd versions may compress the same input differently; nothing is + promised about archive bytes ++/ +@trusted uint zstdVersion() { + return ZSTD_versionNumber(); +} +/+ ↓ the level clamped to what this library actually supports +/ +@trusted int zstdLevelClamped(int _level) { + int _min = ZSTD_minCLevel(); + int _max = ZSTD_maxCLevel(); + if (_level < _min) { return _min; } + if (_level > _max) { return _max; } + return _level; +} +private @trusted string _zstdError(size_t _code) { + import std.string : fromStringz; + return ZSTD_getErrorName(_code).fromStringz.idup; +} +/+ ↓ one buffer to one frame +/ +@trusted ubyte[] zstdCompress(const(ubyte)[] _src, int _level = 19) { + size_t _bound = ZSTD_compressBound(_src.length); + auto _dst = new ubyte[_bound]; + size_t _got = ZSTD_compress( + _dst.ptr, _dst.length, + _src.ptr, _src.length, + zstdLevelClamped(_level), + ); + if (ZSTD_isError(_got)) { + throw new ZstdException("zstd could not compress: " ~ _zstdError(_got)); + } + return _dst[0 .. _got]; +} +/+ ↓ one frame back to one buffer, refusing a frame that will not say how + large it is or that says something absurd ++/ +@trusted ubyte[] zstdDecompress(const(ubyte)[] _src, + size_t _max = ZSTD_MAX_FRAME_CONTENT +) { + import std.conv : to; + if (!(zstdIsFrame(_src))) { + throw new ZstdException("not a zstd frame"); + } + ulong _declared = ZSTD_getFrameContentSize(_src.ptr, _src.length); + if (_declared == ZSTD_CONTENTSIZE_ERROR) { + throw new ZstdException("zstd frame header will not read"); + } + if (_declared == ZSTD_CONTENTSIZE_UNKNOWN) { + /+ every frame spine writes is one whole buffer, so its size is always + declared; a frame without one was made by something else + +/ + throw new ZstdException("zstd frame does not declare its size"); + } + if (_declared > _max) { + throw new ZstdException("zstd frame declares " + ~ _declared.to!string ~ " bytes, over the " + ~ _max.to!string ~ " allowed"); + } + auto _dst = new ubyte[_declared.to!size_t]; + size_t _got = ZSTD_decompress( + _dst.ptr, _dst.length, + _src.ptr, _src.length, + ); + if (ZSTD_isError(_got)) { + throw new ZstdException("zstd could not decompress: " ~ _zstdError(_got)); + } + if (_got != _dst.length) { + throw new ZstdException("zstd frame gave " ~ _got.to!string + ~ " bytes where its header declared " ~ _dst.length.to!string); + } + return _dst; +} +#+END_SRC + +* org includes +** project version + +#+NAME: spine_version +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> +#+END_SRC + +** year + +#+NAME: year +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> +#+END_SRC + +** document header including copyright & license + +#+NAME: doc_header_including_copyright_and_license +#+HEADER: :noweb yes +#+BEGIN_SRC emacs-lisp +<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> +#+END_SRC + +* __END__ diff --git a/org/config_dub.org b/org/config_dub.org index 10dd6e5..83ad384 100644 --- a/org/config_dub.org +++ b/org/config_dub.org @@ -50,6 +50,7 @@ "excludedSourceFiles": [ "./src/sisudoc/ocda/*" ], "importPaths": [ "./views" ], "stringImportPaths": [ "./views" ], + "libs": [ "zstd" ], "buildRequirements": [ "allowWarnings" ], "targetType": "executable", "buildTypes": { @@ -94,6 +95,7 @@ "copyright": "Copyright © 2015 - 2026 Ralph Amissah", "license": "AGPL-3.0+", "targetType": "library", + "libs": [ "zstd" ], "sourcePaths": [ "./src/sisudoc/ocda" ], "importPaths": [ "./src" ], "buildRequirements": [ "allowWarnings" ], diff --git a/org/config_nix.org b/org/config_nix.org index 6216b10..6fb82ad 100644 --- a/org/config_nix.org +++ b/org/config_nix.org @@ -53,7 +53,7 @@ meta.mainProgram = "spine"; executable = true; # not used src = self; - buildInputs = [ pkgs.sqlite ]; + buildInputs = [ pkgs.sqlite pkgs.zstd.out ]; nativeBuildInputs = [ pkgs.dub compilerPkg pkgs.gnumake ]; preBuild = '' export HOME=$TMPDIR @@ -108,6 +108,7 @@ pkgs.dtools pkgs.gnumake pkgs.sqlite + pkgs.zstd pkgs.gnugrep pkgs.gnused pkgs.ripgrep @@ -363,6 +364,7 @@ with pkgs; jq # ❯❯❯ sqlite search related sqlite + zstd # ❯❯❯ xml_and_epub_related # libxml2 # html-tidy @@ -436,6 +438,7 @@ with pkgs; ldc, gnumake, sqlite, + zstd, compilerPkg ? ldc, compilerBin ? "ldmd2", buildType ? "ldmd2", @@ -445,7 +448,7 @@ stdenv.mkDerivation { pname = "spine"; version = "<<spine_version>>"; src = lib.cleanSource ./.; - buildInputs = [ sqlite ]; + buildInputs = [ sqlite zstd.out ]; nativeBuildInputs = [ dub compilerPkg gnumake ]; preBuild = '' export HOME=$TMPDIR @@ -586,7 +589,7 @@ with ( version = "<<spine_version>>"; src = ./.; nativeBuildInputs = with pkgs; [dub ldc]; - buildInputs = with pkgs; [nixVersions.latest sqlite]; + buildInputs = with pkgs; [nixVersions.latest sqlite zstd.out]; meta = with pkgs.lib; { description = "A sisu like parser & document generator"; longDescription = '' diff --git a/org/default_regex.org b/org/default_regex.org index 1a6aac6..b7454cb 100644 --- a/org/default_regex.org +++ b/org/default_regex.org @@ -706,8 +706,8 @@ static template spineRgxFiles() { static src_pth_sst_or_ssm = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.](?P<extension>ss[tm]))$`); static src_pth_pod_sst_or_ssm = ctRegex!(`^(?P<podpath>[/]?(?:[a-zA-Z0-9._-]+/)*)media/text/[a-z]{2}/(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*?[.]ss[tm])$`); static src_pth_contents = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9][a-zA-Z0-9._-]*)/pod[.]manifest$`); -static src_pth_zip = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]zip)$`); -static src_pth_types = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+[.]zip))$`); +static src_pth_zip = ctRegex!(`^(?P<path>[/]?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod))$`); +static src_pth_types = ctRegex!(`^(?P<path>[/]?[a-zA-Z0-9._-]+/)*(?P<gotfile>(?P<filename>[a-zA-Z0-9._-]+[.]ss[tm])|(?P<filelist>[a-zA-Z0-9._-]+/pod[.]manifest)|(?P<filezip>[a-zA-Z0-9._-]+(?:[.]zip|[.]sisupod)))$`); static src_fn = ctRegex!(`^([/]?(?:[a-zA-Z0-9._-]+/)*)(?P<fn_src>(?P<fn_base>[a-zA-Z0-9._-]+)[.](?P<fn_src_suffix>ss[tm]))$`); static src_fn_master = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ssm)$`); static src_fn_find_inserts = ctRegex!(`^(?P<path>/?(?:[a-zA-Z0-9._-]+/)*)(?P<filename>[a-zA-Z0-9._-]+[.]ss[im])$`); diff --git a/org/in_abstraction_artefacts.org b/org/in_abstraction_artefacts.org index 262135e..9da6d32 100644 --- a/org/in_abstraction_artefacts.org +++ b/org/in_abstraction_artefacts.org @@ -96,7 +96,12 @@ template spineAbstractionLoad() { if (_path.endsWith(".ocda.db")) { return AbstractionSource.ocda_db; } if (_path.endsWith(".ssp")) { return AbstractionSource.ssp; } if (_path.endsWith(".sst") || _path.endsWith(".ssm")) { return AbstractionSource.markup; } - if (_path.endsWith(".zip")) { return AbstractionSource.pod_zip; } + /+ ↓ .sisupod is the pod archive spine writes; .zip is what it wrote before, + and every pod already published carries that name, so both are read. + What is inside decides how it is opened, not the name. + +/ + if (_path.endsWith(".sisupod") || _path.endsWith(".zip")) + { return AbstractionSource.pod_zip; } if (_path.endsWith(".db")) { return AbstractionSource.ocda_db; } return AbstractionSource.unknown; } diff --git a/org/in_zip_pod.org b/org/in_zip_pod.org index 2922ecf..c542916 100644 --- a/org/in_zip_pod.org +++ b/org/in_zip_pod.org @@ -39,6 +39,7 @@ template spineExtractZipPod() { import std.regex; import std.stdio; import std.string : indexOf; + import sisudoc.ocda.zstd; /+ security limits for zip extraction +/ enum size_t MAX_ENTRY_SIZE = 50 * 1024 * 1024; /+ 50 MB per entry +/ @@ -98,10 +99,29 @@ template spineExtractZipPod() { } /+ ↓ derive pod name from zip filename +/ string zip_basename = zip_path.baseName.stripExtension; - /+ ↓ read and parse zip archive +/ + /+ ↓ read and parse the pod archive. + a .sisupod is one zstd frame wrapping the archive spine writes, so + the bytes are unwrapped before the archive is parsed. Determined from + the first four bytes and not from the filename: a pod published as a + plain .zip, or one a reader has renamed, is read either way. Everything + after this point sees the same archive it always did, guards included. + +/ + ubyte[] _archive_bytes; + try { + _archive_bytes = cast(ubyte[]) read(zip_path); + if (zstdIsFrame(_archive_bytes)) { + _archive_bytes = zstdDecompress(_archive_bytes, MAX_TOTAL_SIZE); + } + } catch (ZstdException ex) { + result.error_msg = "failed to decompress pod archive: " ~ zip_path ~ " - " ~ ex.msg; + return result; + } catch (Exception ex) { + result.error_msg = "error reading pod archive: " ~ zip_path ~ " - " ~ ex.msg; + return result; + } ZipArchive zip; try { - zip = new ZipArchive(read(zip_path)); + zip = new ZipArchive(_archive_bytes); } catch (ZipException ex) { result.error_msg = "failed to read zip archive: " ~ zip_path ~ " - " ~ ex.msg; return result; @@ -250,7 +270,7 @@ template spineExtractZipPod() { its own, carrying its images. A .ssp is not here: it describes its images without carrying them, so it would arrive without them. +/ - static auto rgx_url_source = ctRegex!(`^https?://[a-zA-Z0-9._:/-]+([.]zip|[.]ocda[.]db)$`); + static auto rgx_url_source = ctRegex!(`^https?://[a-zA-Z0-9._:/-]+([.]sisupod|[.]zip|[.]ocda[.]db)$`); struct DownloadResult { string local_path; /+ path to downloaded temp file +/ |
