From 4c766a8455d05bdf6a9793946f7839a01f179b65 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Mon, 21 Sep 2026 09:35:37 -0400 Subject: pod: write .sisupod (single zstd frame) The pod archive spine writes is now named .sisupod and carries the same archive inside one zstd frame. The reader has accepted both since the previous commit, so every pod already published stays readable and this changes only what is written. free_culture 2863504 -> 1100180 live-manual 6972418 -> 692942 10.06x The wrap is at the pod call site and not in createZipFile, which also writes every epub and every odt. Best compression from whole-stream (rather than per-member compression). (for live-manual per-member deflate manages about 3x compared to 10x for whole pod content compression). The suffix is named as pod_archive_suffix. --pod-compression sets the level, 19 by default: a pod is written once and fetched many times (692942 bytes at 19, 837703 at 9, 2054365 at 1). (A value that is not a number warns and the default is used). Output built from a written .sisupod is byte identical to output built from the markup it came from. (assisted by Claude-Code) --- org/default_paths.org | 12 +++++++++++- org/out_metadata.org | 8 ++++---- org/out_src_pod.org | 25 +++++++++++++++++++++---- org/spine.org | 19 +++++++++++++++++++ 4 files changed, 55 insertions(+), 9 deletions(-) (limited to 'org') diff --git a/org/default_paths.org b/org/default_paths.org index bbc0362..9bf56d5 100644 --- a/org/default_paths.org +++ b/org/default_paths.org @@ -718,8 +718,18 @@ template spinePathsSRC() { #+NAME: template_paths_pods #+BEGIN_SRC d +/+ ↓ the pod archive's suffix, named once. + Previously a literal named at 4 locations, inviting mishaps on renaming. + (missing one meant a link that goes nowhere). + . + A .sisupod is the same archive as before inside one zstd frame. The legacy + spelling is also kept because the reader accepts both: every pod already + published carries it. ++/ +enum string pod_archive_suffix = ".sisupod"; +enum string pod_archive_suffix_legacy = ".zip"; template spinePathsPods() { - string _suffix = ".zip"; + string _suffix = pod_archive_suffix; auto spinePathsPods(M)(M doc_matters) { string _base_dir_pod = (doc_matters.output_path.length > 0) ? doc_matters.output_path ~ "/pod" diff --git a/org/out_metadata.org b/org/out_metadata.org index 8f3a946..b816dad 100644 --- a/org/out_metadata.org +++ b/org/out_metadata.org @@ -176,8 +176,8 @@ if (doc_matters.opt.action.html_link_text) { metadata_ ~= "

"; if (doc_matters.opt.action.html_link_markup_source) { metadata_ ~= "

source: " ~ doc_matters.src.filename_base ~ "

"; - metadata_ ~= "

●  markup source:  the pod [" - ~ " 🫛 zipped | " + metadata_ ~= "

●  markup source:  the pod [" + ~ " 🫛 sisupod | " ~ "" ~ " 🫛 tree ] "; { @@ -199,10 +199,10 @@ if (doc_matters.opt.action.html_link_markup_source) { try { // get sha digest for pod metadata_ ~= "

"; auto data = (cast(byte[]) (fn_pod).read); // prevents code from being safe - metadata_ ~= "" ~ data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip"; + metadata_ ~= "" ~ data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ pod_archive_suffix; metadata_ ~= "

"; } catch (Exception ex) { - writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, ".zip\n ", fn_pod); + writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, pod_archive_suffix, "\n ", fn_pod); } } if (doc_matters.opt.action.ocda_db) { diff --git a/org/out_src_pod.org b/org/out_src_pod.org index eac9882..96f784f 100644 --- a/org/out_src_pod.org +++ b/org/out_src_pod.org @@ -579,16 +579,33 @@ void podArchive_directory_tree(M,P)(M doc_matters, P pths_pod) { // create direc #+NAME: zip_archive #+BEGIN_SRC d void zipArchive(M,F,Z)(M doc_matters, F fn_pod, Z zip) { + import sisudoc.ocda.zstd : zstdCompress, ZstdException; auto fn_src_in = doc_matters.src.filename; if (doc_matters.opt.action.pod) { if (exists(doc_matters.src.file_with_absolute_path)) { try { - createZipFile!()(fn_pod, zip.build()); + /+ ↓ the archive is built exactly as before and then wrapped in one + zstd frame. The wrap is here and not in createZipFile, which + also writes every epub and every odt: those are real zips by + their own specifications. + . + The compression gain is in whole-stream rather than per-member + compression. A zip deflates each member on its own with a 32 KB + window, which sees none of the redundancy between multiple language + versions of a document and the catalogues that may repeat the + English (original language): (live-manual is 10.31x smaller this + way, where per-member deflate manages about 3x). + +/ + auto _archive = cast(ubyte[]) zip.build(); + createZipFile!()(fn_pod, + cast(void[]) zstdCompress(_archive, doc_matters.opt.action.pod_compression)); + } catch (ZstdException ex) { + writeln("WARNING failed to compress pod archive: ", fn_pod, " - ", ex.msg); } catch (ErrnoException ex) { // Handle errors } } else { writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod); } - if (!(exists(fn_pod))) { writeln("WARNING failed to create pod zip archive: ", fn_pod); } + if (!(exists(fn_pod))) { writeln("WARNING failed to create pod archive: ", fn_pod); } } } #+END_SRC @@ -617,9 +634,9 @@ void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) { if (exists(fn_pod)) { try { auto data = (cast(byte[]) (fn_pod).read); - // if (doc_matters.opt.action.vox_gt_2) { writeln(" ", doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip"); } + // if (doc_matters.opt.action.vox_gt_2) { writeln(" ", doc_matters.src.filename, " > ", doc_matters.src.filename_base, pod_archive_suffix); } if (doc_matters.opt.action.pod) { - _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip"); + _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ pod_archive_suffix); if (doc_matters.opt.action.vox_gt_1) { writeln(" ", _zip_digest); } if (doc_matters.opt.action.vox_gt_1) { writeln(" ", pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, "/"); } if (doc_matters.opt.action.vox_gt_1) { writeln(" ", _digest_fn); } diff --git a/org/spine.org b/org/spine.org index aa8b6df..cef324a 100644 --- a/org/spine.org +++ b/org/spine.org @@ -466,6 +466,7 @@ NOTES on configuration hierarchy: see meta_conf_make_meta.org #+BEGIN_SRC d string[string] settings = [ "output" : "", + "pod-compression" : "", "ssp-round-trip" : "", "db-round-trip" : "", "abstraction-source" : "", @@ -575,6 +576,7 @@ auto helpInfo = getopt(args, "source", "document markup source", &opts["source"], "silent", "output to terminal", &opts["vox_is0"], "set-digest", "default hash digest type (e.g. sha256)", &settings["set-digest"], + "pod-compression", "pod archive compression level, zstd 1 to 19 (default 19)", &settings["pod-compression"], "set-papersize", "default papersize (latex pdf eg. a4 or a5 or b4 or letter)", &settings["set-papersize"], "set-textwrap", "default textwrap (e.g. 80 (characters)", &settings["set-textwrap"], "skel", "skel (dummy outline)", &opts["skel"], @@ -1071,6 +1073,23 @@ struct OptActions { @trusted string text_wrap() { return settings["set-textwrap"]; } + /+ ↓ the zstd level a pod archive is written at. 19 by default: a pod + is written once and fetched many times, and there is a significant gain + (live-manual 675970 bytes at 19 against 786683 at 9). Lower it where + build time matters more than the bytes do. + +/ + @trusted int pod_compression() { + import std.conv : to; + if (settings["pod-compression"].length > 0) { + try { + return settings["pod-compression"].to!int; + } catch (Exception) { + writeln("WARNING --pod-compression is not a number, using 19: ", + settings["pod-compression"]); + } + } + return 19; + } @trusted string latex_papersize() { return settings["set-papersize"]; } -- cgit v1.2.3