diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/sisudoc/ocda/io_in/paths_source.d | 12 | ||||
| -rw-r--r-- | src/sisudoc/outputs/io_out/metadata.d | 8 | ||||
| -rw-r--r-- | src/sisudoc/outputs/io_out/source_pod.d | 25 | ||||
| -rw-r--r-- | src/sisudoc/spine.d | 19 |
4 files changed, 55 insertions, 9 deletions
diff --git a/src/sisudoc/ocda/io_in/paths_source.d b/src/sisudoc/ocda/io_in/paths_source.d index 1a4b4da..c0f526a 100644 --- a/src/sisudoc/ocda/io_in/paths_source.d +++ b/src/sisudoc/ocda/io_in/paths_source.d @@ -644,8 +644,18 @@ template spinePathsSRC() { } +/+ ↓ the pod archive's suffix, named once. + Previously a literal named at 4 locations, inviting mishaps on renaming. + (missing one meant a link that goes nowhere). + . + A .sisupod is the same archive as before inside one zstd frame. The legacy + spelling is also kept because the reader accepts both: every pod already + published carries it. ++/ +enum string pod_archive_suffix = ".sisupod"; +enum string pod_archive_suffix_legacy = ".zip"; template spinePathsPods() { - string _suffix = ".zip"; + string _suffix = pod_archive_suffix; auto spinePathsPods(M)(M doc_matters) { string _base_dir_pod = (doc_matters.output_path.length > 0) ? doc_matters.output_path ~ "/pod" diff --git a/src/sisudoc/outputs/io_out/metadata.d b/src/sisudoc/outputs/io_out/metadata.d index a63422f..5a10c64 100644 --- a/src/sisudoc/outputs/io_out/metadata.d +++ b/src/sisudoc/outputs/io_out/metadata.d @@ -510,8 +510,8 @@ string theme_light_1 = format(q"┃ metadata_ ~= "</p>"; if (doc_matters.opt.action.html_link_markup_source) { metadata_ ~= "<hr /><p class=\"lev1\">source: " ~ doc_matters.src.filename_base ~ "</p>"; - metadata_ ~= "<p class=\"lev1\">● markup source: the pod [<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ ".zip\" class=\"lnkicon\">" - ~ " 🫛 zipped </a>| " + metadata_ ~= "<p class=\"lev1\">● markup source: the pod [<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ pod_archive_suffix ~ "\" class=\"lnkicon\">" + ~ " 🫛 sisupod </a>| " ~ "<a href=\"../../" ~ pth_pod.internal_base ~ "/" ~ doc_matters.src.filename_base ~ "/\" class=\"lnkicon\">" ~ " 🫛 tree </a>] "; { @@ -533,10 +533,10 @@ string theme_light_1 = format(q"┃ try { // get sha digest for pod metadata_ ~= "<p class=\"lev2\">"; auto data = (cast(byte[]) (fn_pod).read); // prevents code from being safe - metadata_ ~= "<tt>" ~ data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ "</tt> - " ~ doc_matters.src.filename_base ~ ".zip"; + metadata_ ~= "<tt>" ~ data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ "</tt> - " ~ doc_matters.src.filename_base ~ pod_archive_suffix; metadata_ ~= "</p>"; } catch (Exception ex) { - writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, ".zip\n ", fn_pod); + writeln("WARNING, source doc_matters.src.filename_base not found: ", doc_matters.src.filename_base, pod_archive_suffix, "\n ", fn_pod); } } if (doc_matters.opt.action.ocda_db) { diff --git a/src/sisudoc/outputs/io_out/source_pod.d b/src/sisudoc/outputs/io_out/source_pod.d index 667e809..b929e06 100644 --- a/src/sisudoc/outputs/io_out/source_pod.d +++ b/src/sisudoc/outputs/io_out/source_pod.d @@ -546,16 +546,33 @@ template spinePod() { return zip; } void zipArchive(M,F,Z)(M doc_matters, F fn_pod, Z zip) { + import sisudoc.ocda.zstd : zstdCompress, ZstdException; auto fn_src_in = doc_matters.src.filename; if (doc_matters.opt.action.pod) { if (exists(doc_matters.src.file_with_absolute_path)) { try { - createZipFile!()(fn_pod, zip.build()); + /+ ↓ the archive is built exactly as before and then wrapped in one + zstd frame. The wrap is here and not in createZipFile, which + also writes every epub and every odt: those are real zips by + their own specifications. + . + The compression gain is in whole-stream rather than per-member + compression. A zip deflates each member on its own with a 32 KB + window, which sees none of the redundancy between multiple language + versions of a document and the catalogues that may repeat the + English (original language): (live-manual is 10.31x smaller this + way, where per-member deflate manages about 3x). + +/ + auto _archive = cast(ubyte[]) zip.build(); + createZipFile!()(fn_pod, + cast(void[]) zstdCompress(_archive, doc_matters.opt.action.pod_compression)); + } catch (ZstdException ex) { + writeln("WARNING failed to compress pod archive: ", fn_pod, " - ", ex.msg); } catch (ErrnoException ex) { // Handle errors } } else { writeln("WARNING check missing source file(s): ", doc_matters.opt.action.pod); } - if (!(exists(fn_pod))) { writeln("WARNING failed to create pod zip archive: ", fn_pod); } + if (!(exists(fn_pod))) { writeln("WARNING failed to create pod archive: ", fn_pod); } } } void zipArchiveDigest(M,F,D)(M doc_matters, F fn_pod, D _digests) { @@ -578,9 +595,9 @@ template spinePod() { if (exists(fn_pod)) { try { auto data = (cast(byte[]) (fn_pod).read); - // if (doc_matters.opt.action.vox_gt_2) { writeln(" ", doc_matters.src.filename, " > ", doc_matters.src.filename_base, ".zip"); } + // if (doc_matters.opt.action.vox_gt_2) { writeln(" ", doc_matters.src.filename, " > ", doc_matters.src.filename_base, pod_archive_suffix); } if (doc_matters.opt.action.pod) { - _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ ".zip"); + _zip_digest = (data.sha256Of.toHexString ~ "::" ~ data.length.to!string ~ " - " ~ doc_matters.src.filename_base ~ pod_archive_suffix); if (doc_matters.opt.action.vox_gt_1) { writeln(" ", _zip_digest); } if (doc_matters.opt.action.vox_gt_1) { writeln(" ", pths_pod.pod_dir_(), "/", doc_matters.src.filename_base, "/"); } if (doc_matters.opt.action.vox_gt_1) { writeln(" ", _digest_fn); } diff --git a/src/sisudoc/spine.d b/src/sisudoc/spine.d index 07f04be..9013c3e 100644 --- a/src/sisudoc/spine.d +++ b/src/sisudoc/spine.d @@ -214,6 +214,7 @@ string program_name = "spine"; ]; string[string] settings = [ "output" : "", + "pod-compression" : "", "ssp-round-trip" : "", "db-round-trip" : "", "abstraction-source" : "", @@ -317,6 +318,7 @@ string program_name = "spine"; "source", "document markup source", &opts["source"], "silent", "output to terminal", &opts["vox_is0"], "set-digest", "default hash digest type (e.g. sha256)", &settings["set-digest"], + "pod-compression", "pod archive compression level, zstd 1 to 19 (default 19)", &settings["pod-compression"], "set-papersize", "default papersize (latex pdf eg. a4 or a5 or b4 or letter)", &settings["set-papersize"], "set-textwrap", "default textwrap (e.g. 80 (characters)", &settings["set-textwrap"], "skel", "skel (dummy outline)", &opts["skel"], @@ -800,6 +802,23 @@ string program_name = "spine"; @trusted string text_wrap() { return settings["set-textwrap"]; } + /+ ↓ the zstd level a pod archive is written at. 19 by default: a pod + is written once and fetched many times, and there is a significant gain + (live-manual 675970 bytes at 19 against 786683 at 9). Lower it where + build time matters more than the bytes do. + +/ + @trusted int pod_compression() { + import std.conv : to; + if (settings["pod-compression"].length > 0) { + try { + return settings["pod-compression"].to!int; + } catch (Exception) { + writeln("WARNING --pod-compression is not a number, using 19: ", + settings["pod-compression"]); + } + } + return 19; + } @trusted string latex_papersize() { return settings["set-papersize"]; } |