aboutsummaryrefslogtreecommitdiffhomepage
path: root/org/in_source_artefact.org
diff options
context:
space:
mode:
authorRalph Amissah <ralph.amissah@gmail.com>2026-09-11 21:45:24 -0400
committerRalph Amissah <ralph.amissah@gmail.com>2026-09-12 12:39:06 -0400
commitc6a479b9bd8bdd34ee9699e10247ebdeb183e032 (patch)
tree1eab495f7fbf5467c023651aa7f0ee5009215064 /org/in_source_artefact.org
parentspine: a .ocda.db can be fetched, as a pod zip can (diff)
org renames
Diffstat (limited to 'org/in_source_artefact.org')
-rw-r--r--org/in_source_artefact.org392
1 files changed, 0 insertions, 392 deletions
diff --git a/org/in_source_artefact.org b/org/in_source_artefact.org
deleted file mode 100644
index 07c865f..0000000
--- a/org/in_source_artefact.org
+++ /dev/null
@@ -1,392 +0,0 @@
--*- mode: org -*-
-#+TITLE: sisudoc spine (doc_reform) markup source raw
-#+DESCRIPTION: documents - structuring, publishing in multiple formats & search
-#+FILETAGS: :spine:sourcefile:read:
-#+AUTHOR: Ralph Amissah
-#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
-#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah
-#+LANGUAGE: en
-#+STARTUP: content hideblocks hidestars noindent entitiespretty
-#+PROPERTY: header-args+ :eval never-export :exports code
-#+PROPERTY: header-args+ :noweb yes :padline no
-#+PROPERTY: header-args+ :results silent :cache no
-#+PROPERTY: header-args+ :mkdirp yes
-#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t
-- magic single double-quote → " ← FIX changes hilighting behavior (occuring
- after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX
-
-- [[./doc-reform.org][doc-reform.org]] [[./][org/]]
-
-* A. get doc from artefact
-** _module template_ :module:config_files:
-
-#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_from_artefact.d"
-#+HEADER: :noweb yes
-#+BEGIN_SRC d
-<<doc_header_including_copyright_and_license>>
-module sisudoc.ocda.abstraction.doc_from_artefact;
-@safe:
-/+ ↓ a document, from an artefact rather than from markup
- spineAbstraction() parses markup and hands the output writers a doc:
- an abstraction and a doc_matters. This does the same from a .ssp or a
- .ocda.db, filling the same docMattersMake() arguments, so that what
- comes out is the value the writers already take and they never learn
- which of the two made it.
- .
- Three of the seven arguments come from the run and are the same either
- way: program_info, opt_action, and the conf half of conf_make_meta,
- which is site and run scoped (urls, papersize, the search database) and
- has no business in a document's own artefact. The other four come from
- the artefact:
- .
- conf_make_meta meta and make from the @meta and @make blocks, over
- the site config the run assembled
- doc_has docHasFromAbstraction, over the objects
- doc_digest the markup digest from @source
- manifest synthesised from the artefact's path and the
- document's own name and language
- .
- What is not here, deliberately: insert_file_list, which is read only by
- source_pod.d for --source and --pod2. No artefact carries the markup,
- so a pod cannot be rebuilt from one, and the empty list is the truthful
- answer rather than a gap.
-+/
-template spineDocFromArtefact() {
- import std.algorithm : endsWith;
- import std.array;
- import std.conv : to;
- import std.path;
- import std.process : thisProcessID;
- import std.regex;
- import std.stdio;
- import std.string;
- import sisudoc.ocda.meta.conf_make_meta_structs;
- import sisudoc.ocda.meta.doc_matters;
- import sisudoc.ocda.io_in.paths_source;
- import sisudoc.ocda.abstraction.doc_has;
- import sisudoc.ocda.abstraction.load;
- import sisudoc.ocda.meta.topic_register;
- mixin spineDocMatters;
- mixin spineDocHasFromAbstraction;
- mixin spineAbstractionLoad;
- mixin spineTopicRegister;
- /+ ↓ a .ssp header key from a field name: the first underscore becomes the
- dot that separates the group, so title_main is title.main and
- rights_copyright_text is rights.copyright_text. The writer's keys are
- formed the same way, which is what lets this be read off the struct
- rather than kept as a second list to fall out of step with it.
- +/
- private string _sspKeyOf(string _member) {
- foreach (i, c; _member) {
- if (c == '_') { return _member[0 .. i] ~ "." ~ _member[i + 1 .. $]; }
- }
- return _member;
- }
- /+ ↓ the document's own header, over the site config the run assembled.
- every string field of MetaComposite is looked for under its own key,
- so a field added there and emitted by the writer arrives here without
- anything being added. The arr fields are split from the string they
- come from, as agreed: that is parsing a value, not deriving a
- structure.
- +/
- ConfComposite confMakeMetaFromHeader(S)(S _ssp, ConfComposite _site_conf) {
- ConfComposite _out = _site_conf; // conf: site and run scoped, kept
- static foreach (_m; __traits(allMembers, MetaComposite)) {
- static if (is(typeof(__traits(getMember, _out.meta, _m)) == string)) {{
- enum string _key = _sspKeyOf(_m);
- if (auto _v = _key in _ssp.meta) { __traits(getMember, _out.meta, _m) = *_v; }
- }}
- }
- /+ ↓ title_sub is a copy of title_subtitle, made where the header is
- read, and is what epub3 puts in dc:title id="subtitle". It is not
- emitted, being the same string twice; it is remade here the way the
- yaml reader makes it +/
- _out.meta.title_sub = _out.meta.title_subtitle;
- /+ ↓ creator.author is the author list joined with ", ", so splitting on
- it gives the list back +/
- if (_out.meta.creator_author.length > 0) {
- foreach (_a; _out.meta.creator_author.split(", ")) {
- if (_a.strip.length > 0) { _out.meta.creator_author_arr ~= _a.strip; }
- }
- }
- /+ ↓ the topic register arrays, by the one rule the yaml reader uses:
- the split is not a plain one and deriving it twice would be two
- answers to the same question +/
- {
- auto _tr = topicRegisterArrays(_out.meta.classify_topic_register);
- _out.meta.classify_topic_register_arr = _tr.arr;
- _out.meta.classify_topic_register_expanded_arr = _tr.expanded;
- }
- /+ ↓ make: the keys are the field names, with no group to separate +/
- if (auto _v = "doc_type" in _ssp.make) {
- _out.make.doc_type = *_v; _out.make_str.doc_type = *_v;
- }
- if (auto _v = "auto_num_top_at_level" in _ssp.make) {
- _out.make.auto_num_top_at_level = *_v;
- }
- if (auto _v = "auto_num_top_lv" in _ssp.make) {
- try { _out.make.auto_num_top_lv = (*_v).to!int; } catch (Exception ex) {}
- }
- if (auto _v = "auto_num_depth" in _ssp.make) {
- try { _out.make.auto_num_depth = (*_v).to!int; } catch (Exception ex) {}
- }
- if (auto _v = "breaks" in _ssp.make) {
- _out.make.breaks = *_v; _out.make_str.breaks = *_v;
- }
- if (auto _v = "home_button_text" in _ssp.make) {
- _out.make.home_button_text = *_v; _out.make_str.home_button_text = *_v;
- }
- /+ ↓ footer is a list, joined with the separator the format escapes +/
- if (auto _v = "footer" in _ssp.make) {
- _out.make.footer = sspUnescapeSplit(*_v);
- _out.make_str.footer = _out.make.footer;
- }
- return _out;
- }
- /+ ↓ where a document loaded from an artefact says it is.
- an artefact names its document (% Source:) and its language (@source
- language) but not where the pod holding its images sits, so that is
- taken from where the artefact itself was found:
- .
- pod/<doc>/media/abstraction/<uid>.ssp -> pod/<doc>/
- pod/<uid>.ocda.db -> pod/<doc>/
- .
- which is where --pod2 puts them both. The images beside a .ssp are
- then found where html.d already looks; a .ocda.db carries its own and
- does not need the directory to exist.
- +/
- struct ST_ArtefactPaths {
- string pod_dir;
- string src_file_with_path;
- }
- ST_ArtefactPaths artefactPaths(S)(string _artefact, S _ssp) {
- ST_ArtefactPaths _out;
- string _lang = ("language" in _ssp.source_info) ? _ssp.source_info["language"] : "en";
- string _dir = _artefact.dirName;
- if (_artefact.endsWith(".ssp")) {
- /+ ↓ <pod>/media/abstraction/<uid>.ssp: two directories up is the pod +/
- _out.pod_dir = (_dir.baseName == "abstraction")
- ? _dir.dirName.dirName : _dir;
- } else {
- /+ ↓ <dir>/<uid>.ocda.db, the pod tree beside it named for the document +/
- string _uid = _artefact.baseName;
- foreach (_sfx; [".ocda.db", ".db"]) {
- if (_uid.endsWith(_sfx)) { _uid = _uid[0 .. $ - _sfx.length]; break; }
- }
- if (_uid.endsWith("." ~ _lang)) { _uid = _uid[0 .. $ - _lang.length - 1]; }
- _out.pod_dir = (_dir.chainPath(_uid).array).to!string;
- }
- _out.src_file_with_path = (_out.pod_dir
- .chainPath("media").chainPath("text").chainPath(_lang)
- .chainPath(_ssp.source).array).to!string;
- return _out;
- }
- /+ ↓ a database's images, written where the writers look for them.
- returns the pod directory to use in place of the one beside the
- artefact, or "" when the database carries no images and the pod
- beside it should stand. Each blob is checked against the sha256 the
- database recorded with it: the two were written together, so a
- mismatch means the file is damaged, and saying so is the point of
- having recorded the digest at all.
- +/
- @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) {
- import std.digest : toHexString;
- import std.digest.sha : sha256Of;
- import std.file : mkdirRecurse, write, tempDir;
- import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead;
- mixin spineAbstractionDbRead _dbr;
- auto _files = _dbr.dbReadFiles(_artefact, "image");
- if (_files.length == 0) { return ""; }
- string _root = (tempDir.chainPath("spine-ocda-"
- ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string;
- string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string;
- try {
- _img_dir.mkdirRecurse;
- } catch (Exception ex) {
- stderr.writeln("WARNING: could not make an image directory for ", _artefact,
- ": ", ex.msg);
- return "";
- }
- foreach (_f; _files) {
- string _got = _f.data.sha256Of.toHexString.to!string;
- if (_f.sha256.length > 0 && _got != _f.sha256) {
- stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName,
- " does not match the digest recorded with it (", _f.sha256, " expected, ",
- _got, " found); it is written out as it stands");
- }
- try {
- (_img_dir.chainPath(_f.name).array).to!string.write(_f.data);
- } catch (Exception ex) {
- stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg);
- }
- }
- if (_opt_action.vox_gt_1) {
- writeln(" ", _files.length, " image(s) from the database, in ", _img_dir);
- }
- return _root;
- }
- /+ ↓ the images a .ssp describes, against the ones beside it.
- a .ssp carries no image bytes, so what it can do is say whether the
- files in the pod it sits in are the ones the abstraction was built
- from. Nothing checked that before: a changed or wrong image in the
- directory paired silently.
- +/
- @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) {
- import std.digest : toHexString;
- import std.digest.sha : sha256Of;
- import std.file : exists, read;
- string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string;
- int _checked, _missing, _wrong;
- foreach (_section; _ssp.section_order) {
- foreach (obj; _ssp.abstraction[_section]) {
- foreach (_img; obj.metainfo.sha256.images) {
- if (_img.fileName.length == 0) { continue; }
- string _pth = (_img_dir.chainPath(_img.fileName).array).to!string;
- if (!_pth.exists) { ++_missing; continue; }
- ++_checked;
- string _want = _img.fileHash_sha256.toHexString.to!string;
- string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string;
- if (_want != _got && _want != "00000000000000000000000000000000"
- ~ "00000000000000000000000000000000") {
- ++_wrong;
- stderr.writeln("WARNING: image ", _img.fileName, " beside ",
- _ssp.source, " is not the one the abstraction was built from");
- }
- }
- }
- }
- if (_missing > 0) {
- stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are",
- " not in ", _img_dir);
- }
- if (_opt_action.vox_gt_1 && _checked > 0) {
- writeln(" ", _checked, " image(s) checked against the abstraction's digests",
- (_wrong == 0) ? ", all matching" : "");
- }
- }
- /+ ↓ the doc the output writers take, from an artefact.
- the reading is done here rather than by the caller, and that is not
- only tidiness: main() is @system, so a template mixed in there infers
- @system for everything it declares, and the output writers are @safe
- and will not take objects of such a type. Doing it inside this module,
- which is @safe:, is what makes the loaded objects the same kind of
- thing the parser's are. Called as spineDocFromArtefact!()(...), the
- way spineAbstraction!() is, rather than mixed in.
- +/
- auto spineDocFromArtefact(P,O,E)(
- string _artefact,
- P program_info,
- O _opt_action,
- E _env,
- ConfComposite _site_conf,
- ) {
- auto _loaded = abstractionLoad(_artefact);
- auto _ssp = _loaded.doc;
- auto _pths = artefactPaths(_artefact, _ssp);
- /+ ↓ the images.
- a .ocda.db carries its own, which is what lets it travel on its
- own, so they are taken from it and written where the output writers
- look for images: a directory of this run's making, not the pod
- beside the artefact, which may be somebody's published tree. A .ssp
- only describes its images, so those are the ones in the pod it sits
- in, and are checked against the digests it recorded.
- +/
- string _images_tmp;
- if (_loaded.loaded) {
- if (_artefact.endsWith(".ssp")) {
- _imagesVerify(_pths.pod_dir, _ssp, _opt_action);
- } else {
- string _extracted = _imagesExtract(_artefact, _opt_action);
- if (_extracted.length > 0) {
- /+ ↓ the source path has to move with it: image_dir_path is
- reached from the document's own file, not from the pod +/
- _images_tmp = _extracted;
- _pths.pod_dir = _extracted;
- _pths.src_file_with_path = (_extracted
- .chainPath("media").chainPath("text")
- .chainPath(("language" in _ssp.source_info)
- ? _ssp.source_info["language"] : "en")
- .chainPath(_ssp.source).array).to!string;
- }
- }
- }
- auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path);
- auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf);
- auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action);
- /+ ↓ the digests: the markup digest is what @source carries, and it is
- the one that says which source this abstraction was made from. The
- header and text digests are of parts an artefact does not keep
- separately, and are left unset rather than guessed at
- +/
- ST_ArtefactDigest _dig;
- if (auto _v = "digest" in _ssp.source_info) {
- _dig.markup_doc = _hexToDigest(*_v);
- }
- auto _abstraction = _ssp.abstraction;
- auto _matters = docMattersMake(
- program_info, _opt_action, _manifest, _cmm, _has, _dig, string[].init,
- );
- /+ ↓ the same shape spineAbstraction() returns, built the same way: a
- struct of accessors over the locals, with whether the artefact
- could be read at all, which the caller has to know +/
- auto theDOC() {
- struct ST_DOC {
- const auto abstraction() { return _abstraction; }
- auto matters() { return _matters; }
- bool loaded() { return _loaded.loaded; }
- string images_tmp() { return _images_tmp; }
- string note() { return _loaded.note; }
- string source_name() { return abstractionSourceName(_loaded.source); }
- }
- return ST_DOC();
- }
- return theDOC();
- }
- /+ ↓ the digests doc_matters carries. Only markup_doc can be filled from
- an artefact, @source having recorded it; the header and text digests
- are of parts no artefact keeps separately and are left unset rather
- than guessed at. Shaped to match ST_doc_digest, which is declared
- inside spineRawMarkupContent and reachable only by parsing a file.
- +/
- struct ST_ArtefactDigest {
- ubyte[32] markup_doc;
- ubyte[32] header;
- ubyte[32] text;
- }
- private ubyte[32] _hexToDigest(string _hex) {
- ubyte[32] _out;
- if (_hex.length < 64) { return _out; }
- foreach (i; 0..32) {
- try { _out[i] = _hex[i*2 .. i*2+2].to!ubyte(16); } catch (Exception ex) {}
- }
- return _out;
- }
-}
-#+END_SRC
-
-* org includes
-** project version
-
-#+NAME: spine_version
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>>
-#+END_SRC
-
-** year
-
-#+NAME: year
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>>
-#+END_SRC
-
-** document header including copyright & license
-
-#+NAME: doc_header_including_copyright_and_license
-#+HEADER: :noweb yes
-#+BEGIN_SRC emacs-lisp
-<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>>
-#+END_SRC
-
-* __END__