diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-11 21:45:24 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-12 12:39:06 -0400 |
| commit | c6a479b9bd8bdd34ee9699e10247ebdeb183e032 (patch) | |
| tree | 1eab495f7fbf5467c023651aa7f0ee5009215064 /org/in_source_artefact.org | |
| parent | spine: a .ocda.db can be fetched, as a pod zip can (diff) | |
org renames
Diffstat (limited to 'org/in_source_artefact.org')
| -rw-r--r-- | org/in_source_artefact.org | 392 |
1 files changed, 0 insertions, 392 deletions
diff --git a/org/in_source_artefact.org b/org/in_source_artefact.org deleted file mode 100644 index 07c865f..0000000 --- a/org/in_source_artefact.org +++ /dev/null @@ -1,392 +0,0 @@ --*- mode: org -*- -#+TITLE: sisudoc spine (doc_reform) markup source raw -#+DESCRIPTION: documents - structuring, publishing in multiple formats & search -#+FILETAGS: :spine:sourcefile:read: -#+AUTHOR: Ralph Amissah -#+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] -#+COPYRIGHT: Copyright (C) 2015 (continuously updated, current 2026) Ralph Amissah -#+LANGUAGE: en -#+STARTUP: content hideblocks hidestars noindent entitiespretty -#+PROPERTY: header-args+ :eval never-export :exports code -#+PROPERTY: header-args+ :noweb yes :padline no -#+PROPERTY: header-args+ :results silent :cache no -#+PROPERTY: header-args+ :mkdirp yes -#+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t -- magic single double-quote → " ← FIX changes hilighting behavior (occuring - after it) in org document. INVESTIGATE (org-mode CONFIG?) FIND & FIX - -- [[./doc-reform.org][doc-reform.org]] [[./][org/]] - -* A. get doc from artefact -** _module template_ :module:config_files: - -#+HEADER: :tangle "../src/sisudoc/ocda/abstraction/doc_from_artefact.d" -#+HEADER: :noweb yes -#+BEGIN_SRC d -<<doc_header_including_copyright_and_license>> -module sisudoc.ocda.abstraction.doc_from_artefact; -@safe: -/+ ↓ a document, from an artefact rather than from markup - spineAbstraction() parses markup and hands the output writers a doc: - an abstraction and a doc_matters. This does the same from a .ssp or a - .ocda.db, filling the same docMattersMake() arguments, so that what - comes out is the value the writers already take and they never learn - which of the two made it. - . - Three of the seven arguments come from the run and are the same either - way: program_info, opt_action, and the conf half of conf_make_meta, - which is site and run scoped (urls, papersize, the search database) and - has no business in a document's own artefact. The other four come from - the artefact: - . - conf_make_meta meta and make from the @meta and @make blocks, over - the site config the run assembled - doc_has docHasFromAbstraction, over the objects - doc_digest the markup digest from @source - manifest synthesised from the artefact's path and the - document's own name and language - . - What is not here, deliberately: insert_file_list, which is read only by - source_pod.d for --source and --pod2. No artefact carries the markup, - so a pod cannot be rebuilt from one, and the empty list is the truthful - answer rather than a gap. -+/ -template spineDocFromArtefact() { - import std.algorithm : endsWith; - import std.array; - import std.conv : to; - import std.path; - import std.process : thisProcessID; - import std.regex; - import std.stdio; - import std.string; - import sisudoc.ocda.meta.conf_make_meta_structs; - import sisudoc.ocda.meta.doc_matters; - import sisudoc.ocda.io_in.paths_source; - import sisudoc.ocda.abstraction.doc_has; - import sisudoc.ocda.abstraction.load; - import sisudoc.ocda.meta.topic_register; - mixin spineDocMatters; - mixin spineDocHasFromAbstraction; - mixin spineAbstractionLoad; - mixin spineTopicRegister; - /+ ↓ a .ssp header key from a field name: the first underscore becomes the - dot that separates the group, so title_main is title.main and - rights_copyright_text is rights.copyright_text. The writer's keys are - formed the same way, which is what lets this be read off the struct - rather than kept as a second list to fall out of step with it. - +/ - private string _sspKeyOf(string _member) { - foreach (i, c; _member) { - if (c == '_') { return _member[0 .. i] ~ "." ~ _member[i + 1 .. $]; } - } - return _member; - } - /+ ↓ the document's own header, over the site config the run assembled. - every string field of MetaComposite is looked for under its own key, - so a field added there and emitted by the writer arrives here without - anything being added. The arr fields are split from the string they - come from, as agreed: that is parsing a value, not deriving a - structure. - +/ - ConfComposite confMakeMetaFromHeader(S)(S _ssp, ConfComposite _site_conf) { - ConfComposite _out = _site_conf; // conf: site and run scoped, kept - static foreach (_m; __traits(allMembers, MetaComposite)) { - static if (is(typeof(__traits(getMember, _out.meta, _m)) == string)) {{ - enum string _key = _sspKeyOf(_m); - if (auto _v = _key in _ssp.meta) { __traits(getMember, _out.meta, _m) = *_v; } - }} - } - /+ ↓ title_sub is a copy of title_subtitle, made where the header is - read, and is what epub3 puts in dc:title id="subtitle". It is not - emitted, being the same string twice; it is remade here the way the - yaml reader makes it +/ - _out.meta.title_sub = _out.meta.title_subtitle; - /+ ↓ creator.author is the author list joined with ", ", so splitting on - it gives the list back +/ - if (_out.meta.creator_author.length > 0) { - foreach (_a; _out.meta.creator_author.split(", ")) { - if (_a.strip.length > 0) { _out.meta.creator_author_arr ~= _a.strip; } - } - } - /+ ↓ the topic register arrays, by the one rule the yaml reader uses: - the split is not a plain one and deriving it twice would be two - answers to the same question +/ - { - auto _tr = topicRegisterArrays(_out.meta.classify_topic_register); - _out.meta.classify_topic_register_arr = _tr.arr; - _out.meta.classify_topic_register_expanded_arr = _tr.expanded; - } - /+ ↓ make: the keys are the field names, with no group to separate +/ - if (auto _v = "doc_type" in _ssp.make) { - _out.make.doc_type = *_v; _out.make_str.doc_type = *_v; - } - if (auto _v = "auto_num_top_at_level" in _ssp.make) { - _out.make.auto_num_top_at_level = *_v; - } - if (auto _v = "auto_num_top_lv" in _ssp.make) { - try { _out.make.auto_num_top_lv = (*_v).to!int; } catch (Exception ex) {} - } - if (auto _v = "auto_num_depth" in _ssp.make) { - try { _out.make.auto_num_depth = (*_v).to!int; } catch (Exception ex) {} - } - if (auto _v = "breaks" in _ssp.make) { - _out.make.breaks = *_v; _out.make_str.breaks = *_v; - } - if (auto _v = "home_button_text" in _ssp.make) { - _out.make.home_button_text = *_v; _out.make_str.home_button_text = *_v; - } - /+ ↓ footer is a list, joined with the separator the format escapes +/ - if (auto _v = "footer" in _ssp.make) { - _out.make.footer = sspUnescapeSplit(*_v); - _out.make_str.footer = _out.make.footer; - } - return _out; - } - /+ ↓ where a document loaded from an artefact says it is. - an artefact names its document (% Source:) and its language (@source - language) but not where the pod holding its images sits, so that is - taken from where the artefact itself was found: - . - pod/<doc>/media/abstraction/<uid>.ssp -> pod/<doc>/ - pod/<uid>.ocda.db -> pod/<doc>/ - . - which is where --pod2 puts them both. The images beside a .ssp are - then found where html.d already looks; a .ocda.db carries its own and - does not need the directory to exist. - +/ - struct ST_ArtefactPaths { - string pod_dir; - string src_file_with_path; - } - ST_ArtefactPaths artefactPaths(S)(string _artefact, S _ssp) { - ST_ArtefactPaths _out; - string _lang = ("language" in _ssp.source_info) ? _ssp.source_info["language"] : "en"; - string _dir = _artefact.dirName; - if (_artefact.endsWith(".ssp")) { - /+ ↓ <pod>/media/abstraction/<uid>.ssp: two directories up is the pod +/ - _out.pod_dir = (_dir.baseName == "abstraction") - ? _dir.dirName.dirName : _dir; - } else { - /+ ↓ <dir>/<uid>.ocda.db, the pod tree beside it named for the document +/ - string _uid = _artefact.baseName; - foreach (_sfx; [".ocda.db", ".db"]) { - if (_uid.endsWith(_sfx)) { _uid = _uid[0 .. $ - _sfx.length]; break; } - } - if (_uid.endsWith("." ~ _lang)) { _uid = _uid[0 .. $ - _lang.length - 1]; } - _out.pod_dir = (_dir.chainPath(_uid).array).to!string; - } - _out.src_file_with_path = (_out.pod_dir - .chainPath("media").chainPath("text").chainPath(_lang) - .chainPath(_ssp.source).array).to!string; - return _out; - } - /+ ↓ a database's images, written where the writers look for them. - returns the pod directory to use in place of the one beside the - artefact, or "" when the database carries no images and the pod - beside it should stand. Each blob is checked against the sha256 the - database recorded with it: the two were written together, so a - mismatch means the file is damaged, and saying so is the point of - having recorded the digest at all. - +/ - @trusted private string _imagesExtract(O)(string _artefact, O _opt_action) { - import std.digest : toHexString; - import std.digest.sha : sha256Of; - import std.file : mkdirRecurse, write, tempDir; - import sisudoc.ocda.abstraction.db_in : spineAbstractionDbRead; - mixin spineAbstractionDbRead _dbr; - auto _files = _dbr.dbReadFiles(_artefact, "image"); - if (_files.length == 0) { return ""; } - string _root = (tempDir.chainPath("spine-ocda-" - ~ _artefact.baseName ~ "-" ~ thisProcessID.to!string).array).to!string; - string _img_dir = (_root.chainPath("media").chainPath("image").array).to!string; - try { - _img_dir.mkdirRecurse; - } catch (Exception ex) { - stderr.writeln("WARNING: could not make an image directory for ", _artefact, - ": ", ex.msg); - return ""; - } - foreach (_f; _files) { - string _got = _f.data.sha256Of.toHexString.to!string; - if (_f.sha256.length > 0 && _got != _f.sha256) { - stderr.writeln("WARNING: image ", _f.name, " in ", _artefact.baseName, - " does not match the digest recorded with it (", _f.sha256, " expected, ", - _got, " found); it is written out as it stands"); - } - try { - (_img_dir.chainPath(_f.name).array).to!string.write(_f.data); - } catch (Exception ex) { - stderr.writeln("WARNING: could not write ", _f.name, ": ", ex.msg); - } - } - if (_opt_action.vox_gt_1) { - writeln(" ", _files.length, " image(s) from the database, in ", _img_dir); - } - return _root; - } - /+ ↓ the images a .ssp describes, against the ones beside it. - a .ssp carries no image bytes, so what it can do is say whether the - files in the pod it sits in are the ones the abstraction was built - from. Nothing checked that before: a changed or wrong image in the - directory paired silently. - +/ - @trusted private void _imagesVerify(S,O)(string _pod_dir, S _ssp, O _opt_action) { - import std.digest : toHexString; - import std.digest.sha : sha256Of; - import std.file : exists, read; - string _img_dir = (_pod_dir.chainPath("media").chainPath("image").array).to!string; - int _checked, _missing, _wrong; - foreach (_section; _ssp.section_order) { - foreach (obj; _ssp.abstraction[_section]) { - foreach (_img; obj.metainfo.sha256.images) { - if (_img.fileName.length == 0) { continue; } - string _pth = (_img_dir.chainPath(_img.fileName).array).to!string; - if (!_pth.exists) { ++_missing; continue; } - ++_checked; - string _want = _img.fileHash_sha256.toHexString.to!string; - string _got = (cast(ubyte[]) _pth.read).sha256Of.toHexString.to!string; - if (_want != _got && _want != "00000000000000000000000000000000" - ~ "00000000000000000000000000000000") { - ++_wrong; - stderr.writeln("WARNING: image ", _img.fileName, " beside ", - _ssp.source, " is not the one the abstraction was built from"); - } - } - } - } - if (_missing > 0) { - stderr.writeln("WARNING: ", _missing, " image(s) the abstraction names are", - " not in ", _img_dir); - } - if (_opt_action.vox_gt_1 && _checked > 0) { - writeln(" ", _checked, " image(s) checked against the abstraction's digests", - (_wrong == 0) ? ", all matching" : ""); - } - } - /+ ↓ the doc the output writers take, from an artefact. - the reading is done here rather than by the caller, and that is not - only tidiness: main() is @system, so a template mixed in there infers - @system for everything it declares, and the output writers are @safe - and will not take objects of such a type. Doing it inside this module, - which is @safe:, is what makes the loaded objects the same kind of - thing the parser's are. Called as spineDocFromArtefact!()(...), the - way spineAbstraction!() is, rather than mixed in. - +/ - auto spineDocFromArtefact(P,O,E)( - string _artefact, - P program_info, - O _opt_action, - E _env, - ConfComposite _site_conf, - ) { - auto _loaded = abstractionLoad(_artefact); - auto _ssp = _loaded.doc; - auto _pths = artefactPaths(_artefact, _ssp); - /+ ↓ the images. - a .ocda.db carries its own, which is what lets it travel on its - own, so they are taken from it and written where the output writers - look for images: a directory of this run's making, not the pod - beside the artefact, which may be somebody's published tree. A .ssp - only describes its images, so those are the ones in the pod it sits - in, and are checked against the digests it recorded. - +/ - string _images_tmp; - if (_loaded.loaded) { - if (_artefact.endsWith(".ssp")) { - _imagesVerify(_pths.pod_dir, _ssp, _opt_action); - } else { - string _extracted = _imagesExtract(_artefact, _opt_action); - if (_extracted.length > 0) { - /+ ↓ the source path has to move with it: image_dir_path is - reached from the document's own file, not from the pod +/ - _images_tmp = _extracted; - _pths.pod_dir = _extracted; - _pths.src_file_with_path = (_extracted - .chainPath("media").chainPath("text") - .chainPath(("language" in _ssp.source_info) - ? _ssp.source_info["language"] : "en") - .chainPath(_ssp.source).array).to!string; - } - } - } - auto _manifest = PathMatters!()(_opt_action, _env, _pths.pod_dir, _pths.src_file_with_path); - auto _cmm = confMakeMetaFromHeader(_ssp, _site_conf); - auto _has = docHasFromAbstraction(_ssp.abstraction, _ssp.doc_has, _opt_action); - /+ ↓ the digests: the markup digest is what @source carries, and it is - the one that says which source this abstraction was made from. The - header and text digests are of parts an artefact does not keep - separately, and are left unset rather than guessed at - +/ - ST_ArtefactDigest _dig; - if (auto _v = "digest" in _ssp.source_info) { - _dig.markup_doc = _hexToDigest(*_v); - } - auto _abstraction = _ssp.abstraction; - auto _matters = docMattersMake( - program_info, _opt_action, _manifest, _cmm, _has, _dig, string[].init, - ); - /+ ↓ the same shape spineAbstraction() returns, built the same way: a - struct of accessors over the locals, with whether the artefact - could be read at all, which the caller has to know +/ - auto theDOC() { - struct ST_DOC { - const auto abstraction() { return _abstraction; } - auto matters() { return _matters; } - bool loaded() { return _loaded.loaded; } - string images_tmp() { return _images_tmp; } - string note() { return _loaded.note; } - string source_name() { return abstractionSourceName(_loaded.source); } - } - return ST_DOC(); - } - return theDOC(); - } - /+ ↓ the digests doc_matters carries. Only markup_doc can be filled from - an artefact, @source having recorded it; the header and text digests - are of parts no artefact keeps separately and are left unset rather - than guessed at. Shaped to match ST_doc_digest, which is declared - inside spineRawMarkupContent and reachable only by parsing a file. - +/ - struct ST_ArtefactDigest { - ubyte[32] markup_doc; - ubyte[32] header; - ubyte[32] text; - } - private ubyte[32] _hexToDigest(string _hex) { - ubyte[32] _out; - if (_hex.length < 64) { return _out; } - foreach (i; 0..32) { - try { _out[i] = _hex[i*2 .. i*2+2].to!ubyte(16); } catch (Exception ex) {} - } - return _out; - } -} -#+END_SRC - -* org includes -** project version - -#+NAME: spine_version -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_project_version()>> -#+END_SRC - -** year - -#+NAME: year -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:year()>> -#+END_SRC - -** document header including copyright & license - -#+NAME: doc_header_including_copyright_and_license -#+HEADER: :noweb yes -#+BEGIN_SRC emacs-lisp -<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> -#+END_SRC - -* __END__ |
