/+ - Name: SisuDoc Spine - Description: part of SiSU spine (Doc Reform): for document structuring, processing, publishing and search - Author: Ralph Amissah [ralph.amissah@gmail.com] - Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah, All Rights Reserved. - License: AGPL 3 or later, for everything in this repository unless a file or a directory specifies otherwise. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License (AGPLv3) as published by the Free Software Foundation (FSF), either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see [https://www.gnu.org/licenses/]. If you have Internet connection, the latest version of the AGPL should be available at: [https://www.gnu.org/licenses/agpl-3.0.html] - Spine (by Doc Reform, related to SiSU) uses standard: - docReform markup syntax - standard SiSU markup syntax with modified headers and minor modifications - docReform object numbering - standard SiSU object citation numbering & system - Homepages: [https://www.sisudoc.org] [https://www.doc-reform.org] - Git [https://git.sisudoc.org/] +/ module sisudoc.ocda.abstraction.db_in; @safe: /+ ↓ read a .ocda.db back into the document abstraction the counterpart of sisudoc.outputs.io_out.create_ocda, and the sibling of sisudoc.ocda.abstraction.ssp_in: it returns the same value, so a consumer does not have to know which of the two artefacts it was handed. the check is the same one: read a database, emit it as .ssp with the writer's own record definition, and require the result to equal the .ssp written from the same document. +/ /+ ↓ the languages a database holds, in the order they were written. a caller that wants every language of a document has to be told what they are, and this is the only place that knows: the file is named for the document and says nothing about its languages. . an unreadable or unexpected file gives an empty list rather than an exception, and the caller then treats the artefact as it would any other it could not read. . its own template, and not part of spineAbstractionDbRead: asking a file what languages it holds should not instantiate the whole reader, which brings the object setter and the markup regexes with it. +/ template spineDbLanguages() { @trusted string[] spineDbLanguages(string db_file) { import std.file : exists; import d2sqlite3; string[] _out; if (!db_file.exists) { return _out; } try { auto db = Database(db_file, SQLITE_OPEN_READONLY); foreach (row; db.execute("SELECT lang FROM documents ORDER BY id")) { _out ~= row["lang"].as!string; } } catch (Exception ex) { return string[].init; } return _out; } } /+ ↓ the .ssp digest a database recorded for each of its languages. . What verify holds its answer against: the database says which .ssp it was built from, so re-parsing the markup it carries and emitting the .ssp again either reproduces that digest or does not. . Its own template, for the reason spineDbLanguages is: asking a file what it recorded should not instantiate the reader. +/ template spineDbSspDigests() { @trusted string[string] spineDbSspDigests(string db_file) { import std.file : exists; import d2sqlite3; string[string] _out; if (!db_file.exists) { return _out; } try { auto db = Database(db_file, SQLITE_OPEN_READONLY); foreach (row; db.execute( "SELECT lang, ssp_digest FROM documents ORDER BY id") ) { _out[row["lang"].as!string] = row["ssp_digest"].as!string; } } catch (Exception ex) { return string[string].init; } return _out; } } template spineAbstractionDbRead() { import std.conv : to; import std.file; import std.stdio; import std.string; import std.array; import std.json; import d2sqlite3; import sisudoc.ocda.abstraction.ssp_in : spineAbstractionRead; mixin spineAbstractionRead; // for SSPdocument, and the shared unescape @trusted private int[8] _jsonEight(string _s) { int[8] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (i, v; _j.array) { if (i < _out.length) { _out[i] = v.integer.to!int; } } } catch (Exception ex) {} return _out; } @trusted private int[] _jsonInts(string _s) { int[] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (v; _j.array) { _out ~= v.integer.to!int; } } catch (Exception ex) {} return _out; } @trusted private string[] _jsonStrs(string _s) { string[] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (v; _j.array) { _out ~= v.str; } } catch (Exception ex) {} return _out; } @trusted private double[] _jsonDoubles(string _s) { double[] _out; if (_s.length == 0) { return _out; } try { auto _j = parseJSON(_s); foreach (v; _j.array) { _out ~= (v.type == JSONType.integer) ? v.integer.to!double : v.floating; } } catch (Exception ex) {} return _out; } private ubyte[32] _hex32(string hex) { ubyte[32] _out; if (hex.length < 64) { return _out; } foreach (i; 0..32) { _out[i] = hex[i*2 .. i*2+2].to!ubyte(16); } return _out; } private string _levMarkedUp(int lev) { switch (lev) { case 0: return "A"; case 1: return "B"; case 2: return "C"; case 3: return "D"; case 4: return "1"; case 5: return "2"; case 6: return "3"; case 7: return "4"; default: return ""; } } @trusted SSPdocument dbReadFile(string db_file, string _lang = "") { SSPdocument doc; if (!db_file.exists) { writeln("ERROR: no such file: ", db_file); return doc; } auto db = Database(db_file, SQLITE_OPEN_READONLY); /+ ↓ which document in the file. A file holds one per language; with no language named, take the only one, and say so if there is a choice to be made rather than picking for the caller. +/ long _doc_id = -1; { string[] _langs; foreach (row; db.execute("SELECT id, lang FROM documents ORDER BY id")) { string _l = row["lang"].as!string; _langs ~= _l; if (_lang.length > 0 && _l == _lang) { _doc_id = row["id"].as!long; } else if (_lang.length == 0 && _doc_id < 0) { _doc_id = row["id"].as!long; } } if (_doc_id < 0) { writeln("ERROR: ", db_file, " holds no document in '", _lang, "'; it has: ", _langs.join(" ")); return doc; } if (_lang.length == 0 && _langs.length > 1) { writeln("ERROR: ", db_file, " holds ", _langs.length, " languages and none was named: ", _langs.join(" ")); return doc; } } /+ ↓ the four header blocks, from the metadata table. the key prefix says which block a row belongs to, and insertion order is kept +/ foreach (row; db.execute("SELECT key, value FROM metadata WHERE doc_id = " ~ _doc_id.to!string ~ " OR doc_id IS NULL ORDER BY rowid")) { string _k = row["key"].as!string; string _v = row["value"].as!string; if (_k.startsWith("make.")) { string _kk = _k["make.".length .. $]; doc.make[_kk] = _v; doc.make_order ~= _kk; } else if (_k.startsWith("doc_has.")) { string _kk = _k["doc_has.".length .. $]; doc.doc_has[_kk] = _v; doc.doc_has_order ~= _kk; } else if (_k.startsWith("schema.") || _k.startsWith("build.")) { /+ ↓ what the file is and what wrote it. Neither is part of the document, and only the format version is wanted here. +/ if (_k == "schema.version") { doc.format = "% SiSU Document Abstraction v" ~ _v; } } else if (_k == "source.filename") { doc.source = _v; } else if (_k == "source.ssp_digest") { /+ ↓ this database's own note of the .ssp it was built from. it is not part of the .ssp, so it does not go back into the block +/ doc.ssp_digest = _v; } else if (_k.startsWith("translation.")) { /+ ↓ what the database knows about this language's place among the others, translation.ocn_aligned being the one such key today. A language cannot state it about itself: it is the outcome of a comparison made when every language had been read, which is after the .ssp was written. So it is in the database and not in the .ssp, and putting it back into the header here would make a round trip differ from the file it is held against. Read it from the metadata table, which is where it is. +/ } else if (_k.startsWith("source.")) { string _kk = _k["source.".length .. $]; doc.source_info[_kk] = _v; doc.source_info_order ~= _kk; } else { doc.meta[_k] = _v; doc.meta_order ~= _k; } } if (!sspFormatVersionOk(doc.format, db_file)) { return SSPdocument(); } /+ ↓ the per object lists, gathered first, one sweep of each table. these four tables hold what a single object row cannot: the images an object carries, the links stowed off it, its anchor tags and its lev4 subtoc entries. They are read here in four ordered passes and kept by object id, rather than queried per object as the objects are built. The cost is then what the tables hold, not what the document holds: war and peace has 12,135 objects and 138 rows across all four, which as four statements per object was 48,540 statement preparations and made reading a .ocda.db slower than parsing the markup it came from. +/ ST_file_name_hash_size_[][long] _images_by_id; string[][long] _links_by_id; string[][long] _anchors_by_id; string[][long] _subtoc_by_id; foreach (r; db.execute( "SELECT object_id, name, bytes, sha256, width, height, missing" ~ " FROM object_images WHERE object_id IN (SELECT id FROM objects WHERE doc_id = " ~ _doc_id.to!string ~ ") ORDER BY object_id, seq") ) { ST_file_name_hash_size_ _img; _img.fileName = r["name"].as!string; _img.fileSize = r["bytes"].as!ulong; _img.fileHash_sha256 = _hex32(r["sha256"].as!string); _img.imageWidth = r["width"].as!int; _img.imageHeight = r["height"].as!int; _img.fileMissing = (r["missing"].as!int == 1); _images_by_id[r["object_id"].as!long] ~= _img; } foreach (r; db.execute( "SELECT object_id, url FROM object_links WHERE object_id IN (SELECT id FROM objects WHERE doc_id = " ~ _doc_id.to!string ~ ") ORDER BY object_id, seq") ) { _links_by_id[r["object_id"].as!long] ~= r["url"].as!string; } foreach (r; db.execute( "SELECT object_id, anchor FROM object_anchors WHERE object_id IN (SELECT id FROM objects WHERE doc_id = " ~ _doc_id.to!string ~ ") ORDER BY object_id, seq") ) { _anchors_by_id[r["object_id"].as!long] ~= r["anchor"].as!string; } foreach (r; db.execute( "SELECT object_id, entry FROM object_subtoc WHERE object_id IN (SELECT id FROM objects WHERE doc_id = " ~ _doc_id.to!string ~ ") ORDER BY object_id, seq") ) { _subtoc_by_id[r["object_id"].as!long] ~= r["entry"].as!string; } /+ ↓ the objects, section by section, in the order they were written +/ string[] _sections; foreach (row; db.execute( "SELECT section FROM objects WHERE doc_id = " ~ _doc_id.to!string ~ " GROUP BY section ORDER BY MIN(id)") ) { _sections ~= row["section"].as!string; } foreach (section; _sections) { doc.section_order ~= section; doc.abstraction[section] = []; /+ ↓ column name to index, resolved once for the statement. d2sqlite3's row["name"] is indexForName, a linear scan over the columns that calls sqlite3_column_name and allocates a D string for each one it passes. With some fifty named reads per object over forty columns that is around a thousand of those per object, and it was the larger half of what made reading a database slow. Resolved once here, the reads below are an integer index. +/ int[string] _col; foreach (row; db.execute( "SELECT * FROM objects WHERE doc_id = " ~ _doc_id.to!string ~ " AND section = '" ~ section ~ "' ORDER BY seq") ) { if (_col.length == 0) { foreach (_i; 0 .. row.length) { _col[row.columnName(_i)] = _i.to!int; } } ObjGenericComposite obj; long _id = row[_col["id"]].as!long; obj.metainfo.ocn = row[_col["ocn"]].as!int; obj.metainfo.is_a = row[_col["is_a"]].as!string; obj.metainfo.is_of_part = row[_col["is_of_part"]].as!string; obj.metainfo.is_of_section = (row[_col["is_of_section"]].as!string.length > 0) ? row[_col["is_of_section"]].as!string : section; obj.metainfo.is_of_type = row[_col["is_of_type"]].as!string; obj.metainfo.identifier = row[_col["identifier"]].as!string; obj.metainfo.heading_lev_markup = (obj.metainfo.is_a == "heading") ? row[_col["heading_level"]].as!int : 9; obj.metainfo.heading_lev_collapsed = (row[_col["heading_lev_collapsed"]].as!string.length > 0) ? row[_col["heading_lev_collapsed"]].as!int : 9; obj.metainfo.parent_ocn = row[_col["parent_ocn"]].as!int; obj.metainfo.parent_lev_markup = row[_col["parent_lev"]].as!int; obj.metainfo.last_descendant_ocn = row[_col["last_descendant_ocn"]].as!int; /+ ↓ 2.1: a 2.0 database has no such columns, and reads as no range +/ if ("ocn_range_first" in _col) { obj.metainfo.ocn_range_first = row[_col["ocn_range_first"]].as!int; obj.metainfo.ocn_range_last = row[_col["ocn_range_last"]].as!int; } obj.metainfo.children_headings = _jsonInts(row[_col["children"]].as!string); obj.metainfo.markedup_ancestors = _jsonEight(row[_col["ancestors"]].as!string); obj.metainfo.collapsed_ancestors = _jsonEight(row[_col["ancestors_collapsed"]].as!string); obj.metainfo.dom_structure_markedup_tags_status = _jsonEight(row[_col["dom_status"]].as!string); obj.metainfo.dom_structure_collapsed_tags_status = _jsonEight(row[_col["dom_status_collapsed"]].as!string); obj.metainfo.dummy_heading = (row[_col["dummy_heading"]].as!int == 1); obj.metainfo.object_number_off = (row[_col["object_number_off"]].as!int == 1); obj.metainfo.attrib = row[_col["attrib"]].as!string; obj.metainfo.lang = row[_col["meta_lang"]].as!string; obj.metainfo.syntax = row[_col["meta_syntax"]].as!string; obj.metainfo.sha256.text = _hex32(row[_col["sha256"]].as!string); obj.attrib.indent_base = row[_col["indent_base"]].as!int; obj.attrib.indent_hang = row[_col["indent_hang"]].as!int; obj.attrib.bullet = (row[_col["bullet"]].as!int == 1); obj.attrib.language = row[_col["lang"]].as!string; obj.has.inline_links = (row[_col["has_links"]].as!int == 1); obj.has.inline_notes_reg = (row[_col["has_notes_reg"]].as!int == 1); obj.has.inline_notes_star = (row[_col["has_notes_star"]].as!int == 1); obj.has.images = (row[_col["has_images"]].as!int == 1); obj.has.image_without_dimensions = (row[_col["has_images_no_dim"]].as!int == 1); obj.tags.in_segment_html = row[_col["segment"]].as!string; obj.tags.segname_prev = row[_col["segment_prev"]].as!string; obj.tags.segname_next = row[_col["segment_next"]].as!string; obj.tags.segment_anchor_tag_epub = row[_col["segment_epub"]].as!string; obj.tags.html_segment_anchor_tag_is = row[_col["segment_html_is"]].as!string; obj.tags.epub_segment_anchor_tag_is = row[_col["segment_epub_is"]].as!string; obj.tags.segment_lv4_is = row[_col["segment_lv4_is"]].as!string; obj.tags.anchor_tag_html = row[_col["anchor"]].as!string; obj.tags.heading_lev_anchor_tag = row[_col["heading_lev_anchor"]].as!string; { string[8] _h; foreach (i, h; _jsonStrs(row[_col["heading_ancestors_text"]].as!string)) { if (i < _h.length) { _h[i] = h; } } obj.tags.heading_ancestors_text = _h; } if (obj.metainfo.is_a == "table") { obj.table.number_of_columns = row[_col["table_cols"]].as!int; obj.table.column_widths = _jsonDoubles(row[_col["table_widths"]].as!string); obj.table.column_aligns = _jsonStrs(row[_col["table_aligns"]].as!string); obj.table.heading = (row[_col["table_header"]].as!int == 1); } obj.code_block.linenumbers = (row[_col["code_linenumbers"]].as!int == 1); obj.text = row[_col["text"]].as!string; /+ ↓ the per object lists, from the sweeps above +/ if (auto _v = _id in _images_by_id) { obj.metainfo.sha256.images = *_v; } if (auto _v = _id in _links_by_id) { obj.stow.link = *_v; } if (auto _v = _id in _anchors_by_id) { obj.tags.anchor_tags = *_v; } if (auto _v = _id in _subtoc_by_id) { obj.tags.lev4_subtoc = *_v; } doc.abstraction[section] ~= obj; } } return doc; } /+ ↓ the files a database carries. a .ocda.db holds its images as blobs, which is what lets it travel on its own where a .ssp cannot: a .ssp describes its images by name, size, pixels and sha256, but does not carry them. The role column says what a file is: 'image', and the markup, configuration, manifest and tools carried with it. . data comes back as the original file, decompressed where the row says it was compressed, so that no caller has to know how a blob is stored. bytes and sha256 are the original's either way, and are what a caller checks the data it is given against. +/ struct ST_ArtefactFile { string role; string name; ulong bytes; string sha256; ubyte[] data; } @trusted ST_ArtefactFile[] dbReadFiles(string db_file, string role = "image") { import sisudoc.ocda.zstd : zstdDecompress; ST_ArtefactFile[] _out; if (!db_file.exists) { return _out; } auto db = Database(db_file, SQLITE_OPEN_READONLY); /+ ↓ whether this file has a compression column at all. A database written before the column has none, and selecting it would throw rather than answer; everything in such a file is stored as read. +/ bool _has_compression = false; foreach (row; db.execute( "SELECT count(*) AS n FROM pragma_table_info('files')" ~ " WHERE name = 'compression'") ) { _has_compression = (row["n"].as!long > 0); } auto _stmt = db.prepare( "SELECT role, name, bytes, sha256, " ~ (_has_compression ? "compression" : "NULL AS compression") ~ ", data FROM files WHERE role = :role ORDER BY name" ); _stmt.bind(":role", role); foreach (row; _stmt.execute()) { ST_ArtefactFile _f; _f.role = row["role"].as!string; _f.name = row["name"].as!string; _f.bytes = row["bytes"].as!ulong; _f.sha256 = row["sha256"].as!string; _f.data = row["data"].as!(ubyte[]); string _compression = row["compression"].as!string; if (_compression == "zstd") { try { _f.data = zstdDecompress(_f.data); } catch (Exception ex) { /+ ↓ a row that says it is compressed and will not decompress is not usable, and handing back the frame as if it were the file would put a zstd frame where markup should be. Named, and left out. +/ stderr.writeln("WARNING: ", db_file.baseName, " carries ", _f.name, " as zstd and it will not decompress: ", ex.msg, "; left out"); continue; } } _out ~= _f; } _stmt.finalize(); return _out; } }