/+
- Name: Spine, Doc Reform [a part of]
- Description: documents, structuring, processing, publishing, search
- static content generator
- Author: Ralph Amissah
[ralph.amissah@gmail.com]
- Copyright: (C) 2015 - 2022 Ralph Amissah, All Rights
Reserved.
- License: AGPL 3 or later:
Spine (SiSU), a framework for document structuring, publishing and
search
Copyright (C) Ralph Amissah
This program is free software: you can redistribute it and/or modify it
under the terms of the GNU AFERO General Public License as published by the
Free Software Foundation, either version 3 of the License, or (at your
option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
more details.
You should have received a copy of the GNU General Public License along with
this program. If not, see [https://www.gnu.org/licenses/].
If you have Internet connection, the latest version of the AGPL should be
available at these locations:
[https://www.fsf.org/licensing/licenses/agpl.html]
[https://www.gnu.org/licenses/agpl.html]
- Spine (by Doc Reform, related to SiSU) uses standard:
- docReform markup syntax
- standard SiSU markup syntax with modified headers and minor modifications
- docReform object numbering
- standard SiSU object citation numbering & system
- Homepages:
[https://www.doc_reform.org]
[https://www.sisudoc.org]
- Git
[https://git.sisudoc.org/projects/?p=software/spine.git;a=summary]
+/
module doc_reform.io_out.sqlite;
import doc_reform.io_out;
import
std.file,
std.uri;
import std.conv : to;
import std.typecons : Nullable;
import d2sqlite3;
mixin spineRgxOut;
mixin InternalMarkup;
static auto rgx = RgxO();
static auto mkup = InlineMarkup();
long _metadata_tid_lastrowid;
template SQLiteHubBuildTablesAndPopulate() {
void SQLiteHubBuildTablesAndPopulate(D,M)(
const D doc_abstraction,
M doc_matters,
) {
auto pth_sqlite = spinePathsSQLite!()(doc_matters.sqlite.filename, doc_matters.sqlite.path);
pth_sqlite.base.mkdirRecurse;
auto db = Database(pth_sqlite.sqlite_file);
template SQLiteDbStatementComposite() {
void SQLiteDbStatementComposite(Db,D,M)(
Db db,
const D doc_abstraction,
M doc_matters,
) {
{
{
string _db_statement;
if ((doc_matters.opt.action.sqlite_db_create)) {
auto pth_sqlite = spinePathsSQLite!()(doc_matters.sqlite.filename, doc_matters.sqlite.path);
pth_sqlite.base.mkdirRecurse;
_db_statement ~= SQLiteTablesReCreate!()();
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "TABLE RE-CREATE");
_db_statement = [];
}
if (doc_matters.opt.action.sqlite_delete) {
_db_statement ~= SQLiteDeleteDocument!()(doc_matters);
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "DELETE Document");
_db_statement = [];
}
if (doc_matters.opt.action.sqlite_update) {
_db_statement ~= SQLiteDeleteDocument!()(doc_matters);
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "DELETE Document");
_db_statement = [];
_db_statement ~= SQLiteInsertMetadata!()(doc_matters);
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT MetaData");
_db_statement = [];
/+ get tid (lastrowid or max) for use in doc_objects table +/
_db_statement ~= doc_abstraction.SQLiteInsertDocObjectsLoop!()(doc_matters);
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT DocObjects");
_db_statement = [];
_db_statement ~= SQLiteInsertMetadataTopics!()(doc_matters);
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "INSERT MetaDataTopics");
_db_statement = [];
}
}
db.close;
}
if (!(doc_matters.opt.action.quiet)) {
writeln(" ", pth_sqlite.sqlite_file);
}
}
}
SQLiteDbStatementComposite!()(db, doc_abstraction, doc_matters);
}
}
template SQLiteHubDiscreteBuildTablesAndPopulate() {
void SQLiteHubDiscreteBuildTablesAndPopulate(D,M)(
const D doc_abstraction,
M doc_matters,
) {
auto url_html = spineUrlsHTML!()(doc_matters.conf_make_meta.conf.w_srv_data_root_url_html, doc_matters.src.language);
auto pth_sqlite = spinePathsSQLiteDiscrete!()(doc_matters.output_path, doc_matters.src.language); // doc_matters.db_path
pth_sqlite.base.mkdirRecurse;
auto db = Database(pth_sqlite.sqlite_file(doc_matters.src.filename));
template SQLiteDiscreteDbStatementComposite() {
void SQLiteDiscreteDbStatementComposite(Db,D,M)(
Db db,
const D doc_abstraction,
M doc_matters,
) {
{
{
string _db_statement;
_db_statement ~= SQLiteTablesReCreate!()();
_db_statement ~= SQLiteInsertMetadata!()(doc_matters);
_db_statement ~= SQLiteInsertMetadataTopics!()(doc_matters);
_db_statement ~= doc_abstraction.SQLiteInsertDocObjectsLoop!()(doc_matters);
SQLiteDbRun!()(db, _db_statement, doc_matters.opt.action, "table CREATE Tables, INSERT DocObjects");
}
db.close;
}
if (!(doc_matters.opt.action.quiet)) {
writeln(" ", pth_sqlite.sqlite_file(doc_matters.src.filename));
}
}
}
SQLiteDiscreteDbStatementComposite!()(db, doc_abstraction, doc_matters);
}
}
template SQLiteDbRun() {
void SQLiteDbRun(Db,St,O)(
Db db,
St db_statement,
O opt_action,
string note,
) {
debug(sql_statement) {
writeln(db_statement);
}
try {
db.run(
"\nBEGIN TRANSACTION;\n" ~
db_statement ~
"\nCOMMIT TRANSACTION;\n"
);
} catch (ErrnoException ex) {
writeln("ERROR SQLite : ", ex);
} catch (Exception ex) {
writeln("ERROR SQLite : ", ex);
}
{ /+ debug +/
if (opt_action.debug_do
&& opt_action.verbose) {
writeln(note);
if (opt_action.very_verbose) {
writeln(db_statement);
}
}
}
}
}
template SQLinsertDelimiter() {
string SQLinsertDelimiter(string _txt) {
_txt = _txt
.replaceAll(rgx.quotation_mark_sql_insert_delimiter, "$0$0");
return _txt;
}
}
template SQLiteFormatAndLoadObject() {
auto SQLiteFormatAndLoadObject(M)(
M doc_matters,
) {
mixin spineRgxOut;
struct sqlite_format_and_load_objects {
string generic_munge_sanitize_text_for_search(
string _txt,
) {
string _notes;
string _urls;
if (_txt.matchFirst(rgx.inline_notes_al_gen)) {
foreach (m; _txt.matchAll(rgx.inline_notes_al_gen_text)) {
_notes ~= "\n" ~ m["text"];
}
_txt = _txt.replaceAll(rgx.inline_notes_al_gen, "");
}
if (_txt.matchFirst(rgx.inline_link)) {
foreach (m; _txt.matchAll(rgx.inline_link)) {
if (m["link"].match(rgx.url)) {
_urls ~= "\n" ~ m["link"];
}
}
_txt = _txt.replaceAll(rgx.inline_link_clean, "");
}
if (_notes.length > 0) {
_txt ~= _notes;
}
if (_urls.length > 0) {
_txt ~= _urls;
}
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
writeln(_txt, "\n");
}
}
debug(sql_text_clean) {
writeln(_txt);
}
return _txt;
}
string munge_html(M,O)(
M doc_matters,
const O obj,
) {
string _html_special_characters(string _txt){
_txt = _txt
.replaceAll(rgx.xhtml_ampersand, "&")
.replaceAll(rgx.xhtml_quotation, """)
.replaceAll(rgx.xhtml_less_than, "<")
.replaceAll(rgx.xhtml_greater_than, ">")
.replaceAll(rgx.nbsp_char, " ")
.replaceAll(rgx.br_newline_inline, "
")
.replaceAll(rgx.br_line, "
")
.replaceAll(rgx.xhtml_line_break, "
");
return _txt;
}
string _html_font_face(string _txt){
_txt = _txt
.replaceAll(rgx.inline_emphasis, "$1")
.replaceAll(rgx.inline_bold, "$1")
.replaceAll(rgx.inline_underscore, "$1")
.replaceAll(rgx.inline_italics, "$1")
.replaceAll(rgx.inline_superscript, "$1")
.replaceAll(rgx.inline_subscript, "$1")
.replaceAll(rgx.inline_strike, "$1")
.replaceAll(rgx.inline_insert, "$1")
.replaceAll(rgx.inline_mono, "$1")
.replaceAll(rgx.inline_cite, "$1");
return _txt;
}
string _notes;
string _urls;
string _txt = _html_font_face(_html_special_characters(obj.text));
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
writeln(_txt, "\n");
}
}
return _txt;
}
string html_special_characters(string _txt){
_txt = _txt
.replaceAll(rgx.xhtml_ampersand, "&")
.replaceAll(rgx.xhtml_quotation, """)
.replaceAll(rgx.xhtml_less_than, "<")
.replaceAll(rgx.xhtml_greater_than, ">")
.replaceAll(rgx.nbsp_char, " ")
.replaceAll(rgx.br_newline_inline, "
")
.replaceAll(rgx.br_line, "
")
.replaceAll(rgx.xhtml_line_break, "
");
return _txt;
}
string html_special_characters_code(string _txt){
_txt = _txt
.replaceAll(rgx.xhtml_ampersand, "&")
.replaceAll(rgx.xhtml_quotation, """)
.replaceAll(rgx.xhtml_less_than, "<")
.replaceAll(rgx.xhtml_greater_than, ">")
.replaceAll(rgx.nbsp_char, " ");
return _txt;
}
string html_font_face(string _txt){
_txt = _txt
.replaceAll(rgx.inline_emphasis, "$1")
.replaceAll(rgx.inline_bold, "$1")
.replaceAll(rgx.inline_underscore, "$1")
.replaceAll(rgx.inline_italics, "$1")
.replaceAll(rgx.inline_superscript, "$1")
.replaceAll(rgx.inline_subscript, "$1")
.replaceAll(rgx.inline_strike, "$1")
.replaceAll(rgx.inline_insert, "$1")
.replaceAll(rgx.inline_mono, "$1")
.replaceAll(rgx.inline_cite, "$1");
return _txt;
}
string inline_grouped_text_bullets_indents(M,O)(
M doc_matters,
const O obj,
string _txt,
string _suffix = ".html",
string _xml_type = "seg",
) {
static auto rgx = RgxO();
if (obj.metainfo.is_a == "group") {
_txt = (_txt)
.replaceAll(rgx.grouped_para_indent_1,
" ")
.replaceAll(rgx.grouped_para_indent_2,
" ")
.replaceAll(rgx.grouped_para_indent_3,
" ")
.replaceAll(rgx.grouped_para_indent_4,
" ")
.replaceAll(rgx.grouped_para_indent_5,
" ")
.replaceAll(rgx.grouped_para_indent_6,
" ")
.replaceAll(rgx.grouped_para_indent_7,
" ")
.replaceAll(rgx.grouped_para_indent_8,
" ")
.replaceAll(rgx.grouped_para_indent_9,
" ")
.replaceAll(rgx.grouped_para_indent_hang, " ")
.replaceAll(rgx.grouped_para_bullet, "● ")
.replaceAll(rgx.grouped_para_bullet_indent_1,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_2,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_3,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_4,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_5,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_6,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_7,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_8,
" ● ")
.replaceAll(rgx.grouped_para_bullet_indent_9,
" ● ");
}
return _txt;
}
string inline_images(M,O)(
M doc_matters,
const O obj,
string _txt,
string _suffix = ".html",
string _xml_type = "seg",
) {
string _img_pth;
if (_xml_type == "epub") {
_img_pth = "image/";
} else if (_xml_type == "scroll") {
_img_pth = "../../image/";
} else if (_xml_type == "seg") {
_img_pth = "../../../image/";
}
if (_txt.match(rgx.inline_image)) {
_txt = _txt.replaceAll( // TODO bug where image dimensions (w or h) not given & consequently set to 0; should not be used (calculate earlier, abstraction)
rgx.inline_image,
("$1![](\"")
$6"));
}
return _txt;
}
string inline_links(M,O)(
M doc_matters,
const O obj,
string _txt,
string _xml_type = "seg",
) {
if (obj.has.inline_links) {
if (obj.metainfo.is_a != "code") {
_txt = replaceAll!(m =>
m["linked_text"] ~ "┤" ~ to!string((obj.stow.link[m["num"].to!ulong])).encode ~ "├"
)(_txt, rgx.inline_link_number_only);
}
if ((_txt.match(rgx.mark_internal_site_lnk))
&& (_xml_type == "scroll")) { // conditions reversed to avoid: gdc compiled program run segfault
_txt = _txt.replaceAll(
rgx.inline_seg_link,
"$1");
}
auto pth_html = spinePathsHTML!()(doc_matters.output_path, doc_matters.src.language);
if (_xml_type == "seg") {
foreach (m; _txt.matchAll(rgx.inline_link_seg_and_hash)) {
if (m.captures["hash"] in doc_matters.has.tag_associations) {
if (m.captures["hash"] == doc_matters.has.tag_associations[(m.captures["hash"])]["seg_lv4"]) {
_txt = _txt.replaceFirst(
rgx.inline_link_seg_and_hash,
"┥$1┝┤"
~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html
~ "/"
~ pth_html.tail_fn_seg(doc_matters.src.filename, "$2.html")
~ "├"
);
} else {
_txt = _txt.replaceFirst(
rgx.inline_link_seg_and_hash,
"┥$1┝┤"
~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html
~ "/"
~ doc_matters.src.filename_base
~ "/"
~ doc_matters.has.tag_associations[(m.captures["hash"])]["seg_lv4"]
~ ".html"
~ "#" ~ m.captures["hash"]
~ "├"
);
}
} else {
if (!(doc_matters.opt.action.quiet)) {
writeln(
"WARNING on internal document links, anchor to link <<"
~ m.captures["hash"]
~ ">> not found in document, "
~ "anchor: " ~ m.captures["hash"]
~ " document: " ~ doc_matters.src.filename
);
}
}
}
} else {
if (auto m = _txt.match(rgx.inline_link_seg_and_hash)) {
_txt = _txt.replaceFirst(
rgx.inline_link_seg_and_hash,
"┥$1┝┤"
~ doc_matters.conf_make_meta.conf.w_srv_data_root_url_html
~ "/"
~ pth_html.tail_fn_scroll(doc_matters.src.filename)
~ "#" ~ m.captures["hash"]
~ "├"
);
}
}
_txt = _txt
.replaceAll(
rgx.inline_link_fn_suffix,
("$1.html"))
.replaceAll(
rgx.inline_link,
("$1"))
.replaceAll(
rgx.mark_internal_site_lnk,
"");
}
debug(markup_links) {
if (_txt.match(rgx.inline_link)) {
writeln(__LINE__,
" (missed) markup link identified (",
obj.has.inline_links,
"): ", obj.metainfo.is_a, ": ",
obj.text
);
}
// if (obj.metainfo.is_a == "bookindex") { // DEBUG LINE
// if (_txt.match(regex(r""
~ "" ~ m.captures["num"] ~ "."
~ m.captures["note"]
~ "";
}
_txt = replaceAll!(m =>
(" " ~ "" ~ m["num"] ~ ""))
(_txt, rgx.inline_notes_al_regular_number_note)
~ _endnotes.join("\n");
}
debug(markup_endnotes) {
if (_txt.match(rgx.inline_notes_al_regular_number_note)) {
writeln(__LINE__, " (missed) markup endnote: ", obj.metainfo.is_a, ": ", obj.text);
}
}
debug(markup) {
if (_txt.match(rgx.inline_notes_al_regular_number_note)) {
writeln(__LINE__, " (missed) markup endnote: ", obj.metainfo.is_a, ": ", obj.text);
}
}
return _txt;
}
string xml_type="seg"; /+ set html document type to be linked to here (seg|scroll) +/
string inline_markup(M,O)(
M doc_matters,
const O obj,
string _txt,
) {
if (obj.metainfo.is_a == "group") {
_txt = inline_grouped_text_bullets_indents(doc_matters, obj, _txt, xml_type);
}
_txt = inline_images(doc_matters, obj, _txt, xml_type);
_txt = inline_links(doc_matters, obj, _txt, xml_type);
_txt = inline_notes_scroll(doc_matters, obj, _txt);
return _txt;
}
string html_heading(M,O)(
M doc_matters,
const O obj,
) {
assert(obj.metainfo.is_of_part == "body" || "frontmatter" || "backmatter");
assert(obj.metainfo.is_of_section == "body" || "toc" || "endnotes" || "glossary" || "bibliography" || "bookindex" || "blurb");
assert(obj.metainfo.is_of_type == "para");
assert(obj.metainfo.is_a == "heading");
string _txt = munge_html(doc_matters, obj);
_txt = inline_markup(doc_matters, obj, _txt);
string o = format(q"┃
%s
┃",
obj.metainfo.is_a,
_txt,
);
return o;
}
string html_para(M,O)(
M doc_matters,
const O obj,
) {
assert(obj.metainfo.is_of_part == "body" || "frontmatter" || "backmatter");
assert(obj.metainfo.is_of_section == "body" || "toc" || "endnotes" || "glossary" || "bibliography" || "bookindex" || "blurb");
assert(obj.metainfo.is_of_type == "para");
assert(obj.metainfo.is_a == "para" || "toc" || "endnote" || "glossary" || "bibliography" || "bookindex" || "blurb");
string _txt = munge_html(doc_matters, obj);
_txt = (obj.attrib.bullet) ? ("● " ~ _txt) : _txt;
_txt = inline_markup(doc_matters, obj, _txt);
string o = format(q"┃
%s
┃",
obj.metainfo.is_a,
obj.attrib.indent_hang,
obj.attrib.indent_base,
_txt
);
return o;
}
string html_quote(M,O)(
M doc_matters,
const O obj,
) {
assert(obj.metainfo.is_of_part == "body");
assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb");
assert(obj.metainfo.is_of_type == "block");
assert(obj.metainfo.is_a == "quote");
string _txt = munge_html(doc_matters, obj);
string o = format(q"┃
%s
┃",
obj.metainfo.is_a,
_txt
);
return o;
}
string html_group(M,O)(
M doc_matters,
const O obj,
) {
assert(obj.metainfo.is_of_part == "body");
assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb");
assert(obj.metainfo.is_of_type == "block");
assert(obj.metainfo.is_a == "group");
string _txt = munge_html(doc_matters, obj);
_txt = inline_markup(doc_matters, obj, _txt);
string o = format(q"┃
%s
┃",
obj.metainfo.is_a,
_txt
);
return o;
}
string html_block(M,O)(
M doc_matters,
const O obj,
) {
assert(obj.metainfo.is_of_part == "body");
assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb");
assert(obj.metainfo.is_of_type == "block");
assert(obj.metainfo.is_a == "block");
string _txt = munge_html(doc_matters, obj);
_txt = inline_markup(doc_matters, obj, _txt);
string o = format(q"┃
%s
┃",
obj.metainfo.is_a,
_txt.stripRight
);
return o;
}
string html_verse(M,O)(
M doc_matters,
const O obj,
) {
assert(obj.metainfo.is_of_part == "body");
assert(obj.metainfo.is_of_section == "body" || "glossary" || "bibliography" || "bookindex" || "blurb");
assert(obj.metainfo.is_of_type == "block");
assert(obj.metainfo.is_a == "verse");
string _txt = munge_html(doc_matters, obj);
string o = format(q"┃%s
┃",
obj.metainfo.is_a,
_txt
);
return o;
}
string html_code(O)(
const O obj,
) {
assert(obj.metainfo.is_of_part == "body");
assert(obj.metainfo.is_of_section == "body");
assert(obj.metainfo.is_of_type == "block");
assert(obj.metainfo.is_a == "code");
string _txt = html_special_characters_code(obj.text);
string o = format(q"┃%s
┃",
obj.metainfo.is_a,
_txt
);
return o;
}
string html_table(M,O)(
M doc_matters,
const O obj,
) {
assert(obj.metainfo.is_of_part == "body");
assert(obj.metainfo.is_of_section == "body");
assert(obj.metainfo.is_of_type == "block");
assert(obj.metainfo.is_a == "table");
Tuple!(string, string) _tablarize(O)(
const O obj,
string _txt,
) {
string[] _table_rows = _txt.split(rgx.table_delimiter_row);
string[] _table_cols;
string _table;
string _tablenote;
foreach(row_idx, row; _table_rows) {
_table_cols = row.split(rgx.table_delimiter_col);
_table ~= "";
foreach(col_idx, cell; _table_cols) {
if ((_table_cols.length == 1)
&& (_table_rows.length <= row_idx+2)) { // check row_idx+2 (rather than == ++row_idx)
_tablenote ~= cell;
} else {
string _col_is = (row_idx == 0 && obj.table.heading) ? "th" : "td";
string _align = ("style=\"text-align:"
~ ((obj.table.column_aligns[col_idx] == "l")
? "left\"" : "right\""));
_table ~= "<"
~ _col_is
~ " width=\""
~ obj.table.column_widths[col_idx].to!string
~ "%\" "
~ _align
~ ">";
_table ~= cell;
_table ~= "";
}
}
_table ~= "
";
}
Tuple!(string, string) t = tuple(
_table,
_tablenote,
);
return t;
}
string _txt = munge_html(doc_matters, obj);
Tuple!(string, string) t = _tablarize(obj, _txt);
_txt = t[0];
string _note = t[1];
string o = format(q"┃
%s
┃",
obj.metainfo.is_a,
_txt,
_note
);
return o;
}
string sqlite_load_string(M,O)(
M doc_matters,
const O obj,
) {
string o;
return o;
}
string sqlite_statement(O)(
const O obj,
string _txt,
string _html,
) {
void _sql_exe(O)(
string _sql,
) {
writeln(_html);
writeln(_sql);
}
string _sql;
return _sql;
}
string[string] heading(M,O)(
M doc_matters,
const O obj,
) {
string[string] obj_txt = [
"text": generic_munge_sanitize_text_for_search(obj.text),
"html": html_heading(doc_matters, obj)
];
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
debug(sql_txt) {
writeln(obj_txt["text"]);
}
debug(sql_html) {
writeln(obj_txt["html"]);
}
} else {
// load sql
}
}
return obj_txt;
}
string[string] para(M,O)(
M doc_matters,
const O obj,
) {
string[string] obj_txt = [
"text": generic_munge_sanitize_text_for_search(obj.text),
"html": html_para(doc_matters, obj)
];
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
debug(sql_txt) {
writeln(obj_txt["text"]);
}
debug(sql_html) {
writeln(obj_txt["html"]);
}
} else {
// load sql
}
}
return obj_txt;
}
string[string] quote(M,O)(
M doc_matters,
const O obj,
) {
string[string] obj_txt = [
"text": generic_munge_sanitize_text_for_search(obj.text),
"html": html_quote(doc_matters, obj)
];
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
debug(sql_txt) {
writeln(obj_txt["text"]);
}
debug(sql_html) {
writeln(obj_txt["html"]);
}
} else {
// load sql
}
}
return obj_txt;
}
string[string] group(M,O)(
M doc_matters,
const O obj,
) {
string[string] obj_txt = [
"text": generic_munge_sanitize_text_for_search(obj.text),
"html": html_group(doc_matters, obj)
];
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
debug(sql_txt) {
writeln(obj_txt["text"]);
}
debug(sql_html) {
writeln(obj_txt["html"]);
}
} else {
// load sql
}
}
return obj_txt;
}
string[string] block(M,O)(
M doc_matters,
const O obj,
) {
string[string] obj_txt = [
"text": generic_munge_sanitize_text_for_search(obj.text),
"html": html_block(doc_matters, obj)
];
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
debug(sql_txt) {
writeln(obj_txt["text"]);
}
debug(sql_html) {
writeln(obj_txt["html"]);
}
} else {
// load sql
}
}
return obj_txt;
}
string[string] verse(M,O)(
M doc_matters,
const O obj,
) {
string[string] obj_txt = [
"text": generic_munge_sanitize_text_for_search(obj.text),
"html": html_verse(doc_matters, obj)
];
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
debug(sql_txt) {
writeln(obj_txt["text"]);
}
debug(sql_html) {
writeln(obj_txt["html"]);
}
} else {
// load sql
}
}
return obj_txt;
}
string[string] code(M,O)(
M doc_matters,
const O obj,
) {
string[string] obj_txt = [
"text": generic_munge_sanitize_text_for_search(obj.text),
"html": html_code(obj)
];
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
debug(sql_txt) {
writeln(obj_txt["text"]);
}
debug(sql_html) {
writeln(obj_txt["html"]);
}
} else {
// load sql
}
}
return obj_txt;
}
string[string] table(M,O)(
M doc_matters,
const O obj,
) {
string[string] obj_txt = [
"text": generic_munge_sanitize_text_for_search(obj.text),
"html": html_table(doc_matters, obj)
];
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
debug(sql_txt) {
writeln(obj_txt["text"]);
}
debug(sql_html) {
writeln(obj_txt["html"]);
}
} else {
// load sql
}
}
return obj_txt;
}
}
return sqlite_format_and_load_objects();
}
}
template SQLiteTablesReCreate() {
string SQLiteTablesReCreate()() {
string _sql_instruct;
_sql_instruct = format(q"┃
DROP INDEX IF EXISTS idx_ocn;
DROP INDEX IF EXISTS idx_uid;
DROP INDEX IF EXISTS idx_digest_clean;
DROP INDEX IF EXISTS idx_digest_all;
DROP INDEX IF EXISTS idx_clean;
DROP INDEX IF EXISTS idx_title;
DROP INDEX IF EXISTS idx_author;
DROP INDEX IF EXISTS src_filename_base;
DROP INDEX IF EXISTS idx_language_document_char;
DROP INDEX IF EXISTS idx_classify_topic_register;
DROP INDEX IF EXISTS idx_topic_list;
DROP TABLE IF EXISTS metadata_and_text;
DROP TABLE IF EXISTS topic_register;
DROP TABLE IF EXISTS doc_objects;
DROP TABLE IF EXISTS urls;
CREATE TABLE IF NOT EXISTS metadata_and_text (
uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/])
src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code
src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename
title VARCHAR(800) NOT NULL,
title_main VARCHAR(400) NOT NULL,
title_sub VARCHAR(400) NULL,
title_short VARCHAR(400) NULL,
title_edition VARCHAR(10) NULL,
title_language VARCHAR(100) NULL,
title_language_char VARCHAR(6) NULL,
creator_author VARCHAR(600) NOT NULL,
creator_author_last_first VARCHAR(600) NOT NULL,
creator_author_email VARCHAR(100) NULL,
creator_author_hon VARCHAR(100) NULL,
creator_author_nationality VARCHAR(100) NULL,
creator_editor VARCHAR(600) NULL,
creator_contributor VARCHAR(600) NULL,
creator_illustrator VARCHAR(600) NULL,
creator_photographer VARCHAR(600) NULL,
creator_translator VARCHAR(600) NULL,
creator_prepared_by VARCHAR(600) NULL,
creator_digitized_by VARCHAR(600) NULL,
creator_audio VARCHAR(600) NULL,
creator_video VARCHAR(600) NULL,
language_document VARCHAR(100) NULL,
language_document_char VARCHAR(6) NOT NULL,
language_original VARCHAR(100) NULL,
language_original_char VARCHAR(6) NULL,
date_added_to_site VARCHAR(10) NULL,
date_available VARCHAR(10) NULL,
date_created VARCHAR(10) NULL,
date_issued VARCHAR(10) NULL,
date_modified VARCHAR(10) NULL,
date_published VARCHAR(10) NULL,
date_valid VARCHAR(10) NULL,
date_translated VARCHAR(10) NULL,
date_original_publication VARCHAR(10) NULL,
date_generated VARCHAR(10) NULL,
original_title VARCHAR(800) NULL,
original_publisher VARCHAR(600) NULL,
original_language VARCHAR(100) NULL,
original_language_char VARCHAR(6) NULL,
original_source VARCHAR(600) NULL,
original_institution VARCHAR(600) NULL,
original_nationality VARCHAR(100) NULL,
rights_copyright VARCHAR(2500) NULL,
rights_copyright_audio VARCHAR(2500) NULL,
rights_copyright_cover VARCHAR(2500) NULL,
rights_copyright_illustrations VARCHAR(2500) NULL,
rights_copyright_photographs VARCHAR(2500) NULL,
rights_copyright_text VARCHAR(2500) NULL,
rights_copyright_translation VARCHAR(2500) NULL,
rights_copyright_video VARCHAR(2500) NULL,
rights_license VARCHAR(2500) NULL,
identifier_oclc VARCHAR(30) NULL,
identifier_isbn VARCHAR(16) NULL,
classify_topic_register VARCHAR(2500) NULL,
classify_subject VARCHAR(600) NULL,
classify_loc VARCHAR(30) NULL,
classify_dewey VARCHAR(30) NULL,
classify_keywords VARCHAR(600) NULL,
notes_abstract TEXT NULL,
notes_description TEXT NULL,
notes_comment TEXT NULL,
notes_coverage VARCHAR(200) NULL,
notes_relation VARCHAR(200) NULL,
notes_history VARCHAR(600) NULL,
notes_type VARCHAR(600) NULL,
notes_format VARCHAR(600) NULL,
notes_prefix TEXT NULL,
notes_prefix_a TEXT NULL,
notes_prefix_b TEXT NULL,
notes_suffix TEXT NULL,
publisher VARCHAR(600) NULL,
src_filename_base VARCHAR(256) NOT NULL,
src_filename_suffix VARCHAR(6) NOT NULL,
src_fingerprint VARCHAR(256) NULL,
src_filesize VARCHAR(10) NULL,
src_wordcount VARCHAR(10) NULL,
pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here
pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here
pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here
site_url_doc_root VARCHAR(256) NULL, -- url path to doc root
site_url_html_toc VARCHAR(256) NULL,
site_url_html_scroll VARCHAR(256) NULL,
site_url_epub VARCHAR(256) NULL,
links TEXT NULL
);
CREATE TABLE IF NOT EXISTS topic_register (
-- tid BIGINT PRIMARY KEY,
uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE,
-- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code
-- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename
topic_register VARCHAR(250) NOT NULL,
site_url_doc_root VARCHAR(256) NULL, -- url path to doc root
site_url_html_toc VARCHAR(256) NULL,
site_url_html_scroll VARCHAR(256) NULL
);
CREATE TABLE IF NOT EXISTS doc_objects (
lid BIGINT PRIMARY KEY,
uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE,
ocn SMALLINT,
obj_id VARCHAR(6) NULL,
clean TEXT NULL,
body TEXT NULL,
seg VARCHAR(256) NULL,
lev_an VARCHAR(1),
is_of_type VARCHAR(16),
is_a VARCHAR(16),
lev SMALLINT NULL,
node VARCHAR(16) NULL,
parent VARCHAR(16) NULL,
last_decendant VARCHAR(16) NULL, -- headings only
digest_clean CHAR(256),
digest_all CHAR(256),
seg_name CHAR(256),
types CHAR(1) NULL
);
CREATE INDEX IF NOT EXISTS idx_ocn ON doc_objects(ocn);
CREATE INDEX IF NOT EXISTS idx_digest_clean ON doc_objects(digest_clean);
CREATE INDEX IF NOT EXISTS idx_digest_all ON doc_objects(digest_all);
CREATE INDEX IF NOT EXISTS idx_clean ON doc_objects(clean);
CREATE INDEX IF NOT EXISTS idx_title ON metadata_and_text(title);
CREATE INDEX IF NOT EXISTS idx_author ON metadata_and_text(creator_author_last_first);
CREATE INDEX IF NOT EXISTS idx_uid ON metadata_and_text(uid);
CREATE INDEX IF NOT EXISTS idx_filename ON metadata_and_text(src_filename_base);
CREATE INDEX IF NOT EXISTS idx_language ON metadata_and_text(language_document_char);
CREATE INDEX IF NOT EXISTS idx_topics ON metadata_and_text(classify_topic_register);
CREATE INDEX IF NOT EXISTS idx_topic_list ON topic_register(topic_register);
┃",);
return _sql_instruct;
}
}
template SQLiteDeleteDocument() {
string SQLiteDeleteDocument(M)(
M doc_matters,
) {
string _uid = doc_matters.src.doc_uid;
string _delete_uid = format(q"┃
DELETE FROM metadata_and_text
WHERE uid = '%s';
DELETE FROM doc_objects
WHERE uid_metadata_and_text = '%s';
┃",
_uid,
_uid,
);
return _delete_uid;
}
}
template SQLiteInsertMetadata() {
string SQLiteInsertMetadata(M)(
M doc_matters,
) {
string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid);
string _insert_metadata = format(q"┃
INSERT INTO metadata_and_text (
uid,
src_filename_base,
src_filename_suffix,
src_composite_id_per_txt,
src_composite_id_per_pod,
title,
title_main,
title_sub,
title_short,
title_edition,
title_language,
creator_author,
creator_author_last_first,
creator_author_email,
creator_illustrator,
creator_translator,
language_document,
language_document_char,
date_added_to_site,
date_available,
date_created,
date_issued,
date_modified,
date_published,
date_valid,
rights_copyright,
rights_copyright_audio,
rights_copyright_cover,
rights_copyright_illustrations,
rights_copyright_photographs,
rights_copyright_text,
rights_copyright_translation,
rights_copyright_video,
rights_license,
identifier_oclc,
identifier_isbn,
classify_dewey,
classify_keywords,
classify_loc,
classify_subject,
classify_topic_register,
original_title,
original_publisher,
original_language,
original_language_char,
original_source,
notes_abstract,
notes_description,
publisher,
site_url_doc_root
)
VALUES (
'%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s'
);
┃",
_uid,
SQLinsertDelimiter!()(doc_matters.src.filename_base),
SQLinsertDelimiter!()(doc_matters.src.filename_extension),
SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_doc),
SQLinsertDelimiter!()(doc_matters.src.docname_composite_unique_per_src_pod),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_full),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_main),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_subtitle),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_short),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_edition),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.title_language),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_surname_fn),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_author_email),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_illustrator),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.creator_translator),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.language_document_char),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_added_to_site),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_available),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_created),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_issued),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_modified),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_published),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.date_valid),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_audio),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_cover),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_illustrations),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_photographs),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_text),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_translation),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_copyright_video),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.rights_license),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_oclc),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.identifier_isbn),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_dewey),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_keywords),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_loc),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_subject),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.classify_topic_register),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_abstract),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.notes_description),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_title),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_publisher),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_language_char),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.original_source),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.meta.publisher),
SQLinsertDelimiter!()(doc_matters.conf_make_meta.conf.w_srv_data_root_url_html)
);
return _insert_metadata;
}
}
template SQLiteInsertMetadataTopics() {
string SQLiteInsertMetadataTopics(M)(
M doc_matters,
) {
string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid);
string[] _insert_topics;
foreach (topic_line; doc_matters.conf_make_meta.meta.classify_topic_register_expanded_arr) {
// writeln(topic_line);
_insert_topics ~= format(q"┃
INSERT INTO topic_register (
uid_metadata_and_text,
topic_register
)
VALUES (
'%s',
'%s'
);
┃",
_uid,
SQLinsertDelimiter!()(topic_line)
);
}
return _insert_topics.join.to!(char[]).toUTF8;
}
}
template SQLiteInsertDocObjectsLoop() {
string SQLiteInsertDocObjectsLoop(D,M)(
const D doc_abstraction,
M doc_matters,
) {
string _uid = SQLinsertDelimiter!()(doc_matters.src.doc_uid);
auto url_html = spineUrlsHTML!()(doc_matters.conf_make_meta.conf.w_srv_data_root_url_html, doc_matters.src.language);
string insertDocObjectsRow(O)(O obj) {
string _insert_doc_objects_row = format(q"┃
INSERT INTO doc_objects (
uid_metadata_and_text,
ocn,
obj_id,
clean,
body,
lev,
is_of_type,
is_a,
seg_name
)
VALUES (
'%s', %s, '%s', '%s', '%s', %s, '%s', '%s', '%s'
);
┃",
_uid,
obj.metainfo.ocn,
obj.metainfo.identifier,
SQLinsertDelimiter!()(obj_txt["text"]),
SQLinsertDelimiter!()(obj_txt["html"]),
obj.metainfo.heading_lev_markup,
obj.metainfo.is_of_type,
obj.metainfo.is_a,
obj.tags.html_segment_anchor_tag_is
);
return _insert_doc_objects_row;
}
auto format_and_sqlite_load = SQLiteFormatAndLoadObject!()(doc_matters);
string[string] obj_txt;
string doc_text;
string[] _insert_doc_objects;
foreach (part; doc_matters.has.keys_seq.sql) {
foreach (obj; doc_abstraction[part]) {
switch (obj.metainfo.is_of_part) {
case "frontmatter": assert(part == "head", part);
switch (obj.metainfo.is_of_type) {
case "para":
switch (obj.metainfo.is_a) {
case "heading":
obj_txt = format_and_sqlite_load.heading(doc_matters, obj);
break;
default:
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a);
}
}
break;
}
break;
default:
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type);
}
}
break;
}
break;
case "body": // assert(part == "body", part);
switch (obj.metainfo.is_of_type) {
case "para":
switch (obj.metainfo.is_a) {
case "heading":
debug (asserts) {
if (part != "body") {
writeln(__LINE__, ": ", obj.text);
}
}
obj_txt = format_and_sqlite_load.heading(doc_matters, obj);
break;
case "para":
obj_txt = format_and_sqlite_load.para(doc_matters, obj);
break;
default:
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a);
}
}
break;
}
break;
case "block":
switch (obj.metainfo.is_a) {
case "quote":
obj_txt = format_and_sqlite_load.quote(doc_matters, obj);
break;
case "group":
obj_txt = format_and_sqlite_load.group(doc_matters, obj);
break;
case "block":
obj_txt = format_and_sqlite_load.block(doc_matters, obj);
break;
case "poem": // double check on keeping both poem & verse
break;
case "verse":
obj_txt = format_and_sqlite_load.verse(doc_matters, obj);
break;
case "code":
obj_txt = format_and_sqlite_load.code(doc_matters, obj);
break;
case "table":
obj_txt = format_and_sqlite_load.table(doc_matters, obj);
break;
default:
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a);
}
}
break;
}
break;
default:
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type);
}
}
break;
}
break;
case "backmatter":
assert(part == "glossary" || "bibliography" || "bookindex" || "blurb" || "tail", part);
switch (obj.metainfo.is_of_type) {
case "para":
switch (obj.metainfo.is_a) {
case "heading":
obj_txt = format_and_sqlite_load.heading(doc_matters, obj);
break;
case "glossary": assert(part == "glossary", part);
obj_txt = format_and_sqlite_load.para(doc_matters, obj);
break;
case "bibliography": assert(part == "bibliography", part);
obj_txt = format_and_sqlite_load.para(doc_matters, obj);
break;
case "bookindex": assert(part == "bookindex", part);
obj_txt = format_and_sqlite_load.para(doc_matters, obj);
break;
case "blurb": assert(part == "blurb", part);
obj_txt = format_and_sqlite_load.para(doc_matters, obj);
break;
default:
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a);
}
}
break;
}
break;
default:
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_type);
}
}
break;
}
break;
case "comment":
break;
default:
{ /+ debug +/
if (doc_matters.opt.action.debug_do
&& doc_matters.opt.action.verbose) {
writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_of_part); // check where empty value could come from
writeln(__FILE__, ":", __LINE__, ": ", obj.metainfo.is_a);
writeln(__FILE__, ":", __LINE__, ": ", obj.text); // check where empty value could come from
}
}
break;
}
if (obj.metainfo.is_a == "heading") {
if ((doc_matters.opt.action.very_verbose)) {
writeln(
"markup: ", obj.metainfo.heading_lev_markup,
"> ", obj.metainfo.dom_structure_markedup_tags_status,
"; collapsed: ", obj.metainfo.heading_lev_collapsed,
"> ", obj.metainfo.dom_structure_collapsed_tags_status,
"; ocn: ", obj.metainfo.ocn,
" node: ", obj.metainfo.node,
"; parent: ", obj.metainfo.parent_lev_markup,
"; ocn: ", obj.metainfo.parent_ocn,
"; ",
);
}
}
if (!(obj.metainfo.is_a == "comment")) {
_insert_doc_objects ~= insertDocObjectsRow(obj);
}
} // loop closes
}
return _insert_doc_objects.join.to!(char[]).toUTF8;
}
}
template SQLiteTablesCreate() {
void SQLiteTablesCreate(E,O,C)(E env, O opt_action, C config) {
import d2sqlite3;
template SQLiteTablesReCreate() {
string SQLiteTablesReCreate()() {
string _sql_instruct;
_sql_instruct = format(q"┃
DROP INDEX IF EXISTS idx_ocn;
DROP INDEX IF EXISTS idx_uid;
DROP INDEX IF EXISTS idx_digest_clean;
DROP INDEX IF EXISTS idx_digest_all;
DROP INDEX IF EXISTS idx_clean;
DROP INDEX IF EXISTS idx_title;
DROP INDEX IF EXISTS idx_author;
DROP INDEX IF EXISTS src_filename_base;
DROP INDEX IF EXISTS idx_language_document_char;
DROP INDEX IF EXISTS idx_classify_topic_register;
DROP INDEX IF EXISTS idx_topic_list;
DROP TABLE IF EXISTS metadata_and_text;
DROP TABLE IF EXISTS topic_register;
DROP TABLE IF EXISTS doc_objects;
DROP TABLE IF EXISTS urls;
CREATE TABLE IF NOT EXISTS metadata_and_text (
uid VARCHAR(256) UNIQUE, -- filename, language char, pod/txt (decide on delimiter [,;:/])
src_composite_id_per_txt VARCHAR(256) NOT NULL, -- UNIQUE, z pod name if any + src filename + language code
src_composite_id_per_pod VARCHAR(256) NOT NULL, -- z pod name if any + src filename
title VARCHAR(800) NOT NULL,
title_main VARCHAR(400) NOT NULL,
title_sub VARCHAR(400) NULL,
title_short VARCHAR(400) NULL,
title_edition VARCHAR(10) NULL,
title_language VARCHAR(100) NULL,
title_language_char VARCHAR(6) NULL,
creator_author VARCHAR(600) NOT NULL,
creator_author_last_first VARCHAR(600) NOT NULL,
creator_author_email VARCHAR(100) NULL,
creator_author_hon VARCHAR(100) NULL,
creator_author_nationality VARCHAR(100) NULL,
creator_editor VARCHAR(600) NULL,
creator_contributor VARCHAR(600) NULL,
creator_illustrator VARCHAR(600) NULL,
creator_photographer VARCHAR(600) NULL,
creator_translator VARCHAR(600) NULL,
creator_prepared_by VARCHAR(600) NULL,
creator_digitized_by VARCHAR(600) NULL,
creator_audio VARCHAR(600) NULL,
creator_video VARCHAR(600) NULL,
language_document VARCHAR(100) NULL,
language_document_char VARCHAR(6) NOT NULL,
language_original VARCHAR(100) NULL,
language_original_char VARCHAR(6) NULL,
date_added_to_site VARCHAR(10) NULL,
date_available VARCHAR(10) NULL,
date_created VARCHAR(10) NULL,
date_issued VARCHAR(10) NULL,
date_modified VARCHAR(10) NULL,
date_published VARCHAR(10) NULL,
date_valid VARCHAR(10) NULL,
date_translated VARCHAR(10) NULL,
date_original_publication VARCHAR(10) NULL,
date_generated VARCHAR(10) NULL,
original_title VARCHAR(800) NULL,
original_publisher VARCHAR(600) NULL,
original_language VARCHAR(100) NULL,
original_language_char VARCHAR(6) NULL,
original_source VARCHAR(600) NULL,
original_institution VARCHAR(600) NULL,
original_nationality VARCHAR(100) NULL,
rights_copyright VARCHAR(2500) NULL,
rights_copyright_audio VARCHAR(2500) NULL,
rights_copyright_cover VARCHAR(2500) NULL,
rights_copyright_illustrations VARCHAR(2500) NULL,
rights_copyright_photographs VARCHAR(2500) NULL,
rights_copyright_text VARCHAR(2500) NULL,
rights_copyright_translation VARCHAR(2500) NULL,
rights_copyright_video VARCHAR(2500) NULL,
rights_license VARCHAR(2500) NULL,
identifier_oclc VARCHAR(30) NULL,
identifier_isbn VARCHAR(16) NULL,
classify_topic_register VARCHAR(2500) NULL,
classify_subject VARCHAR(600) NULL,
classify_loc VARCHAR(30) NULL,
classify_dewey VARCHAR(30) NULL,
classify_keywords VARCHAR(600) NULL,
notes_abstract TEXT NULL,
notes_description TEXT NULL,
notes_comment TEXT NULL,
notes_coverage VARCHAR(200) NULL,
notes_relation VARCHAR(200) NULL,
notes_history VARCHAR(600) NULL,
notes_type VARCHAR(600) NULL,
notes_format VARCHAR(600) NULL,
notes_prefix TEXT NULL,
notes_prefix_a TEXT NULL,
notes_prefix_b TEXT NULL,
notes_suffix TEXT NULL,
publisher VARCHAR(600) NULL,
src_filename_base VARCHAR(256) NOT NULL,
src_filename_suffix VARCHAR(6) NOT NULL,
src_fingerprint VARCHAR(256) NULL,
src_filesize VARCHAR(10) NULL,
src_wordcount VARCHAR(10) NULL,
pod_name VARCHAR(256) NULL, -- zipped pod, work to be done here
pod_fingerprint VARCHAR(256) NULL, -- zipped pod, work to be done here
pod_size VARCHAR(10) NULL, -- zipped pod, work to be done here
site_url_doc_root VARCHAR(256) NULL, -- url path to doc root
site_url_html_toc VARCHAR(256) NULL,
site_url_html_scroll VARCHAR(256) NULL,
site_url_epub VARCHAR(256) NULL,
links TEXT NULL
);
CREATE TABLE IF NOT EXISTS topic_register (
-- tid BIGINT PRIMARY KEY,
uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE,
-- src_composite_id_per_txt VARCHAR(256) NOT NULL, - UNIQUE, - z pod name if any + src filename + language code
-- src_composite_id_per_pod VARCHAR(256) NOT NULL, - z pod name if any + src filename
topic_register VARCHAR(250) NOT NULL,
site_url_doc_root VARCHAR(256) NULL, -- url path to doc root
site_url_html_toc VARCHAR(256) NULL,
site_url_html_scroll VARCHAR(256) NULL
);
CREATE TABLE IF NOT EXISTS doc_objects (
lid BIGINT PRIMARY KEY,
uid_metadata_and_text VARCHAR(256) REFERENCES metadata_and_text(uid) ON DELETE CASCADE,
ocn SMALLINT,
obj_id VARCHAR(6) NULL,
clean TEXT NULL,
body TEXT NULL,
seg VARCHAR(256) NULL,
lev_an VARCHAR(1),
is_of_type VARCHAR(16),
is_a VARCHAR(16),
lev SMALLINT NULL,
node VARCHAR(16) NULL,
parent VARCHAR(16) NULL,
last_decendant VARCHAR(16) NULL, -- headings only
digest_clean CHAR(256),
digest_all CHAR(256),
seg_name CHAR(256),
types CHAR(1) NULL
);
CREATE INDEX IF NOT EXISTS idx_ocn ON doc_objects(ocn);
CREATE INDEX IF NOT EXISTS idx_digest_clean ON doc_objects(digest_clean);
CREATE INDEX IF NOT EXISTS idx_digest_all ON doc_objects(digest_all);
CREATE INDEX IF NOT EXISTS idx_clean ON doc_objects(clean);
CREATE INDEX IF NOT EXISTS idx_title ON metadata_and_text(title);
CREATE INDEX IF NOT EXISTS idx_author ON metadata_and_text(creator_author_last_first);
CREATE INDEX IF NOT EXISTS idx_uid ON metadata_and_text(uid);
CREATE INDEX IF NOT EXISTS idx_filename ON metadata_and_text(src_filename_base);
CREATE INDEX IF NOT EXISTS idx_language ON metadata_and_text(language_document_char);
CREATE INDEX IF NOT EXISTS idx_topics ON metadata_and_text(classify_topic_register);
CREATE INDEX IF NOT EXISTS idx_topic_list ON topic_register(topic_register);
┃",);
return _sql_instruct;
}
}
if (opt_action.sqlite_db_create) {
string _db_statement;
string db_filename = (opt_action.sqliteDB_filename.length > 0)
? opt_action.sqliteDB_filename
: (config.conf.w_srv_db_sqlite_filename.length > 0)
? config.conf.w_srv_db_sqlite_filename
: "";
string db_path = (opt_action.sqliteDB_path.length > 0)
? opt_action.sqliteDB_path
: (config.conf.w_srv_db_sqlite_path.length > 0)
? config.conf.w_srv_db_sqlite_path
: "";
if (db_filename.length > 0 && db_path.length > 0) {
if ((opt_action.verbose)) {
writeln("db name & path: ", db_path, db_filename);
}
auto pth_sqlite = spinePathsSQLite!()(db_filename, db_path);
pth_sqlite.base.mkdirRecurse;
auto db = Database(pth_sqlite.sqlite_file);
{
_db_statement ~= SQLiteTablesReCreate!()();
}
SQLiteDbRun!()(db, _db_statement, opt_action, "TABLE RE-CREATE");
} else {
writeln("must provide db name & output root path either on the command line or in configuration file");
writeln("db name: ", db_filename);
writeln("db path: ", db_path);
}
}
}
}
template SQLiteDbDrop() {
void SQLiteDbDrop(O,C)(O opt_action, C config) {
writeln("db drop");
if ((opt_action.sqlite_db_drop)) {
string db_filename = (opt_action.sqliteDB_filename.length > 0)
? opt_action.sqliteDB_filename
: (config.conf.w_srv_db_sqlite_filename.length > 0)
? config.conf.w_srv_db_sqlite_filename
: "";
string db_path = (opt_action.sqliteDB_path.length > 0) //
? opt_action.sqliteDB_path
: (config.conf.w_srv_db_sqlite_path.length > 0)
? config.conf.w_srv_db_sqlite_path
: "";
if (db_filename.length > 0 && db_path.length > 0) {
auto pth_sqlite = spinePathsSQLite!()(db_filename, db_path);
writeln("remove(", pth_sqlite.sqlite_file, ")");
try {
remove(pth_sqlite.sqlite_file);
} catch (FileException ex) {
// handle error
}
} else {
writeln("must provide db name & output root path either on the command line or in configuration file");
writeln("db name: ", db_filename);
writeln("db path: ", db_path);
}
}
}
}