-*- mode: org -*- #+TITLE: sisudoc spine ≅ (doc-reform) hub #+DESCRIPTION: documents - structuring, various output representations & search #+FILETAGS: :spine:hub: #+AUTHOR: Ralph Amissah #+EMAIL: [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]] #+COPYRIGHT: Copyright (C) 2015 - 2024 Ralph Amissah #+LANGUAGE: en #+STARTUP: content hideblocks hidestars noindent entitiespretty #+PROPERTY: header-args :exports code #+PROPERTY: header-args+ :noweb yes #+PROPERTY: header-args+ :results no #+PROPERTY: header-args+ :cache no #+PROPERTY: header-args+ :padline no #+PROPERTY: header-args+ :mkdirp yes #+OPTIONS: H:3 num:nil toc:t \n:t ::t |:t ^:nil -:t f:t *:t - [[./doc-reform.org][doc-reform.org]] [[./][org/]] - [[./spine_build_scaffold.org][make/build]] VERSION * spine (sisu document parser) :spine: ** notes - deal with imports - get options - get command line instructions - read config instructions - process files as instructed by options - read in file - process file - output ** spine src/spine :template: - process files (act according to requirements of each type) - by sourcefilename - by sourcefiles contents identifier - by zip filename #+HEADER: :tangle "../src/sisudoc/spine.d" #+HEADER: :shebang "#!/usr/bin/env rdmd" #+HEADER: :noweb yes #+BEGIN_SRC d <> module sisudoc.sisu_document_parser; /++ name "spine" description "A SiSU inspired document parser written in D." homepage "https://sisudoc.org" +/ @safe: <> <> <> <> string project_name = "spine"; string program_name = "spine"; @system void main(string[] args) { <> <> <> <> <> <> <> <> <> <> <> <> if (_manifests.length > 1 // _manifests[0] initialized dummy element && _opt_action.abstraction) { <> if (_opt_action.parallelise) { // see else import std.parallelism; foreach(manifest; parallel(_manifests[1..$])) { if (!empty(manifest.src.filename)) { <> <> <> <> <> } else { <> } } } else { // note cannot parallelise sqlite shared db foreach(manifest; _manifests[1..$]) { if (_opt_action.vox_gt2) { writeln("parallelisation off: actions include sqlite shared db"); } if (!empty(manifest.src.filename)) { <> <> <> <> <> } else { <> } } } } if (hvst.curates.length > 0) { if (_opt_action.curate_topics) { spineMetaDocCuratesTopics!()(hvst, _make_and_meta_struct, _opt_action); } if (_opt_action.curate_authors) { spineMetaDocCuratesAuthors!()(hvst.curates, _make_and_meta_struct, _opt_action); } if (_opt_action.vox_gt0) { import sisudoc.io_out.paths_output; auto out_pth = spinePathsHTML!()(_make_and_meta_struct.conf.output_path, ""); if (_opt_action.curate_authors) { writeln("- ", out_pth.curate("authors.html")); } if (_opt_action.curate_topics) { writeln("- ", out_pth.curate("topics.html")); } } } // else { writeln("NO METADATA CURATED"); } } #+END_SRC ** pre-loop init :init: *** init **** imports :import: ***** spine :spine: #+NAME: imports_spine #+BEGIN_SRC d import std.algorithm, std.datetime, std.getopt, std.file, std.path, std.process; import sisudoc.conf.compile_time_info, sisudoc.meta, sisudoc.meta.metadoc, sisudoc.meta.metadoc_curate, sisudoc.meta.metadoc_curate_authors, sisudoc.meta.metadoc_curate_topics, sisudoc.meta.metadoc_from_src, sisudoc.meta.conf_make_meta_structs, sisudoc.meta.conf_make_meta_json, sisudoc.meta.defaults, sisudoc.meta.doc_debugs, sisudoc.meta.rgx, sisudoc.meta.rgx_yaml, sisudoc.meta.rgx_files, sisudoc.io_in.paths_source, sisudoc.io_in.read_config_files, sisudoc.io_in.read_source_files, sisudoc.io_out.hub; #+END_SRC ***** spine metadoc :spine: #+NAME: imports_spine_metadoc #+BEGIN_SRC d import std.datetime, std.digest.crc, std.digest.sha; import sisudoc.meta, sisudoc.meta.metadoc_from_src, sisudoc.meta.conf_make_meta_structs, sisudoc.meta.conf_make_meta_json, sisudoc.meta.defaults, sisudoc.io_in.paths_source, sisudoc.io_in.read_config_files, sisudoc.io_in.read_source_files, sisudoc.io_out.hub; #+END_SRC ****** notes ├── src │   ├── sisudoc.d │   └── spine │ ├── conf │ ├── meta │ └── output └── views    └── version.txt - [[./doc-reform.org][doc-reform.org]] - [[./meta_abstraction.org][meta_abstraction]] - [[./meta_conf_make_meta.org][meta_conf_make_meta]] - [[./meta_defaults.org][meta_defaults]] - [[./meta_output_debugs.org][meta_output_debugs]] - [[./source_read_files.org][source_read_files]] - [[./compile_time_info.org][compile time info]] - [[./output.org][output]] keep up to date, configuration in [[./spine_build_scaffold.org][make/build]] **** mixins :mixin: ***** version.txt :version: #+NAME: mixin_spine_version #+BEGIN_SRC d mixin(import("version.txt")); #+END_SRC ***** configuration.txt :configuration: #+NAME: mixin_spine_configuration #+BEGIN_SRC d mixin(import("configuration.txt")); #+END_SRC ***** pre main mixins #+NAME: mixin_pre_main #+BEGIN_SRC d mixin CompileTimeInfo; #+END_SRC ***** spine "main" mixins :spine: #+NAME: spine_mixin #+BEGIN_SRC d mixin spineRgxIn; mixin spineRgxYamlTags; mixin spineRgxFiles; mixin spineBiblio; mixin outputHub; #+END_SRC #+NAME: spine_metadoc_mixin #+BEGIN_SRC d mixin spineBiblio; mixin outputHub; #+END_SRC **** init :init: #+NAME: spine_init_curate_metadata #+BEGIN_SRC d auto hvst = spineCurateMetadata!(); #+END_SRC **** args :args: #+NAME: spine_init_few #+BEGIN_SRC d string flag_action; string arg_unrecognized; enum dAM { abstraction, matters } static auto rgx = RgxI(); static auto rgx_y = RgxYaml(); static auto rgx_files = RgxFiles(); #+END_SRC *** scope (run complete) :scope: #+NAME: spine_init_scope #+BEGIN_SRC d scope(success) { writefln( "~ run complete, ok ~ (%s-%s.%s.%s, %s D:%s, %s %s)", program_name, _ver.major, _ver.minor, _ver.patch, __VENDOR__, __VERSION__, bits, os, ); } scope(failure) { debug(checkdoc) { stderr.writefln( "run failure", ); } } #+END_SRC *** getopt args for loop :args:getopt: **** set getopt options - set getopt options ***** initialize opts #+NAME: spine_args_init_opts #+BEGIN_SRC d bool[string] opts = [ "abstraction" : false, "allow-downloads" : false, "assertions" : false, "concordance" : false, "dark" : false, "debug" : false, "debug-curate" : false, "debug-curate-authors" : false, "debug-curate-topics" : false, "debug-epub" : false, "debug-harvest" : false, "debug-html" : false, "debug-latex" : false, "debug-manifest" : false, "debug-metadata" : false, "debug-pod" : false, "debug-sqlite" : false, "debug-stages" : false, "digest" : false, "epub" : false, "generated-by" : false, "curate" : false, "curate-authors" : false, "curate-topics" : false, "html" : false, "html-link-curate" : false, "html-link-markup" : false, "html-link-pdf" : false, "html-link-pdf-a4" : false, "html-link-pdf-letter" : false, "html-link-search" : false, "html-seg" : false, "html-scroll" : false, "latex" : false, "latex-color-links" : false, "latex-init" : false, "latex-header-sty" : false, "light" : false, "manifest" : false, "hide-ocn" : false, "ocn-off" : false, "odf" : false, "odt" : false, "parallel" : false, "parallel-subprocesses" : false, "pdf" : false, "pdf-color-links" : false, "pdf-init" : false, "pod" : false, "serial" : false, "show-config" : false, "show-curate" : false, "show-curate-authors" : false, "show-curate-topics" : false, "show-epub" : false, "show-html" : false, "show-latex" : false, "show-make" : false, "show-manifest" : false, "show-metadata" : false, "show-pod" : false, "show-sqlite" : false, "show-summary" : false, "source" : false, "sqlite-discrete" : false, "sqlite-db-create" : false, "sqlite-db-drop" : false, "sqlite-db-recreate" : false, "sqlite-delete" : false, "sqlite-insert" : false, "sqlite-update" : false, "text" : false, "vox_is0" : false, "vox_gt1" : false, "vox_gt2" : false, "xhtml" : false, "section_toc" : true, "section_body" : true, "section_endnotes" : true, "section_glossary" : true, "section_biblio" : true, "section_bookindex" : true, "section_blurb" : true, "backmatter" : true, "skip-output" : false, "theme-dark" : false, "theme-light" : false, "workon" : false, ]; #+END_SRC ***** initialize settings NOTES on configuration hierarchy: see meta_conf_make_meta.org #+NAME: spine_args_init_settings #+BEGIN_SRC d string[string] settings = [ "output" : "", "www-http" : "", "www-host" : "", "www-host-doc-root" : "", "www-url-doc-root" : "", "cgi-http" : "", "cgi-host" : "", "cgi-bin-root" : "", "cgi-sqlite-search-filename" : "", "cgi-url-root" : "", "cgi-url-action" : "", "cgi-search-title" : "", "config" : "", "lang" : "all", "set-papersize" : "", "set-textwrap" : "", "set-digest" : "", "sqlite-db-path" : "", "sqlite-db-filename" : "", ]; #+END_SRC ***** set & describe #+NAME: spine_args_get_options_aa #+BEGIN_SRC d auto helpInfo = getopt(args, std.getopt.config.passThrough, "abstraction", "document abstraction", &opts["abstraction"], "allow-downloads", "allow downloads (includes cgi.d from github)", &opts["allow-downloads"], "assert", "set optional assertions on", &opts["assertions"], "cgi-bin-root", "path to cgi-bin directory", &settings["cgi-bin-root"], "cgi-url-root", "url to cgi-bin (to find cgi-bin)", &settings["cgi-url-root"], "cgi-url-action", "url to post to cgi-bin search form", &settings["cgi-url-action"], "cgi-search-title", "if generating a cgi search form the title to use for it", &settings["cgi-search-title"], "cgi-sqlite-search-filename", "=[filename] default is spine-search", &settings["cgi-sqlite-search-filename"], "concordance", "file for document", &opts["concordance"], "curate", "extract info on authors & topics from document header metadata", &opts["curate"], "curate-authors", "extract info on authors from document header metadata", &opts["curate-authors"], "curate-topics", "extract info on topics from document header metadata", &opts["curate-topics"], "dark", "alternative dark theme", &opts["dark"], "digest", "hash digest for each object", &opts["digest"], "epub", "process epub output", &opts["epub"], "generated-by", "generated by headers (software version & time)", &opts["generated-by"], "hide-ocn", "object cite numbers", &opts["hide-ocn"], "html", "process html output", &opts["html"], "html-link-curate", "place links back to curate in segmented html", &opts["html-link-curate"], "html-link-markup", "provide html link to markup source, shared optionally", &opts["html-link-markup"], "html-link-pdf", "provide html link to pdf a4 & letter output", &opts["html-link-pdf"], "html-link-pdf-a4", "provide html link to pdf a4 output", &opts["html-link-pdf-a4"], "html-link-pdf-letter", "provide html link to pdf letter size output", &opts["html-link-pdf-letter"], "html-link-search", "html embedded search submission", &opts["html-link-search"], "html-seg", "process html output", &opts["html-seg"], "html-scroll", "process html output", &opts["html-scroll"], "lang", "=[lang code e.g. =en or =en,es]", &settings["lang"], "latex", "latex output (for pdfs)", &opts["latex"], "latex-color-links", "mono or color links for pdfs", &opts["latex-color-links"], "latex-init", "initialise latex shared files (see latex-header-sty)", &opts["latex-init"], "latex-header-sty", "latex document header sty files", &opts["latex-header-sty"], "light", "default light theme", &opts["light"], "manifest", "process manifest output", &opts["manifest"], "ocn-off", "object cite numbers", &opts["ocn-off"], "odf", "open document format text (--odt)", &opts["odf"], "odt", "open document format text", &opts["odt"], "output", "=/path/to/output/dir specify where to place output", &settings["output"], "parallel", "parallelisation", &opts["parallel"], "parallel-subprocesses", "nested parallelisation", &opts["parallel-subprocesses"], "pdf", "latex output for pdfs", &opts["pdf"], "pdf-color-links", "mono or color links for pdfs", &opts["pdf-color-links"], "pdf-init", "initialise latex shared files (see latex-header-sty)", &opts["pdf-init"], "pod", "spine (doc reform) pod source content bundled", &opts["pod"], "quiet|q", "output to terminal", &opts["vox_is0"], "section-backmatter", "document backmatter (default)" , &opts["backmatter"], "section-biblio", "document biblio (default)", &opts["section_biblio"], "section-blurb", "document blurb (default)", &opts["section_blurb"], "section-body", "document body (default)", &opts["section_body"], "section-bookindex", "document bookindex (default)", &opts["section_bookindex"], "section-endnotes", "document endnotes (default)", &opts["section_endnotes"], "section-glossary", "document glossary (default)", &opts["section_glossary"], "section-toc", "table of contents (default)", &opts["section_toc"], "serial", "serial processing", &opts["serial"], "skip-output", "skip output", &opts["skip-output"], "show-config", "show config", &opts["show-config"], "show-curate", "show curate", &opts["show-curate"], "show-curate-authors", "show curate authors", &opts["show-curate-authors"], "show-curate-topics", "show curate topics", &opts["show-curate-topics"], "show-epub", "show epub", &opts["show-epub"], "show-html", "show html", &opts["show-html"], "show-latex", "show latex", &opts["show-latex"], "show-make", "show make", &opts["show-make"], "show-manifest", "show manifest", &opts["show-manifest"], "show-metadata", "show metadata", &opts["show-metadata"], "show-pod", "show pod", &opts["show-pod"], "show-sqlite", "show sqlite", &opts["show-sqlite"], "show-summary", "show summary", &opts["show-summary"], "source", "document markup source", &opts["source"], "set-digest", "default hash digest type (e.g. sha256)", &settings["set-digest"], "set-papersize", "default papersize (latex pdf eg. a4 or a5 or b4 or letter)", &settings["set-papersize"], "set-textwrap", "default textwrap (e.g. 80 (characters)", &settings["set-textwrap"], "sqlite-discrete", "process discrete sqlite output", &opts["sqlite-discrete"], "sqlite-db-create", "create db, create tables", &opts["sqlite-db-create"], "sqlite-db-drop", "drop tables & db", &opts["sqlite-db-drop"], "sqlite-db-filename", "sqlite db to create, populate & make available for search", &settings["sqlite-db-filename"], "sqlite-db-path", "sqlite db path", &settings["sqlite-db-path"], "sqlite-db-recreate", "create db, create tables", &opts["sqlite-db-recreate"], "sqlite-delete", "sqlite output", &opts["sqlite-delete"], "sqlite-insert", "sqlite output", &opts["sqlite-insert"], "sqlite-update", "sqlite output", &opts["sqlite-update"], "www-http", "http or https", &settings["www-http"], "www-host", "web server host (domain) name", &settings["www-host"], "www-host-doc-root", "web host host (domain) name with path to doc root", &settings["www-host-doc-root"], "www-url-doc-root", "e.g. http://localhost", &settings["www-url-doc-root"], "text", "text output", &opts["text"], "theme-dark", "alternative dark theme", &opts["theme-dark"], "theme-light", "default light theme", &opts["theme-light"], "txt", "text output", &opts["text"], "verbose|v", "output to terminal", &opts["vox_gt1"], "very-verbose", "output to terminal", &opts["vox_gt2"], "workon", "(reserved for some matters under development & testing)", &opts["workon"], "xhtml", "xhtml output", &opts["xhtml"], "config", "=/path/to/config/file/including/filename", &settings["config"], "debug", "debug", &opts["debug"], "debug-curate", "debug curate", &opts["debug-curate"], "debug-curate-authors", "debug curate authors", &opts["debug-curate-authors"], "debug-curate-topics", "debug curate topics", &opts["debug-curate-topics"], "debug-epub", "debug epub", &opts["debug-epub"], "debug-harvest", "debug harvest", &opts["debug-harvest"], "debug-html", "debug html", &opts["debug-html"], "debug-latex", "debug latex", &opts["debug-latex"], "debug-manifest", "debug manifest", &opts["debug-manifest"], "debug-metadata", "debug metadata", &opts["debug-metadata"], "debug-pod", "debug pod", &opts["debug-pod"], "debug-sqlite", "debug sqlite", &opts["debug-sqlite"], "debug-stages", "debug stages", &opts["debug-stages"], // "sqlite-db-filename", "=[filename].sql.db", &settings["sqlite-db-filename"], ); if (helpInfo.helpWanted) { defaultGetoptPrinter("Some information about the program.", helpInfo.options); } #+END_SRC **** getopt hash to struct - move getopt options to struct #+NAME: spine_args_get_options_aa2str #+BEGIN_SRC d enum outTask { source_or_pod, sqlite, sqlite_multi, latex, odt, epub, html_scroll, html_seg, html_stuff } struct OptActions { @trusted bool allow_downloads() { return opts["allow-downloads"]; } @trusted bool assertions() { return opts["assertions"]; } @trusted bool concordance() { return opts["concordance"]; } @trusted string config_path_set() { return settings["config"]; } @trusted bool css_theme_default() { bool _is_light; if (opts["light"] || opts["theme-light"]) { _is_light = true; } else if (opts["dark"] || opts["theme-dark"]) { _is_light = false; } else { _is_light = true; } return _is_light; } @trusted bool debug_do() { bool _dbg; if (opts["debug"]) { _dbg = true; } else { _dbg = false; } return _dbg; } @trusted bool debug_do_curate() { return (opts["debug"] || opts["debug-curate"]) ? true : false; } @trusted bool debug_do_curate_authors() { return (opts["debug"] || opts["debug-curate"] || opts["debug-curate-authors"]) ? true : false; } @trusted bool debug_do_curate_topics() { return (opts["debug"] || opts["debug-curate"] || opts["debug-curate-topics"]) ? true : false; } @trusted bool debug_do_epub() { return (opts["debug"] || opts["debug-epub"]) ? true : false; } @trusted bool debug_do_harvest() { return (opts["debug"] || opts["debug-harvest"]) ? true : false; } @trusted bool debug_do_html() { return (opts["debug"] || opts["debug-html"]) ? true : false; } @trusted bool debug_do_latex() { return (opts["debug"] || opts["debug-latex"]) ? true : false; } @trusted bool debug_do_manifest() { return (opts["debug"] || opts["debug-manifest"]) ? true : false; } @trusted bool debug_do_metadata() { return (opts["debug"] || opts["debug-metadata"]) ? true : false; } @trusted bool debug_do_pod() { return (opts["debug"] || opts["debug-pod"]) ? true : false; } @trusted bool debug_do_sqlite() { return (opts["debug"] || opts["debug-sqlite"]) ? true : false; } @trusted bool debug_do_stages() { return (opts["debug"] || opts["debug-stages"]) ? true : false; } @trusted bool debug_do_xmls() { return (opts["debug"] || opts["debug-html"] || opts["debug-epub"]) ? true : false; } @trusted bool curate() { return (opts["curate"] || opts["curate-authors"] || opts["curate-topics"]) ? true : false; } @trusted bool curate_authors() { return (opts["curate"] || opts["curate-authors"]) ? true : false; } @trusted bool curate_topics() { return (opts["curate"] || opts["curate-topics"]) ? true : false; } @trusted bool digest() { return opts["digest"]; } @trusted bool epub() { return opts["epub"]; } @trusted bool generated_by() { return opts["generated-by"]; } @trusted bool html_link_curate() { return (opts["html-link-curate"]) ? true : false; } @trusted bool html_link_markup_source() { return (opts["html-link-markup"]) ? true : false; } @trusted bool html_link_pdf() { return (opts["html-link-pdf"]) ? true : false; } @trusted bool html_link_pdf_a4() { return (opts["html-link-pdf-a4"]) ? true : false; } @trusted bool html_link_pdf_letter() { return (opts["html-link-pdf-letter"]) ? true : false; } @trusted bool html_link_search() { return (opts["html-link-search"]) ? true : false; } @trusted bool html() { return (opts["html"] || opts["html-seg"] || opts["html-scroll"]) ? true : false; } @trusted bool html_seg() { return (opts["html"] || opts["html-seg"]) ? true : false; } @trusted bool html_scroll() { return (opts["html"] || opts["html-scroll"]) ? true : false; } @trusted bool html_stuff() { return (opts["html"] || opts["html-scroll"] || opts["html-seg"]) ? true : false; } @trusted bool latex() { return (opts["latex"] || opts["pdf"]) ? true : false; } @trusted bool latex_color_links() { return (opts["latex-color-links"] || opts["pdf-color-links"]) ? true : false; } @trusted bool latex_document_header_sty() { return (opts["latex-init"] || opts["latex-header-sty"] || opts["pdf-init"]) ? true : false; } @trusted bool manifest() { return opts["manifest"]; } @trusted bool odt() { return (opts["odf"] || opts["odt"]) ? true : false; } @trusted bool ocn_hidden() { return opts["hide-ocn"]; } @trusted bool ocn_off() { return opts["ocn-off"]; } @trusted bool pod() { return opts["pod"]; } @trusted bool show_config() { return opts["show-config"]; } @trusted bool show_curate() { return opts["show-curate"]; } @trusted bool show_curate_authors() { return (opts["show-curate"] || opts["show-curate-authors"] || opts["vox_gt1"] || opts["vox_gt2"]) ? true : false; } @trusted bool show_curate_topics() { return (opts["show-curate"] || opts["show-curate-topics"] || opts["vox_gt2"]) ? true : false; } @trusted bool show_epub() { return opts["show-epub"]; } @trusted bool show_html() { return opts["show-html"]; } @trusted bool show_latex() { return opts["show-latex"]; } @trusted bool show_make() { return opts["show-make"]; } @trusted bool show_manifest() { return opts["show-manifest"]; } @trusted bool show_metadata() { return opts["show-metadata"]; } @trusted bool show_pod() { return opts["show-pod"]; } @trusted bool show_sqlite() { return (opts["show-sqlite"] || opts["vox_gt2"]) ? true : false; } @trusted bool show_summary() { return (opts["show-summary"] || opts["vox_gt1"] || opts["vox_gt2"]) ? true : false; } @trusted bool source() { return opts["source"]; } @trusted bool source_or_pod() { return (opts["pod"] || opts["source"]) ? true : false; } @trusted bool sqlite_discrete() { return opts["sqlite-discrete"]; } @trusted bool sqlite_db_drop() { return (opts["sqlite-db-recreate"] || opts["sqlite-db-drop"]) ? true : false; } @trusted bool sqlite_db_create() { return (opts["sqlite-db-recreate"] || opts["sqlite-db-create"]) ? true : false; } @trusted bool sqlite_delete() { return opts["sqlite-delete"]; } @trusted bool sqlite_update() { return (opts["sqlite-update"] || opts["sqlite-insert"]) ? true : false; } @trusted bool sqlite_shared_db_action() { return ( opts["sqlite-db-recreate"] || opts["sqlite-db-create"] || opts["sqlite-delete"] || opts["sqlite-insert"] || opts["sqlite-update"] ) ? true : false; } @trusted bool vox_is0() { // --quiet -q return opts["vox_is0"]; } @trusted bool vox_gt0() { // normal, minimal, without flag return (!(opts["vox_is0"]) || opts["vox_gt1"] || opts["vox_gt2"]) ? true : false; } @trusted bool vox_gt1() { // -- verbose -v return (opts["vox_gt1"] || opts["vox_gt2"]) ? true : false; } @trusted bool vox_gt2() { // --very-verbose return opts["vox_gt2"]; } @trusted bool text() { return opts["text"]; } @trusted bool xhtml() { return opts["xhtml"]; } @trusted bool section_toc() { return opts["section_toc"]; } @trusted bool section_body() { return opts["section_body"]; } @trusted bool section_endnotes() { return opts["section_endnotes"]; } @trusted bool section_glossary() { return opts["section_glossary"]; } @trusted bool section_biblio() { return opts["section_biblio"]; } @trusted bool section_bookindex() { return opts["section_bookindex"]; } @trusted bool section_blurb() { return opts["section_blurb"]; } @trusted bool backmatter() { return opts["backmatter"]; } @trusted bool skip_output() { return opts["skip-output"]; } @trusted bool workon() { return opts["workon"]; } @trusted string[] languages_set() { return settings["lang"].split(","); } @trusted string output_dir_set() { return settings["output"]; } @trusted string sqliteDB_filename() { return settings["sqlite-db-filename"]; } @trusted string sqliteDB_path() { return settings["sqlite-db-path"]; } @trusted string cgi_bin_root() { return settings["cgi-bin-root"]; } @trusted string cgi_search_title() { return settings["cgi-search-title"]; } @trusted string cgi_sqlite_search_filename() { return settings["cgi-sqlite-search-filename"]; } @trusted string cgi_sqlite_search_filename_d() { return (settings["cgi-sqlite-search-filename"].length > 0) ? (settings["cgi-sqlite-search-filename"].translate(['-' : "_"]) ~ ".d") : ""; } @trusted string cgi_url_root() { return settings["cgi-url-root"]; } @trusted string cgi_url_action() { return settings["cgi-url-action"]; } @trusted string hash_digest_type() { return settings["set-digest"]; } @trusted string text_wrap() { return settings["set-textwrap"]; } @trusted string latex_papersize() { return settings["set-papersize"]; } @trusted string webserver_host_name() { return settings["www-host"]; } @trusted string webserver_host_doc_root() { return settings["www-host-doc-root"]; } @trusted string webserver_url_doc_root() { return settings["www-url-doc-root"]; } @trusted string webserver_http() { return settings["www-http"]; } @trusted bool parallelise() { bool _is; if (opts["serial"] == true) { _is = false; } else if ( sqlite_shared_db_action || source_or_pod ) { _is = false; } else if (opts["parallel"] == true) { _is = true; if ( sqlite_shared_db_action || source_or_pod ) { _is = false; } } else if ( opts["abstraction"] || concordance || curate || html || epub || odt || latex || manifest || sqlite_discrete ) { _is = true; } else { _is = false; } return _is; } @trusted bool parallelise_subprocesses() { return opts["parallel-subprocesses"]; } auto output_task_scheduler() { int[] schedule; if (source_or_pod) { schedule ~= outTask.source_or_pod; } if (sqlite_discrete) { schedule ~= outTask.sqlite; } if (epub) { schedule ~= outTask.epub; } if (html_scroll) { schedule ~= outTask.html_scroll; } if (html_seg) { schedule ~= outTask.html_seg; } if (html_stuff) { schedule ~= outTask.html_stuff; } if (odt) { schedule ~= outTask.odt; } if (latex) { schedule ~= outTask.latex; } return schedule.sort().uniq; } @trusted bool abstraction() { return ( opts["abstraction"] || concordance || source_or_pod || curate || html || epub || odt || latex || manifest || sqlite_discrete || sqlite_delete || sqlite_update ) ? true : false; } @trusted bool require_processing_files() { return ( opts["abstraction"] || epub || curate || html || html_seg || html_scroll || latex || odt || manifest || show_make || show_metadata || show_summary || source_or_pod || sqlite_discrete || sqlite_update || text || xhtml ) ? true : false; } @trusted bool meta_processing_general() { return ( opts["abstraction"] || curate || html || epub || odt || latex || sqlite_discrete || sqlite_update ) ? true :false; } @trusted bool meta_processing_xml_dom() { return ( opts["abstraction"] || html || epub || odt || sqlite_discrete || sqlite_update ) ? true : false; } } OptActions _opt_action = OptActions(); #+END_SRC **** env - environmental info #+NAME: spine_args_program_info #+BEGIN_SRC d auto program_info() { struct ProgramInfo { string project() { return project_name; } string name() { return program_name; } string ver() { return format("%s.%s.%s", _ver.major, _ver.minor, _ver.patch, ); } string compiler() { return format ("%s D:%s, %s %s", __VENDOR__, __VERSION__, bits, os, ); } @trusted string name_and_version() { return format("%s-%s", name, ver); } @trusted string name_version_and_compiler() { return format("%s-%s (%s)", name, ver, compiler); } auto time_output_generated() { auto _st = Clock.currTime(UTC()); auto _t = TimeOfDay(_st.hour, _st.minute, _st.second); auto _time = _st.year.to!string ~ "-" ~ _st.month.to!int.to!string // prefer as month number ~ "-" ~ _st.day.to!string ~ " [" ~ _st.isoWeek.to!string ~ "/" ~ _st.dayOfWeek.to!int.to!string ~ "]" ~ " - " ~ _t.toISOExtString // ~ " " ~ _st.hour.to!string ~ ":" ~ _st.minute.to!string ~ ":" ~ _st.second.to!string ~ " UTC"; return _time; // return _st.toISOExtString(); } } return ProgramInfo(); } auto _env = [ "pwd" : environment["PWD"], "home" : environment["HOME"], ]; #+END_SRC *** get/read site config #+NAME: spine_args_get_configuration_details #+BEGIN_SRC d auto _manifested = PathMatters!()(_opt_action, _env, ""); auto _manifests = [ _manifested ]; auto _conf_file_details = configFilePaths!()(_manifested, _env, _opt_action.config_path_set); ConfComposite _siteConfig; if ( _opt_action.require_processing_files && _opt_action.config_path_set.empty ) { foreach(arg; args[1..$]) { if (!(arg.match(rgx.flag_action))) { /+ cli markup source path +/ // get first input markup source file names for processing _manifested = PathMatters!()(_opt_action, _env, arg); { /+ local site config +/ _conf_file_details = configFilePaths!()(_manifested, _env, _opt_action.config_path_set); auto _config_local_site_struct = readConfigSite!()(_conf_file_details, _opt_action, _cfg); import sisudoc.meta.conf_make_meta_yaml; _siteConfig = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_siteConfig, _manifested, _opt_action, _cfg); // - get local site config break; } } } } else { /+ local site config +/ auto _config_local_site_struct = readConfigSite!()(_conf_file_details, _opt_action, _cfg); import sisudoc.meta.conf_make_meta_yaml; _siteConfig = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_siteConfig, _manifested, _opt_action, _cfg); // - get local site config } if (_opt_action.show_config) { import sisudoc.meta.metadoc_show_config; spineShowSiteConfig!()(_opt_action, _siteConfig); } #+END_SRC *** use config for operations that do not require file processing #+NAME: spine_args_call_output_hub #+BEGIN_SRC d if (!(_opt_action.skip_output)) { if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("step0 commence → (without processing files)"); } outputHubOp!()(_env, _opt_action, _siteConfig); if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("- step0 complete"); } } #+END_SRC *** opt action on processing files (loop args) #+NAME: spine_args_process_args #+BEGIN_SRC d ConfComposite _make_and_meta_struct = _siteConfig; destroy(_siteConfig); foreach(arg; args[1..$]) { if (arg.match(rgx.flag_action)) { /+ cli instruction, flag do +/ flag_action ~= " " ~ arg; // flags not taken by getopt } else if (_opt_action.require_processing_files) { /+ cli, assumed to be path to source files +/ auto _manifest_start = PodManifest!()(_opt_action, arg); if ( /+ pod files +/ !(arg.match(rgx_files.src_pth_sst_or_ssm)) && _manifest_start.pod_manifest_file_with_path && _opt_action.abstraction ) { string pod_manifest_root_content_paths_to_markup_location_raw_; string markup_contents_location_; string sisudoc_txt_ = _manifest_start.pod_manifest_file_with_path; enforce( exists(sisudoc_txt_)!=0, "file not found: «" ~ sisudoc_txt_ ~ "»" ); if (exists(sisudoc_txt_)) { try { if (exists(sisudoc_txt_)) { import dyaml; try { Node pod_manifest_yaml; try { pod_manifest_yaml = Loader.fromFile(sisudoc_txt_).load(); } catch (ErrnoException ex) { } catch (FileException ex) { writeln("ERROR failed to read config file"); } catch (Throwable) { writeln("ERROR failed to read config file content, not parsed as yaml"); } if ("doc" in pod_manifest_yaml) { if (pod_manifest_yaml["doc"].type.mapping && pod_manifest_yaml["doc"].tag.match(rgx_y.yaml_tag_is_map) ) { if ("path" in pod_manifest_yaml["doc"]) { if (pod_manifest_yaml["doc"]["path"].tag.match(rgx_y.yaml_tag_is_seq)) { foreach (string _path; pod_manifest_yaml["doc"]["path"]) { markup_contents_location_ ~= _path ~ "\n"; pod_manifest_root_content_paths_to_markup_location_raw_ ~= _path ~ "\n"; } } else if ( pod_manifest_yaml["doc"]["path"].type.string && pod_manifest_yaml["doc"]["path"].tag.match(rgx_y.yaml_tag_is_str) ) { markup_contents_location_ = pod_manifest_yaml["doc"]["path"].get!string; pod_manifest_root_content_paths_to_markup_location_raw_ = pod_manifest_yaml["doc"]["path"].get!string; } } if ("filename" in pod_manifest_yaml["doc"]) { if (pod_manifest_yaml["doc"]["filename"].tag.match(rgx_y.yaml_tag_is_seq)) { foreach (string _filename; pod_manifest_yaml["doc"]["filename"]) { if ("language" in pod_manifest_yaml["doc"]) { if (pod_manifest_yaml["doc"]["language"].tag.match(rgx_y.yaml_tag_is_seq)) { foreach (string _lang; pod_manifest_yaml["doc"]["language"]) { markup_contents_location_ ~= "media/text/" ~ _lang ~ "/" ~ _filename ~ "\n"; } } else if (pod_manifest_yaml["doc"]["language"].tag.match(rgx_y.yaml_tag_is_str) ) { markup_contents_location_ = "media/text/" ~ pod_manifest_yaml["doc"]["language"].get!string ~ "/" ~ _filename ~ "\n"; } else { string _lang_default = "en"; markup_contents_location_ ~= "media/text/" ~ _lang_default ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } } else { string _lang_default = "en"; markup_contents_location_ ~= "media/text/" ~ _lang_default ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } } } else if ( pod_manifest_yaml["doc"]["filename"].type.string && pod_manifest_yaml["doc"]["filename"].tag.match(rgx_y.yaml_tag_is_str) ) { if ("language" in pod_manifest_yaml["doc"]) { if (pod_manifest_yaml["doc"]["language"].tag.match(rgx_y.yaml_tag_is_seq)) { foreach (string _lang; pod_manifest_yaml["doc"]["language"]) { markup_contents_location_ ~= "media/text/" ~ _lang ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } } else if (pod_manifest_yaml["doc"]["language"].tag.match(rgx_y.yaml_tag_is_str)) { markup_contents_location_ = "media/text/" ~ pod_manifest_yaml["doc"]["language"].get!string ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } else { string _lang_default = "en"; markup_contents_location_ ~= "media/text/" ~ _lang_default ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } } else { string _lang_default = "en"; markup_contents_location_ ~= "media/text/" ~ _lang_default ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; } } } } } } catch (ErrnoException ex) { } } } catch (ErrnoException ex) { } catch (FileException ex) { // Handle errors } } else { writeln("manifest not found: ", sisudoc_txt_); } auto markup_contents_locations_arr = (cast(char[]) markup_contents_location_).split; auto tmp_dir_ = (sisudoc_txt_).dirName.array; foreach (markup_contents_location; markup_contents_locations_arr) { assert(markup_contents_location.match(rgx_files.src_pth_sst_or_ssm), "not a recognised file: «" ~ markup_contents_location ~ "»" ); auto markup_contents_location_pth_ = (markup_contents_location).to!string; Regex!(char) lang_rgx_ = regex(r"/(" ~ _opt_action.languages_set.join("|") ~ ")/"); if (_opt_action.languages_set[0] == "all" || (markup_contents_location_pth_).match(lang_rgx_) ) { auto _fns = (((tmp_dir_).chainPath(markup_contents_location_pth_)).array).to!string; _manifested = PathMatters!()(_opt_action, _env, arg, _fns, markup_contents_locations_arr); _manifests ~= _manifested; } } } else if (arg.match(rgx_files.src_pth_sst_or_ssm)) { /+ markup txt files +/ if (exists(arg)==0) { writeln("ERROR >> Processing Skipped! File not found: ", arg); } else { _manifested = PathMatters!()(_opt_action, _env, arg, arg); _manifests ~= _manifested; } } else if (arg.match(rgx_files.src_pth_zip)) { // fns_src ~= arg; // gather input markup source file names for processing } else { // anything remaining, unused arg_unrecognized ~= " " ~ arg; } } } #+END_SRC *** config files load & read #+NAME: spine_conf_files_in_yaml #+BEGIN_SRC d { /+ document config/make file +/ auto _config_document_struct = readConfigDoc!()(_manifest, _env); import sisudoc.meta.conf_make_meta_yaml; _make_and_meta_struct = _config_document_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifest, _opt_action, _cfg); } #+END_SRC ** _processing: (loop each file)_ [+2] :loop:files: *** scope (loop) :scope: #+NAME: spine_each_file_do_scope #+BEGIN_SRC d scope(success) { if (_opt_action.vox_gt0) { writeln("-- ~ document complete, ok ~ ------------------------------------"); } } scope(failure) { debug(checkdoc) { stderr.writefln( "~ document run failure ~ (%s v%s)\n\t%s\n%s", __VENDOR__, __VERSION__, manifest.src.filename, "------------------------------------------------------------------", ); } } enforce( manifest.src.filename.match(rgx_files.src_pth_types), "not a sisu markup filename: «" ~ manifest.src.filename ~ "»" ); #+END_SRC *** 1. _document abstraction_ [#A] - return tuple of: - doc_abstraction (the document) - doc_matters #+NAME: spine_each_file_do_abstraction #+BEGIN_SRC d if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("--->\nstepX commence → (document abstraction) [", manifest.src.filename, "]"); } auto t = spineAbstraction!()(_env, program_info, _opt_action, _cfg, manifest, _make_and_meta_struct); static assert(t.length==2); auto doc_abstraction = t[dAM.abstraction]; auto doc_matters = t[dAM.matters]; if ((doc_matters.opt.action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("- stepX complete for [", manifest.src.filename, "]"); } #+END_SRC *** 2. _output processing_ (post abstraction processing) **** 0. abstraction ***** show document summary - abstraction _print summary_ (to screen) :abstraction:summary: #+NAME: spine_each_file_do_show #+HEADER: :noweb yes #+BEGIN_SRC d <> <> <> <> <> <> #+END_SRC --show-summary (abstraction summary) #+NAME: spine_each_file_do_debugs_checkdoc_0 #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt.action.show_summary) { import sisudoc.meta.metadoc_show_summary; spineMetaDocSummary!()(doc_abstraction, doc_matters); } #+END_SRC ***** show metadata (doc headers metadata) --show-metadata #+NAME: spine_each_file_do_debugs_checkdoc_1 #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt.action.show_metadata) { import sisudoc.meta.metadoc_show_metadata; spineShowMetaData!()(doc_matters); } #+END_SRC ***** show make (make files & doc headers) --show-make #+NAME: spine_each_file_do_debugs_checkdoc_2 #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt.action.show_make) { import sisudoc.meta.metadoc_show_make; spineShowMake!()(doc_matters); } #+END_SRC ***** show config (config files & doc headers) --show-config #+NAME: spine_each_file_do_debugs_checkdoc_3 #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt.action.show_config) { import sisudoc.meta.metadoc_show_config; spineShowConfig!()(doc_matters); } #+END_SRC ***** abstraction curate :abstraction:curate: - abstraction curate #+NAME: spine_each_file_do_debugs_checkdoc_4 #+BEGIN_SRC d if (doc_matters.opt.action.curate) { auto _hvst = spineMetaDocCurate!()(doc_matters, hvst); if ( _hvst.title.length > 0 && _hvst.author_surname_fn.length > 0 ) { hvst.curates ~= _hvst; } else { if ((doc_matters.opt.action.debug_do) || (_opt_action.debug_do_curate) || (doc_matters.opt.action.vox_gt2) ) { writeln("WARNING curate: document header yaml does not contain information related to: title or author: ", _hvst.path_html_segtoc); } } } #+END_SRC **** 1. _debug_ (document parts, checkdoc) :debug:checkdoc: - [[./meta_output_debugs.org][meta_output_debugs]] #+NAME: spine_each_file_do_debugs_checkdoc_5 #+BEGIN_SRC d /+ ↓ debugs +/ if (doc_matters.opt.action.debug_do) { spineDebugs!()(doc_abstraction, doc_matters); } #+END_SRC **** 2. _process outputs_ :outputs: - [[./output_hub.org][output_hub]] #+NAME: spine_initialize_output #+BEGIN_SRC d /+ ↓ output hub +/ if (!(_opt_action.skip_output)) { outputHubInitialize!()(_opt_action, program_info); } #+END_SRC #+NAME: spine_each_file_do_selected_output #+BEGIN_SRC d /+ ↓ output hub +/ if (!(doc_matters.opt.action.skip_output)) { if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("step5 commence → (process outputs) [", manifest.src.filename, "]"); } doc_abstraction.outputHub!()(doc_matters); if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("- step5 complete for [", manifest.src.filename, "]"); } } #+END_SRC *** scope (on loop exit) :scope:exit: #+NAME: spine_each_file_do_scope_exit #+BEGIN_SRC d scope(exit) { if (_opt_action.vox_gt0) { writefln( "processed file: %s [%s]", manifest.src.filename, manifest.src.language ); } destroy(manifest); } #+END_SRC ** +no valid filename provided+ #+NAME: spine_no_filename_provided #+BEGIN_SRC d /+ no recognized filename provided +/ writeln("no recognized filename"); break; // terminate, stop #+END_SRC * pre-processing ** Output _document abstraction functions_ :module:spine:abstraction: *** 0 module template - abstraction template #+HEADER: :tangle "../src/sisudoc/meta/metadoc.d" #+BEGIN_SRC d <> module sisudoc.meta.metadoc; @safe: template spineAbstraction() { <> <> enum makeMeta { make, meta } enum docAbst { doc_abstract_obj, doc_has } @system auto spineAbstraction(E,P,O,Cfg,M)( E _env, P program_info, O _opt_action, Cfg _cfg, M _manifest, ConfComposite _make_and_meta_struct ){ <> <> <> <> <> struct ST_DocumentMatters { <> <> <> <> <> } auto doc_matters = ST_DocumentMatters(); <> auto t = tuple(doc_abstraction, doc_matters); return t; } } #+END_SRC ** Output & Curate pre-processing *** 1. raw file content split, doc: _header_, _content_ +(lists: subdocs? images?) - [[./source_read_files.org][source_read_files]] return tuple: header; body; insert file list; image list - read in the _marked up source document_ and - split the document into: - document header - document body - from markup source - if master document from sub documents content - if a master document - make a list of insert files - if build source pod requested - scan for list of images (action avoided if not needed at this stage) - _return a tuple of_: - header - body - insert file list - image list (if build source pod requested) if build source pod requested all information needed to build it available at this point - manifest related information _manifest - insert file list _header_body_insertfilelist_imagelist.insert_file_list] - image list _header_body_insertfilelist_imagelist.image_list] #+NAME: spine_each_file_do_read_and_split_dr_markup_file_content_into_header_and_body #+BEGIN_SRC d /+ ↓ read file (filename with path) +/ /+ ↓ file tuple of header and content +/ if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("step1 commence → (get document header & body & insert file list & if needed image list) [", _manifest.src.filename, "]"); } auto _header_body_insertfilelist_imagelist = spineRawMarkupContent!()(_opt_action, _manifest.src.path_and_fn); auto doc_digests = _header_body_insertfilelist_imagelist.doc_digest; // CHECK, REVIEW, discard likely, other route taken if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("- step1 complete for [", _manifest.src.filename, "]"); } debug(header_and_body) { writeln(header); writeln(_header_body_insertfilelist_imagelist.length); // writeln(_header_body_insertfilelist_imagelist.length.body_content[0]); } #+END_SRC *** 2. _document metadata_ & _make instructions_ (struct from yaml) - [[./meta_conf_make_meta.org][meta_conf_make_meta]] return tuple: document metadata; make instructions - read _document header_, split into: - metadata - make instructions - read config files - consolidate make instructions - _return tuple of_: - document metadata - make instructions (from configuration files & document header make instructions) #+NAME: spine_each_file_do_split_dr_markup_file_header_into_make_and_meta_structs #+BEGIN_SRC d /+ ↓ split header into make and meta +/ if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("step2 commence → (read document header (yaml) return struct) [", _manifest.src.filename, "]"); } import sisudoc.meta.conf_make_meta_yaml; _make_and_meta_struct = docHeaderMakeAndMetaTupYamlExtractAndConvertToStruct!()( _header_body_insertfilelist_imagelist.header_raw, _make_and_meta_struct, _manifest, _opt_action, _cfg, ); if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("- step2 complete for [", _manifest.src.filename, "]"); } #+END_SRC *** 3. _document abstraction, tuple_ (output-pre-processing) [#A] - [[./meta_abstraction.org][meta_abstraction]] return tuple: document abstraction; abstraction keys; segnames; image list - prepare the document abstraction used in downstream processing - _return tuple of_: - document abstraction (_the_document_ or doc_abstraction) - document abstraction keys - (head, toc, body, endnotes, glossary, bibliography, bookindex, blurb, tail) - (transfer to _doc_matters_) - segnames for html epub (transfer to _doc_matters_) - image list (transfer to _doc_matters_) #+NAME: spine_each_file_do_document_abstraction #+BEGIN_SRC d /+ ↓ document abstraction: process document, return abstraction as tuple +/ if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("step3 commence → (document abstraction (da); da keys; segnames; doc_matters) [", _manifest.src.filename, "]"); } auto da = docAbstraction!()( _header_body_insertfilelist_imagelist.sourcefile_body_content, _make_and_meta_struct, _opt_action, _manifest, true, ); auto doc_abstraction = da.document_the; auto _doc_has_struct = da.doc_has; if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("- step3 complete for [", _manifest.src.filename, "]"); } #+END_SRC *** 4. _document matters_ (doc info gathered, various sources) [#A] - gather doc matters - prepare document_matters, miscellany about processing and the document of use in downstream processing **** verbose message #+NAME: spine_each_file_do_document_matters_msg_step4_start #+BEGIN_SRC d if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("step4 commence → (doc_matters) [", _manifest.src.filename, "]"); } #+END_SRC **** DocumentMatters struct { **** generator related #+NAME: spine_each_file_do_document_matters_1_detail_program_time #+BEGIN_SRC d auto generator_program() { struct Prog_ { string project_name() { return "spine"; } string name() { return program_info.name; } string ver() { return program_info.ver; } @trusted string name_and_version() { return program_info.name_and_version; } @trusted string name_version_and_compiler() { return program_info.name_version_and_compiler; } string url_home() { return "https://sisudoc.org"; } string url_git() { return "https://git.sisudoc.org/projects/"; } auto compiler() { return program_info.compiler; } auto time_output_generated() { return program_info.time_output_generated; } } return Prog_(); } auto generated_time() { auto _st = Clock.currTime(UTC()); auto _time = _st.year.to!string ~ "-" ~ _st.month.to!int.to!string // prefer as month number ~ "-" ~ _st.day.to!string ~ " [" ~ _st.isoWeek.to!string ~ "/" ~ _st.dayOfWeek.to!int.to!string ~ "]" ~ " " ~ _st.hour.to!string ~ ":" ~ _st.minute.to!string ~ ":" ~ _st.second.to!string; return _time; } #+END_SRC ***** config make & meta #+NAME: spine_each_file_do_document_matters_2_meta #+BEGIN_SRC d ConfComposite conf_make_meta() { return _make_and_meta_struct; } auto doc_digest() { return doc_digests; } auto has() { return _doc_has_struct; } #+END_SRC ***** env related #+NAME: spine_each_file_do_document_matters_3_env #+BEGIN_SRC d auto env() { struct Env_ { auto pwd() { return _manifest.env.pwd; } auto home() { return _manifest.env.home; } } return Env_(); } #+END_SRC ***** opt #+NAME: spine_each_file_do_document_matters_4_opt #+BEGIN_SRC d auto opt() { struct Opt_ { auto action() { /+ getopt options, commandline instructions, raw - processing instructions --epub --html etc. - command line config instructions --output +/ return _opt_action; } } return Opt_(); } #+END_SRC ***** output related #+NAME: spine_each_file_do_document_matters_5_doc #+BEGIN_SRC d auto src() { return _manifest.src; } auto src_path_info() { return spinePathsSRC!()(_manifest.env.pwd, _manifest.src.file_with_absolute_path); // would like (to have and use) relative path } auto pod() { return _manifest.pod; } auto sqlite() { struct SQLite_ { string filename() { string _fn = ""; string _pth = ""; if (_opt_action.sqliteDB_filename.length > 0) { _fn = _opt_action.sqliteDB_filename; } else if (_make_and_meta_struct.conf.w_srv_db_sqlite_filename.length > 0) { _fn = _make_and_meta_struct.conf.w_srv_db_sqlite_filename; } return _fn; } string path() { string _pth = ""; if (_opt_action.sqliteDB_path.length > 0) { _pth = _opt_action.sqliteDB_path; } else if (_make_and_meta_struct.conf.w_srv_db_sqlite_path.length > 0) { _pth = _make_and_meta_struct.conf.w_srv_db_sqlite_path; } return _pth; } string cgi_filename() { string _fn = ""; if (_opt_action.cgi_sqlite_search_filename.length > 0) { _fn = _opt_action.cgi_sqlite_search_filename; } else if (_make_and_meta_struct.conf.w_srv_cgi_search_script.length > 0) { _fn = _make_and_meta_struct.conf.w_srv_cgi_search_script; } return _fn; } string cgi_filename_d() { string _fn = ""; if (_opt_action.cgi_sqlite_search_filename_d.length > 0) { _fn = _opt_action.cgi_sqlite_search_filename_d; } else if (_make_and_meta_struct.conf.w_srv_cgi_search_script_raw_fn_d.length > 0) { _fn = _make_and_meta_struct.conf.w_srv_cgi_search_script_raw_fn_d; } return _fn; } } return SQLite_(); } auto output_path() { return _make_and_meta_struct.conf.output_path; } auto srcs() { struct SRC_ { auto file_insert_list() { return _header_body_insertfilelist_imagelist.insert_file_list; } auto image_list() { return _doc_has_struct.imagelist; } } return SRC_(); } #+END_SRC **** } close **** step complete message #+NAME: spine_each_file_do_document_matters_msg_step4_end #+BEGIN_SRC d if ((_opt_action.debug_do) || (_opt_action.debug_do_stages) ) { writeln("- step4 complete for [", _manifest.src.filename, "]"); } #+END_SRC * document header including copyright & license #+NAME: doc_header_including_copyright_and_license #+HEADER: :noweb yes #+BEGIN_SRC emacs-lisp <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:spine_doc_header_including_copyright_and_license()>> #+END_SRC * __END__ dev notes ** the document notes *** document sections (table) |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | section | part | opt. | | objects | ocn | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | front matter | head | * | | | no | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | toc | toc | | | generated from headings | no | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | body | body | * | | default section | yes | | | | | | | | - headings | | | | | | | | | - paras | | | | | | | | | - code | | | | | | | | | - poem | | | | | | | | | - group | | | | | | | | | - block | | | | | | | | | - quote | | | | | | | | | - table | | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | back matter | endnote | | | generated from inline note markup | no (each endnote belongs to | | | | | | | | | a (body) object) | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | | glossary | | | identified section, limited markup | possibly, to make searchable | | | | | | | | - heading | hidden | | | | | | | | - paras | | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | | bibliography | | | generated from inline special markup | possibly, to make searchable | | | | | | | | appended to paragraphs contained in body section | hidden | | | | | | | | - heading | | | | | | | | | - paras | | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | | book index | | | generated from inline special markup | possibly, special numbering or | | | | | | | | - heading | could use term as anchor? | | | | | | | | - paras | to make searchable | | | | | | | | | hidden | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| | | blurb | | | identified section, limited markup | no (unless non-substantive | | | | | | | | - heading | given special numbering) | | | | | | | | - paras | | | | |--------------+--------------+------+---+--------------------------------------------------+--------------------------------+---+---| *** document objects (table) - check, keep up to date |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | doc object | doc object | attributes | inline | appended | structure | delimiters | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | is_a | is_of_type | | | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | heading | para | - level | - font face | - object number off | - level | - two newlines | | | | - object number | - endnotes | - book index meta | (document structure) | | | | | - object number off | | | | | | | | - dummy (toc & seg) | | | | | | | | - tags (internal links) | | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | toc | para | - level | - font face | | | - auto generated from headings | | | | (auto-indent) | - links (auto) | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | para | para | - bullet | - font face | - object number off | | - two newlines | | | | - indent | - links/urls * | - book index meta | | | | | | - object number | - images* | | | | | | | - object number off | - endnotes | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | bookindex | para | - auto indent | - font face | | | - two newlines | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | blurb | para | - bullet | - font face | - object number off | | - two newlines | | | | - indent | - links/urls * | - book index meta | | | | | | - object number | - images* | | | | | | | - object number off | - endnotes | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | group | block | - object number | - font face | - book index meta | - para break | - block tags | | | | - object number off | - links/urls * | | | (group) | | | | | - images* | | | | | | | | - endnotes | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | block | block | - object number | - font face | - book index meta | - new line | - block tags | | | | - object number off | - links/urls * | | | (block) | | | | | - images* | | | | | | | | - endnotes | | | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | quote | block | - object number | - font face | - book index meta | | - block tags | | | | | - endnotes | | | (quote) | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | poem (see verse) | block | | | - book index meta | | - block tags | | | | | | | | (poem) | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | verse (of poem) | | - object number | - font face | | - new line | - (see poem delimiter) | | | | | - endnotes | | - preceeding spaces | | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | code | block | - syntax | | | - new line | - block tags | | | | - numbered | | | - preceeding spaces | (code) | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | table | block | - object number | | | | - block tags (table) | | | | | | | | (table) | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| | endnote | | | - font face | | | (generated from | | | | | | | | inline markup tags) | | | | | | | | - two newlines | |------------------+------------+-------------------------+----------------+---------------------+----------------------+--------------------------------| - consider special treatment for links/urls (& for images?) take them out of document munge (for various outputs), by storing in own array (within each object struct), and providing info on where in array to extract them from, debating whether necessary or even worthwhile as is extra work **** check |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | | | identified by | object notes | attributes | inline | embedded | special | | | | | | | | appended | characters | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | para | heading | level markers | | | - italics | - endnotes | | | | | at start of line | | | | - bibliography | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | | paragraph | delimited by two new lines | default object | - indent | - bold | - endnotes | | | | | | [discard leading & | - bullet | - italics | - bibliography | | | | | | newline whitespace] | | - underscore | | | | | | | | | - strikethrough | | | | | | | | | - superscript | | | | | | | | | - subscript | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | block | | open and close tags | | | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | TODO | quote | | | - language? | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | TODO | group | | - inline markup applied | - language? | as paragraph | - endnotes | | | | | | - [discard leading & | | | - bibliography | | | | | | newline whitespace] | | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | TODO | block | | - inline markup applied | | as paragraph | - endnotes | | | | | | - whitespace indentation | | | - bibliography | | | | | | & newlines | | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | | poem / verse | open and close tags | verse is the object | | | - endnotes | | | | | | - inline markup applied | | | - bibliography | | | | | (for poem) | - whitespace indentation | | | | | | | | | & newlines | | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | | code | | - contents untouched | - syntax | | | | | | | | - whitespace indentation | - numbered | | | | | | | | & newlines | | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| | | table | | | - column width | | | | | | | | | - heading row | | | | |-------+--------------+----------------------------+--------------------------+----------------+-----------------+----------------+------------| *** on abstraction - abstract for downstream processing - identify document structure and objects - identify document structure (headings/levels/sections) - identify objects (headings, paragraphs, tables, code blocks, verse ...) - set document, generate common abstraction for downstream parsing - set different _document sections_: - _head_, toc, _body_, endnotes, glossary, bibliography, book index, blurb - _object numbers_, heading/ chapter numbering etc, endnote numbers - _regular ocn_ - body objects - glossary objects - bibliography objects - _special ocn_ - non substantive text (provide special numbers) - blurb objects - book index - special (_exceptions_) - endnotes - unify object representations - multiple markups for same object type given single representation - extract object attributes - unify inline markup on objects - inline markup made easier to identify - simplify downstream parsing *** ocn |-------------+-----------------------+-----------------------+----------------+------| | objects | section / part | ocn described | how used | type | |-------------+-----------------------+-----------------------+----------------+------| | regular ocn | | | | | |-------------+-----------------------+-----------------------+----------------+------| | | body objects | seq. digit | anchor | ocn | | | | [0-9]+ | visible | | |-------------+-----------------------+-----------------------+----------------+------| | | glossary objects | seq. digit | anchor | ocn | | | | [0-9]+ | not-visible | | | | | | (for search) | | |-------------+-----------------------+-----------------------+----------------+------| | | bibliography objects | seq. digit | anchor | ocn | | | | [0-9]+ | not-visible | | | | | | (for search) | | |-------------+-----------------------+-----------------------+----------------+------| | special ocn | | | | | |-------------+-----------------------+-----------------------+----------------+------| | | non-substantive text | x char + seq. digit | anchor | non | | | (within body & blurb) | x[0-9]+ | not-visible | | | | | | (for search) | | |-------------+-----------------------+-----------------------+----------------+------| | | book index | i char + seq. digit | anchor | idx | | | | i[0-9]+ | not-visible | | | | | | (for search) | | |-------------+-----------------------+-----------------------+----------------+------| | without ocn | | | | | |-------------+-----------------------+-----------------------+----------------+------| | | endnotes | ocn of parent object | no ocn | fn | | | | + footnote seq. digit | anchor visible | | |-------------+-----------------------+-----------------------+----------------+------| ** spine glossary / terms |-------+-------------------------------------| | spine | sisu document parser | |-------+-------------------------------------| | dmso | document markup, structure, objects | |-------+-------------------------------------| | meta | meta document, document abstraction | | mda | meta, meta document abstraction | | adr | abstract document representation | | dar | document abstract representation | | (da) | (document abstraction) | | | (code representation of document) | |-------+-------------------------------------| | ao | abstract objects | | | (code representation of objects) | |-------+-------------------------------------| consider |-------+----------------------------------------------| | dao | document abstraction, objects | |-------+----------------------------------------------| | daso | document abstraction, structure, objects | |-------+----------------------------------------------| | drso | document representation, structure, objects | |-------+----------------------------------------------| | daows | document abstraction, objects with structure | |-------+----------------------------------------------| ** make config - _composite make_ work on composite make a unification of make instructions for each document run extract instructions from all config files, unify the make instructions and provide the result as a single set of make instructions for each document parsed - 1. general, document_make config file (to be applied to all documents unless overridden by document or command line instruction) - 2. local, site specific (site local instructions such as the site's url, cgi location etc.) - 3. each document header, make (the document header contains metadata and may include make instructions for that document) - make - meta - 4. command line instruction, make (some make instructions may be passed through the command line) *** instruction sources |----+---------------------------------+----------------------------------------+---------------------+---| | | make instruction source | | varies (applies to) | | |----+---------------------------------+----------------------------------------+---------------------+---| | 0. | unify the following as a single | take into account all the instructions | | | | | set of make instructions | provided below, provide interface | | | |----+---------------------------------+----------------------------------------+---------------------+---| | 1. | document_make file | to be applied to all documents | per directory | | | | "config_document" | (unless subsequently overridden) | (all docs within) | | |----+---------------------------------+----------------------------------------+---------------------+---| | 2. | config file | local site specific | per directory | | | | "config_local_site" | | (all docs within) | | |----+---------------------------------+----------------------------------------+---------------------+---| | 3. | document header make | make instructions contained | per document | | | | | in document header | (single doc) | | |----+---------------------------------+----------------------------------------+---------------------+---| | 4. | command line instruction | make instruction passed | each command | | | | | | (all docs within) | | |----+---------------------------------+----------------------------------------+---------------------+---| *** config & metadata (from instruction sources) |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | 1. document make file | 2. config file | 3. document header | 4. command line instruction | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | comment, fixed: | per dir (pod) | per dir | per document (pod) | per command instruction | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | config_document | config_local_site | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | local site specific | | * | | *? | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | webserv | | | | | | - url_root | | | | | | - path | | | | | | - images | | | | | | - cgi | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | webserv_cgi | | | | | | - host | | | | | | - base_path | | | | | | - port | | | | | | - user | | | | | | - file_links | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | processing | | | | | | - path | | | | | | - dir | | | | | | - concord_max | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | flag (configure) | | (call) | | | | - act0 | | act0 | | | | - act1 | | act1 | | | | - act2 | | act2 | | | | - act3 | | act3 | | | | - act4 | | act4 | | | | - act5 | | act5 | | | | - act6 | | act6 | | | | - act7 | | act7 | | | | - act8 | | act8 | | | | - act9 | | act9 | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | default | | | | | | - papersize | | | | | | - text_wrap | | | | | | - emphasis | | | | | | - language | | | | | | - digest | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | permission | | | | | | - share_source | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | program_select | | | | | | - editor | | | | | | - epub_viewer | | | | | | - html_viewer | | | | | | - odf_viewer | | | | | | - pdf_viewer | | | | | | - xml_viewer | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | search | | | | | | - flag | | | | | | - action | | | | | | - db | | | | | | - title | | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | make instruction | ** | omit or override share? | ** | *? | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | make | make | make | | | | - bold | - bold | - bold | | | | - breaks | - breaks | - breaks | | | | - cover_image | - cover_image | - cover_image | | | | - css | - css | - css | | | | - emphasis | - emphasis | - emphasis | | | | - footer | - footer | - footer | | | | - headings | - headings | - headings | | | | - home_button_image | - home_button_image | - home_button_image | | | | - home_button_text | - home_button_text | - home_button_text | | | | - italics | - italics | - italics | | | | - num_top | - num_top | - num_top | | | | - auto_num_depth | - auto_num_depth | - auto_num_depth | | | | - substitute | - substitute | - substitute | | | | - texpdf_font | - texpdf_font | - texpdf_font | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | actions | | | | * | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | | assertions | | | | | | concordance | | | | | | debug | | | | | | digest | | | | | | docbook | | | | | | epub | | | | | | html | | | | | | html-seg | | | | | | html-scroll | | | | | | manifest | | | | | | ocn | | | | | | odt | | | | | | pdf | | | | | | postgresql | | | | | | qrcode | | | | | | pod | | | | | | source | | | | | | sqlite | | | | | | sqlite-db-create | | | | | | sqlite-db-drop | | | | | | text | | | | | | verbose | | | | | | xhtml | | | | | | xml-dom | | | | | | xml-sax | | | | | | section_toc | | | | | | section_body | | | | | | section_endnotes | | | | | | section_glossary | | | | | | section_biblio | | | | | | section_bookindex | | | | | | section_blurb | | | | | | backmatter | | | | | | skip-output | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | metadata | | | * | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | classify | | | | | | - dewey | | | | | | - keywords | | | | | | - loc | | | | | | - subject | | | | | | - topic_register | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | creator | | | | | | - author | | | | | | - author_email | | | | | | - illustrator | | | | | | - translator | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | date | | | | | | - added_to_site | | | | | | - available | | | | | | - created | | | | | | - issued | | | | | | - modified | | | | | | - published | | | | | | - valid | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | identifier | | | | | | - isbn | | | | | | - oclc | | | | | | - pg | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | links | | | | | | - link | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | notes | | | | | | - abstract | | | | | | - description | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | original | | | | | | - language | | | | | | - source | | | | | | - title | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | publisher | | | | | | - name | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | rights | | | | | | - copyright | | | | | | - cover | | | | | | - illustrations | | | | | | - license | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------| | | | | title | | | | | | - edition | | | | | | - full | | | | | | - language | | | | | | - main | | | | | | - note | | | | | | - sub | | | | | | - subtitle | | |---------------------+-----------------------+-------------------------+---------------------+-----------------------------|