/+
- Name: SisuDoc Spine
- Description: part of SiSU spine (Doc Reform):
for document structuring, processing, publishing and search
- Author: Ralph Amissah
[ralph.amissah@gmail.com]
- Copyright: (C) 2015 (continuously updated, current 2026) Ralph Amissah,
All Rights Reserved.
- License: AGPL 3 or later, for everything in this repository unless a file or
a directory specifies otherwise.
This program is free software: you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License (AGPLv3) as
published by the Free Software Foundation (FSF), either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see [https://www.gnu.org/licenses/].
If you have Internet connection, the latest version of the AGPL should be
available at: [https://www.gnu.org/licenses/agpl-3.0.html]
- Spine (by Doc Reform, related to SiSU) uses standard:
- docReform markup syntax
- standard SiSU markup syntax with modified headers and minor modifications
- docReform object numbering
- standard SiSU object citation numbering & system
- Homepages:
[https://www.sisudoc.org]
[https://www.doc-reform.org]
- Git
[https://git.sisudoc.org/]
+/
module sisudoc.outputs.io_out.markdown;
@safe:
/+ ↓ the markdown output: the document as CommonMark.
.
An alternative to --text, and the better one where a reader will render it:
the text file has the object numbers, the markdown has the object numbers
*and the graph between them* - every citation, every note, every index
entry and every line of the table of contents is a working link, and the
headings are headings.
.
The dialect is **CommonMark**, plus pipe tables, which every renderer worth
using understands. Raw inline html is used in exactly two places, both for
the object number, because markdown has no anchor syntax and nothing else
will do:
.
The text of the paragraph.[12](#12)
.
The anchor is at the head of the object so that a link to it lands on the
object and not past it; the number is at the end, as a link to itself, so
that a reader can take the citation out of the rendered page. That is what
the html output does with an object number.
.
**Markdown's own footnote syntax is not used.** It is an extension rather
than CommonMark, and a renderer numbers footnotes itself, in the order it
meets them. The document's note numbers are part of its citation and are
not a renderer's to choose. So a note is written where it is referred to,
with the document's own number and a link back to the object - the same
decision the typst output makes.
.
A port of tools/sisudoc-ocda-writers/dlang/src/write/markdown.d, which is
held over the 36 document reference collection against the html writer's
objects word for word, and against the Gleam writer byte for byte.
+/
template outputMarkdown() {
import sisudoc.outputs.io_out;
import sisudoc.outputs.io_out.rgx;
import sisudoc.outputs.io_out.paths_output;
import std.algorithm : canFind, max;
import std.array : appender, array, join, replace;
import std.array : asplit = split;
import std.conv : to;
import std.exception : ErrnoException;
import std.file;
import std.regex : matchAll, matchFirst, replaceAll, split;
import std.stdio;
import std.string : strip;
mixin spineRgxOut;
static auto rgx = RgxO();
enum newline = "\n";
enum newlines = "\n\n";
/+ ↓ the markers that protect the markdown this writer emits from the escaping
that follows.
.
The escaping has to come *last*, and this is why: the abstraction's own
markers are built from the characters markdown reserves - "⑆*┨" for
emphasis, "⑆_┨" for an underscore face - so escaping first turns them
into "⑆\*┨" and no face is ever recognised.
+/
enum keep_open = "";
enum keep_close = "";
/+ ↓ a third marker of the same kind, standing for the directory the images are
in.
.
An image reference is written while an object is written, and *where this
file sits* is known only to spinePathsMarkdown. Naming the path here as
well would be two places agreeing about one path until one of them
changed, which is the defect the pdf links had. So the reference carries a
marker and the marker is substituted once, where the document is written,
out of the one place that knows.
+/
enum image_dir_mark = "";
string keep(string markdown_source) {
return keep_open ~ markdown_source ~ keep_close;
}
string unprotect(string txt) {
return txt.replace(keep_open, "").replace(keep_close, "");
}
/+ ↓ the characters markdown reads as syntax, escaped wherever they occur and
only outside a protected run.
.
The same blunt rule the typst output uses, and for the same reason: a
closed set is checkable and a rule about positions is not. The three html
characters become entities rather than backslash escapes, because
markdown passes raw html through and " 0) { depth--; } continue; }
if (depth > 0) { _out ~= c; continue; }
switch (c) {
case '&': _out ~= "&"; break;
case '<': _out ~= "<"; break;
case '>': _out ~= ">"; break;
case '\\': case '`': case '*': case '_': case '[': case ']':
case '#': case '|': case '~':
_out ~= '\\'; _out ~= c; break;
default: _out ~= c; break;
}
}
return _out.data;
}
/+ ↓ the document's own text, escaped for a protected run: the same rule, run
early, and then flattened so that the result is inert +/
string inert(string txt) {
return unprotect(escape(txt));
}
/+ ↓ a "- ", "+ ", "> " or "1. " at the start of a line is markdown's own: a
list item, an enumeration item, a blockquote. The writer uses the first of
them itself, and a line of the document's that begins the same way would
be read as one - which is what a verse or a group can do. Guarded rather
than escaped everywhere, because these are special only in that position.
+/
string guardLineStarts(string s) {
auto _out = appender!string;
bool at_start = true;
for (size_t i = 0; i < s.length; i++) {
if (at_start) {
size_t j = i;
while (j < s.length && s[j] == ' ') { j++; }
if (j < s.length) {
char c = s[j];
bool marker = (c == '-' || c == '+' || c == '>')
&& j + 1 < s.length && (s[j + 1] == ' ' || s[j + 1] == '\t');
size_t k = j;
while (k < s.length && s[k] >= '0' && s[k] <= '9') { k++; }
bool enumerated = k > j && k + 1 < s.length
&& (s[k] == '.' || s[k] == ')') && s[k + 1] == ' ';
if (marker || enumerated) {
out_put(_out, s[i .. (enumerated ? k : j)]);
_out ~= '\\';
i = (enumerated ? k : j) - 1;
at_start = false;
continue;
}
}
at_start = false;
}
_out ~= s[i];
if (s[i] == '\n') { at_start = true; }
}
return _out.data;
}
void out_put(T)(ref T sink, string s) { sink ~= s; }
/+ ↓ a yaml scalar. Quoted always, so that a colon or a leading marker in a
title cannot change the shape of the document's own metadata. +/
string yaml(string s) {
return "\"" ~ s.replace("\\", "\\\\").replace("\"", "\\\"")
.replace("\n", " ") ~ "\"";
}
/+ ↓ which sections, in which order: the latex sequence, less the endnotes,
because a note is written where it is referred to +/
string[] markdownSections(M)(M doc_matters) {
string[] _seq;
foreach (part; doc_matters.has.keys_seq.latex) {
if (part == "endnotes") { continue; }
_seq ~= part;
}
return _seq;
}
/+ ↓ the table of contents depth of each indent_hang the document uses.
.
spine's indent_hang is not a depth: it reserves 1 to 3 for levels above
the body, so a toc uses 1 and then jumps to 4, 5, 6. A markdown list
nests by two spaces a level and cannot take a jump, so the values the
document actually uses are ranked.
+/
size_t[int] tocDepths(D,M)(const D doc_abstraction, M doc_matters) {
bool[int] _seen;
foreach (part; markdownSections!()(doc_matters)) {
if (part != "toc") { continue; }
foreach (obj; doc_abstraction[part]) {
if (obj.metainfo.is_of_type == "comment") { continue; }
_seen[obj.attrib.indent_hang.to!int] = true;
}
}
import std.algorithm : sort;
auto _levels = _seen.keys.sort.array;
size_t[int] _depth;
foreach (i, level; _levels) { _depth[level] = i; }
return _depth;
}
/+ ↓ the document's metadata as yaml front matter, which is how a markdown file
carries metadata and what every static site generator reads +/
string markdownHead(M)(M doc_matters) {
auto _out = appender!string;
_out ~= "---" ~ newline;
void put(string key, string value) {
if (value.length > 0) { _out ~= key ~ ": " ~ yaml(value) ~ newline; }
}
put("title", doc_matters.conf_make_meta.meta.title_full);
put("author", doc_matters.conf_make_meta.meta.creator_author);
put("date", doc_matters.conf_make_meta.meta.date_published);
put("language", doc_matters.src.language);
put("copyright", doc_matters.conf_make_meta.meta.rights_copyright);
put("license", doc_matters.conf_make_meta.meta.rights_license);
_out ~= "---" ~ newlines;
return _out.data;
}
/+ ↓ an image, with its alt text where the document gave one +/
string images(string txt) {
return replaceAll!((m) {
string _rest = m["post"].to!string;
string _alt;
if (auto a = _rest.matchFirst(rgx.inline_image_alt)) {
_alt = a["alt"].to!string;
_rest = a.post.to!string;
}
return m["pre"].to!string
~ keep("")
~ _rest;
})(txt, rgx.inline_image);
}
/+ ↓ a fragment that names an object of this document +/
bool isObjectNumber(string s) {
if (s.length == 0 || s == "0") { return false; }
foreach (c; s) { if (c < '0' || c > '9') { return false; } }
return true;
}
/+ ↓ a link target is a url or a fragment and not prose, so the escaping the
text went through comes off it again +/
string unescapeTarget(string target) {
auto _out = appender!string;
for (size_t i = 0; i < target.length; i++) {
if (target[i] == 0x5c && i + 1 < target.length) { _out ~= target[++i]; continue; }
_out ~= target[i];
}
return _out.data.replace("&", "&").replace("<", "<").replace(">", ">");
}
/+ ↓ the font faces. Emphasis and strong are markdown's own; the rest have no
markdown and take the html element markdown itself would produce.
.
The spaces inside a run are dropped: the abstraction's marker takes in the
space that was beside the run, the text before it already ends in one, and
markdown's emphasis will not open on a space - "* lex *" is three words
and two asterisks, not an italic.
+/
string fontFace(string txt) {
string wrap(string open, string close, string inner) {
size_t a = 0, b = inner.length;
while (a < b && inner[a] == ' ') { a++; }
while (b > a && inner[b - 1] == ' ') { b--; }
return keep(open) ~ inner[a .. b] ~ keep(close);
}
string apply(alias pattern)(string s, string open, string close) {
return replaceAll!((m) => wrap(open, close, m.captures[1].to!string))(s, pattern);
}
txt = apply!(rgx.inline_emphasis)(txt, "**", "**");
txt = apply!(rgx.inline_bold)(txt, "**", "**");
txt = apply!(rgx.inline_italics)(txt, "*", "*");
txt = apply!(rgx.inline_underscore)(txt, "", "");
txt = apply!(rgx.inline_superscript)(txt, "", "");
txt = apply!(rgx.inline_subscript)(txt, "", "");
txt = apply!(rgx.inline_mono)(txt, "`", "`");
txt = apply!(rgx.inline_strike)(txt, "~~", "~~");
txt = apply!(rgx.inline_insert)(txt, "", "");
txt = apply!(rgx.inline_cite)(txt, "", "");
return txt;
}
/+ ↓ a link. An internal target becomes a fragment, which is the object number,
and so resolves against the anchor every object carries.
.
And *only* an object number: those are the anchors this writer defines, so
a target that names something else - the endnotes section, an anchor tag
the markup placed - would be a link to nothing. The text stands on its own
instead, which is what a reader sees anyway.
+/
string links(O)(string txt, const O obj) {
auto _stow = obj.stow.link;
txt = replaceAll!((m) {
size_t _num = m["num"].to!size_t;
string _url = (_num < _stow.length) ? _stow[_num].to!string : "";
return m["linked_text"].to!string ~ "┤" ~ _url ~ "├";
})(txt, rgx.inline_link_number_only);
return replaceAll!((m) {
string _text = m.captures[1].to!string;
string _target = m.captures[2].to!string;
if (auto im = _target.matchFirst(rgx.inline_link_seg_and_hash)) {
_target = "#" ~ im["hash"].to!string;
}
string _shown = inert(fontFace(_text));
if (_target.length == 0) { return keep(_shown); }
string _clean = unescapeTarget(_target);
if (_clean.length > 0 && _clean[0] == '#'
&& !isObjectNumber(_clean[1 .. $])) {
return keep(_shown);
}
return keep("[" ~ _shown ~ "](" ~ _clean ~ ")");
})(txt, rgx.inline_link);
}
/+ ↓ a note reference: a superscript link to the note, which sits under the
object that refers to it +/
string noteRefs(string txt) {
return replaceAll!((m) {
string _mark = m["num"].to!string;
return keep("[" ~ inert(_mark) ~ "](#note-" ~ _mark ~ ")");
})(txt, rgx.inline_notes_al_all_note);
}
/+ ↓ the line breaks the markup asked for, and the grouped indents +/
string breaks(O)(string txt, const O obj) {
if (obj.metainfo.is_a == "code") { return txt; }
if (obj.metainfo.is_a == "group" || obj.metainfo.is_a == "block") {
txt = txt
.replaceAll(rgx.grouped_para_indent_hang, "$2$2")
.replaceAll(rgx.grouped_para_bullet_indent, "$1● ")
.replaceAll(rgx.grouped_para_bullet, "● ")
.replaceAll(rgx.grouped_para_indent, "$1$1");
}
return txt
.replaceAll(rgx.nbsp_char, " ")
.replaceAll(rgx.br_line, newline)
.replaceAll(rgx.br_line_inline, newline)
.replaceAll(rgx.br_line_spaced, newlines)
.replaceAll(rgx.line_break, newline)
.replaceAll(rgx.mark_internal_site_lnk, "");
}
/+ ↓ an object's text as markdown.
.
The order is the one every writer here uses: images before links, because
an image sits inside a link; then the note references, the faces, the
anchors and the breaks. The escaping is last.
+/
string inlineText(O)(string txt, const O obj) {
txt = images(txt);
txt = links!()(txt, obj);
txt = noteRefs(txt);
txt = fontFace(txt);
txt = txt.replaceAll(rgx.inline_link_anchor, "");
txt = breaks!()(txt, obj);
return guardLineStarts(unprotect(escape(txt)));
}
/+ ↓ six levels of heading, which is all markdown has. A document may be eight
deep; the two deepest take the sixth, and the object number, the anchor and
the table of contents carry the true depth.
+/
int headingLevel(string level) {
switch (level) {
case "A": return 1;
case "B": return 2;
case "C": return 3;
case "D": return 4;
case "1": return 5;
case "2": case "3": case "4": return 6;
default: return 1;
}
}
/+ ↓ the object's anchor, so that a citation can point at it +/
string anchor(O)(const O obj) {
return (obj.metainfo.ocn == 0)
? "" : "";
}
/+ ↓ the object's number, at the end of it, as a link to itself +/
string ocnMark(O)(const O obj) {
if (obj.metainfo.ocn == 0) { return ""; }
string _n = obj.metainfo.ocn.to!string;
return "[" ~ _n ~ "](#" ~ _n ~ ")";
}
/+ ↓ the notes the object referred to, each where it was referred to, with the
document's own number and a link back to the object +/
string notes(O)(const O obj) {
auto _out = appender!string;
string _back = (obj.metainfo.ocn == 0)
? "" : " [↩](#" ~ obj.metainfo.ocn.to!string ~ ")";
foreach (m; obj.text.matchAll(rgx.inline_notes_al_all_note)) {
string _mark = m["num"].to!string;
_out ~= newlines ~ ""
~ inert(_mark) ~ ". "
~ inlineText!()(m["note"].to!string, obj) ~ _back;
}
/+ ↓ and a blank line after the last of them, or the next object begins on the
same line and markdown reads the two as one paragraph +/
return (_out.data.length > 0) ? _out.data ~ newlines : "";
}
/+ ↓ the object a table of contents entry points at, where that is an object
number +/
string internalTarget(O)(const O obj) {
foreach (m; obj.text.matchAll(rgx.any_internal_target)) {
string _frag = m["frag"].to!string;
if (isObjectNumber(_frag)) { return _frag; }
}
return "";
}
/+ ↓ one object, and then the notes it referred to.
.
The notes are gathered here rather than inside each kind, because a
reference can be anywhere - the markup manual has one in a heading - and a
reference whose note was never written is a link to nothing.
+/
string object(O,M)(const O obj, M doc_matters, size_t[int] toc_depth) {
return objectBody!()(obj, doc_matters, toc_depth) ~ notes!()(obj);
}
string objectBody(O,M)(const O obj, M doc_matters, size_t[int] toc_depth) {
switch (obj.metainfo.is_a) {
case "heading":
string _hashes;
foreach (_; 0 .. headingLevel(obj.metainfo.marked_up_level.to!string)) {
_hashes ~= "#";
}
return _hashes ~ " " ~ anchor!()(obj) ~ inlineText!()(obj.text, obj)
~ ocnMark!()(obj) ~ newlines;
case "toc":
/+ ↓ a list item at its depth, the heading as a link, and the object number
as the locator. The number and not a page: a page is a fact about one
typesetting, and the object number is the same reference in every
output of the document.
+/
string _indent;
if (auto d = obj.attrib.indent_hang.to!int in toc_depth) {
foreach (_; 0 .. *d) { _indent ~= " "; }
}
string _n = internalTarget!()(obj);
string _locator = (_n.length > 0)
? (" [" ~ _n ~ "](#" ~ _n ~ ")") : "";
return _indent ~ "- " ~ inlineText!()(obj.text, obj) ~ _locator ~ newline;
case "table":
return table!()(obj, doc_matters);
case "code":
/+ ↓ a fenced block, long enough that the code cannot close it +/
string _txt = obj.text.replaceAll(rgx.nbsp_char, " ");
string _fence = "```";
while (_txt.canFind(_fence)) { _fence ~= "`"; }
return anchor!()(obj) ~ newline ~ _fence ~ obj.metainfo.syntax.to!string
~ newline ~ _txt ~ newline ~ _fence ~ newline ~ ocnMark!()(obj) ~ newlines;
case "quote":
return "> " ~ anchor!()(obj) ~ inlineText!()(obj.text, obj)
~ ocnMark!()(obj) ~ newlines;
case "verse": case "poem": case "group": case "block":
/+ ↓ every line kept, with CommonMark's own hard line break - a backslash at
the end of a line - rather than the two trailing spaces an editor will
strip +/
auto _rows = inlineText!()(obj.text, obj).asplit(newline);
auto _out = appender!string;
_out ~= anchor!()(obj);
foreach (i, row; _rows) {
_out ~= row;
if (i + 1 < _rows.length) { _out ~= "\\" ~ newline; }
}
return _out.data ~ ocnMark!()(obj) ~ newlines;
case "bookindex":
/+ ↓ a list item, and every object it names a link +/
string _txt = obj.text.replaceAll(rgx.trailing_backslash, "");
return "- " ~ anchor!()(obj) ~ inlineText!()(_txt, obj)
~ ocnMark!()(obj) ~ newlines;
case "para": case "blurb": case "glossary": case "bibliography":
string _bullet = obj.attrib.bullet ? "- " : "";
string _indent;
foreach (_; 0 .. obj.attrib.indent_base) { _indent ~= " "; }
return _indent ~ _bullet ~ anchor!()(obj) ~ inlineText!()(obj.text, obj)
~ ocnMark!()(obj) ~ newlines;
default:
return "";
}
}
/+ ↓ a pipe table, which needs a header row: markdown has no table without one,
so a table the document did not give a header gets an empty row and keeps
all of its own.
+/
string table(O,M)(const O obj, M doc_matters) {
auto _rows = obj.text.split(rgx.table_delimiter_row);
int _cols = max(obj.table.number_of_columns.to!int, 1);
string[][] _body;
foreach (row; _rows) {
if (row.strip.length == 0) { continue; }
string[] _cells;
foreach (cell; row.split(rgx.table_delimiter_col)) {
_cells ~= inlineText!()(cell, obj).replace("|", "\\|");
}
_body ~= _cells;
}
string rule() {
auto r = appender!string;
r ~= "|";
foreach (i; 0 .. _cols) {
string _a = (i < obj.table.column_aligns.length)
? obj.table.column_aligns[i].to!string : "l";
r ~= (_a == "c") ? " :---: |" : (_a == "r") ? " ---: |" : " :--- |";
}
return r.data ~ newline;
}
string line(string[] cells) {
auto l = appender!string;
l ~= "|";
foreach (i; 0 .. _cols) {
l ~= " " ~ ((i < cells.length) ? cells[i] : "") ~ " |";
}
return l.data ~ newline;
}
auto _out = appender!string;
_out ~= anchor!()(obj) ~ newline;
if (_body.length > 0 && obj.table.heading) {
_out ~= line(_body[0]) ~ rule();
foreach (row; _body[1 .. $]) { _out ~= line(row); }
} else {
_out ~= line([]) ~ rule();
foreach (row; _body) { _out ~= line(row); }
}
return _out.data ~ ocnMark!()(obj) ~ newlines;
}
string markdownBody(D,M)(const D doc_abstraction, M doc_matters) {
auto _toc_depth = tocDepths!()(doc_abstraction, doc_matters);
auto _out = appender!string;
foreach (part; markdownSections!()(doc_matters)) {
foreach (obj; doc_abstraction[part]) {
if (obj.metainfo.is_of_type == "comment") { continue; }
if (obj.metainfo.dummy_heading) { continue; }
if (obj.text.strip.length == 0) { continue; }
_out ~= object!()(obj, doc_matters, _toc_depth);
}
}
return _out.data;
}
void outputMarkdown(D,M)(
const D doc_abstraction,
M doc_matters,
) {
auto pth_md = spinePathsMarkdown(doc_matters);
try {
if (!exists(pth_md.base_pth)) { (pth_md.base_pth).mkdirRecurse; }
} catch (ErrnoException ex) {
}
if (doc_matters.opt.action.vox_gt_1) {
writeln(" ", pth_md.markdown_file);
}
{
auto f = File(pth_md.markdown_file, "w");
f.write(markdownHead!()(doc_matters));
/+ ↓ the image directory, named once, here, out of the paths module that
decided where this file goes +/
f.write(markdownBody!()(doc_abstraction, doc_matters)
.replace(image_dir_mark, pth_md.images_rel));
}
/+ ↓ the images the document names, into the shared image directory at the
output root, because markdown names them by path and does not carry them.
.
The same set the html references, and not a copy of it: a file already
there is left alone, so this costs nothing when the html output has
written them and still works when --markdown is asked for on its own.
+/
if (doc_matters.srcs.image_list.length > 0) {
try {
if (!exists(pth_md.images)) { (pth_md.images).mkdirRecurse; }
foreach (image; doc_matters.srcs.image_list) {
string _in = doc_matters.src.image_dir_path ~ "/" ~ image;
string _out = pth_md.images ~ "/" ~ image;
if (exists(_in) && !exists(_out)) { _in.copy(_out); }
}
} catch (Exception ex) {
}
}
}
}