diff options
| author | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-21 12:00:43 -0400 |
|---|---|---|
| committer | Ralph Amissah <ralph.amissah@gmail.com> | 2026-09-22 14:54:35 -0400 |
| commit | 774302487c8595ec4383558e9d8387a6219ef2ed (patch) | |
| tree | 8a2001340c39e50f2f85b26cb5fabfb939c2b8a0 /org | |
| parent | digests: db digest taken once on file completion (diff) | |
test db: multilingual database test
one file holding several languages.
Both assumed <uid>.<lang>.ocda.db beside <uid>.<lang>.ssp, one for
one.
- test-abstraction-db.sh takes the language from the .ssp name, the
database from the stem, and scopes every query by the doc_id that
language resolves to: objects by doc_id, the four child tables
through their objects, metadata by doc_id, schema.version by doc_id
is null. It also checks what a per language loop cannot see, that the
languages a database holds are exactly the languages there are .ssp
files for.
- test-abstraction-db-roundtrip.sh asks for each language by name and
holds it against its own .ssp, and requires a database of more than
one language to refuse a round trip that names none: a reader quietly
emitting the first language would otherwise pass against the first
.ssp and say nothing about the other nine. Its language list comes
from the .ssp filenames, so it still needs no sqlite3.
databases (re-)emitting all .ssp files exactly.
(assisted by Claude-Code)
Diffstat (limited to 'org')
| -rw-r--r-- | org/tests_for_document_abstraction_shell_scripts.org | 161 |
1 files changed, 121 insertions, 40 deletions
diff --git a/org/tests_for_document_abstraction_shell_scripts.org b/org/tests_for_document_abstraction_shell_scripts.org index e321286..0f41ca6 100644 --- a/org/tests_for_document_abstraction_shell_scripts.org +++ b/org/tests_for_document_abstraction_shell_scripts.org @@ -30,7 +30,7 @@ # SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \ # ./test/run-tests.sh ./bin/spine-ldc # -# sqlite3 is needed by two of the four. If it is not on PATH the script says +# sqlite3 is needed by one of the five. If it is not on PATH the script says # so and re-runs itself inside a nix shell that has it, so this works: # SpinePOD=... ./test/run-tests.sh ./bin/spine-ldc # @@ -533,23 +533,38 @@ lev4_subtoc|object_subtoc report() { echo " MISMATCH $1: .ssp $2, db $3"; } -# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), the -# database beside it (pod/<uid>.ocda.db) +# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), one +# per language, and the database beside it (pod/<uid>.ocda.db), one per +# document holding every language of it. So the .ssp name carries the +# language and the database name does not: <uid>.<lang>.ssp beside +# <uid>.ocda.db, and inside the database the language's rows are the ones +# whose doc_id is the id documents gives that language. for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do [ -f "$ssp" ] || continue base=$(basename "$ssp" .ssp) - db="$OUT_DIR/pod/$base.ocda.db" + lang="${base##*.}" + stem="${base%.*}" + db="$OUT_DIR/pod/$stem.ocda.db" if [ ! -f "$db" ]; then echo "MISSING db for $base" FAILURES=$((FAILURES + 1)) continue fi + did=$(sqlite3 "$db" "select id from documents where lang='$lang';" 2>/dev/null || echo '') + if [ -z "$did" ]; then + echo "MISSING $lang in $(basename "$db"); it holds:" \ + "$(sqlite3 "$db" "select group_concat(lang,' ') from documents;" 2>/dev/null)" + FAILURES=$((FAILURES + 1)) + continue + fi + # the objects of this one language, for the child tables to key on + mine="object_id in (select id from objects where doc_id=$did)" DOCS=$((DOCS + 1)) doc_fail=0 # object population s_objs=$(grep -c '^\[' "$ssp" || true) - d_objs=$(sqlite3 "$db" "select count(*) from objects;" 2>/dev/null || echo ERR) + d_objs=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did;" 2>/dev/null || echo ERR) if [ "$s_objs" != "$d_objs" ]; then echo "$base"; report "objects" "$s_objs" "$d_objs"; doc_fail=1 fi @@ -557,7 +572,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do # per section for sec in head toc body endnotes glossary bibliography bookindex blurb; do s_sec=$(awk -v s="@$sec {" '$0==s{f=1;next} /^\}$/{f=0} f&&/^\[/{n++} END{print n+0}' "$ssp") - d_sec=$(sqlite3 "$db" "select count(*) from objects where section='$sec';" 2>/dev/null || echo ERR) + d_sec=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and section='$sec';" 2>/dev/null || echo ERR) if [ "$s_sec" != "$d_sec" ]; then [ $doc_fail -eq 0 ] && echo "$base" report "section $sec" "$s_sec" "$d_sec"; doc_fail=1 @@ -568,10 +583,10 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do echo "$CHECKS" | while IFS='|' read -r prop pred; do [ -z "$prop" ] && continue case "$prop" in - ,*": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;; - ,*) s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;; + *": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;; + *) s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;; esac - d_n=$(sqlite3 "$db" "select count(*) from objects where $pred;" 2>/dev/null || echo ERR) + d_n=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and ($pred);" 2>/dev/null || echo ERR) if [ "$s_n" != "$d_n" ]; then echo "$base: MISMATCH .$prop: .ssp $s_n, db $d_n" fi @@ -587,7 +602,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do echo "$LISTS" | while IFS='|' read -r prop table; do [ -z "$prop" ] && continue s_n=$(grep -c "^\.$prop: " "$ssp" || true) - d_n=$(sqlite3 "$db" "select count(*) from $table;" 2>/dev/null || echo ERR) + d_n=$(sqlite3 "$db" "select count(*) from $table where $mine;" 2>/dev/null || echo ERR) if [ "$s_n" != "$d_n" ]; then echo "$base: MISMATCH .$prop: .ssp $s_n, $table $d_n" fi @@ -599,10 +614,15 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do fi rm -f "$OUT_DIR/.lists.$$" - # every image the abstraction referred to and found is carried in the file - want=$(sqlite3 "$db" "select count(distinct name) from object_images where missing = 0;" 2>/dev/null || echo ERR) + # every image the abstraction referred to and found is carried in the file. + # the files table is shared: the languages of a document refer to the same + # images and one copy serves them all, so what is counted here is the names + # this language referred to and the stored files that answer them. + want=$(sqlite3 "$db" "select count(distinct name) from object_images + where missing = 0 and $mine;" 2>/dev/null || echo ERR) have=$(sqlite3 "$db" "select count(*) from files f where f.role='image' - and exists (select 1 from object_images i where i.name = f.name);" 2>/dev/null || echo ERR) + and exists (select 1 from object_images i where i.name = f.name + and i.missing = 0 and i.$mine);" 2>/dev/null || echo ERR) if [ "$want" != "$have" ]; then [ $doc_fail -eq 0 ] && echo "$base" report "images carried" "$want referenced" "$have stored"; doc_fail=1 @@ -610,9 +630,11 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do # the text itself, on the first and last citable object for pick in "min" "max"; do - ocn=$(sqlite3 "$db" "select $pick(ocn) from objects where ocn > 0 and section='body';" 2>/dev/null || echo "") + ocn=$(sqlite3 "$db" "select $pick(ocn) from objects + where doc_id=$did and ocn > 0 and section='body';" 2>/dev/null || echo "") [ -z "$ocn" ] && continue - sqlite3 "$db" "select text from objects where ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true + sqlite3 "$db" "select text from objects + where doc_id=$did and ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true awk -v o="[$ocn] " 'index($0,o)==1{f=1;next} f&&/^$/{exit} f&&/^\| /{print substr($0,3)}' \ "$ssp" > "$OUT_DIR/.ssptxt.$$" if ! diff -q "$OUT_DIR/.dbtxt.$$" "$OUT_DIR/.ssptxt.$$" > /dev/null 2>&1; then @@ -638,8 +660,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do } ' "$ssp" | sort > "$OUT_DIR/.sspmeta.$$" sqlite3 -separator "$(printf '\t')" "$db" \ - "select key, value from metadata where key not in - ('schema.name','schema.version','source.filename','source.ssp_digest');" \ + "select key, value from metadata where doc_id=$did and key not in + ('source.filename','source.ssp_digest');" \ 2>/dev/null | sort > "$OUT_DIR/.dbmeta.$$" if ! diff -q "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" > /dev/null 2>&1; then [ $doc_fail -eq 0 ] && echo "$base" @@ -650,7 +672,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do rm -f "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" # the source filename, and the version the two share - sf=$(sqlite3 "$db" "select value from metadata where key='source.filename';" 2>/dev/null || echo '') + sf=$(sqlite3 "$db" "select value from metadata + where doc_id=$did and key='source.filename';" 2>/dev/null || echo '') sf_ssp=$(sed -n 's/^% Source: //p' "$ssp" | head -1) if [ "$sf" != "$sf_ssp" ]; then [ $doc_fail -eq 0 ] && echo "$base" @@ -658,7 +681,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do fi sv_ssp=$(sed -n 's/^% SiSU Document Abstraction v//p' "$ssp" | head -1) # the file says what it is - sv=$(sqlite3 "$db" "select value from metadata where key='schema.version';" 2>/dev/null || echo '') + sv=$(sqlite3 "$db" "select value from metadata + where doc_id is null and key='schema.version';" 2>/dev/null || echo '') if [ -z "$sv" ]; then [ $doc_fail -eq 0 ] && echo "$base" echo " MISSING metadata schema.version"; doc_fail=1 @@ -670,10 +694,31 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do FAILURES=$((FAILURES + doc_fail)) done +# one database per document, holding every language of it: the languages the +# documents table names are exactly the languages there are .ssp files for. +# A per language loop cannot see this - it checks the languages it finds and +# would not notice one the database had gained or lost - so it is checked per +# database, once, here. +DBS=0 +for db in $(find "$OUT_DIR/pod" -maxdepth 1 -name "*.ocda.db" 2>/dev/null | sort); do + [ -f "$db" ] || continue + DBS=$((DBS + 1)) + stem=$(basename "$db" .ocda.db) + d_langs=$(sqlite3 "$db" "select lang from documents order by lang;" 2>/dev/null \ + | tr '\n' ' ' | sed 's/ $//') + s_langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \ + | sed "s|.*/$stem\.||; s|\.ssp$||" | sort | tr '\n' ' ' | sed 's/ $//') + if [ "$d_langs" != "$s_langs" ]; then + echo "$stem" + echo " MISMATCH languages: db '$d_langs', .ssp files '$s_langs'" + FAILURES=$((FAILURES + 1)) + fi +done + rm -rf "$OUT_DIR" if [ "$FAILURES" -eq 0 ]; then - echo "PASS: .ssp and .ocda.db agree for all $DOCS documents" + echo "PASS: .ssp and .ocda.db agree for all $DOCS documents in $DBS databases" exit 0 else echo "FAIL: $FAILURES document(s) disagree" @@ -691,7 +736,10 @@ fi # # Reads each <doc>.ocda.db back into the abstraction, emits it as # .ssp, and requires the result to be byte identical to the .ssp written -# from the same document. +# from the same document. A database holds every language of its document, +# so each language is asked for by name (--lang) and held against its own +# .ssp, and a database of more than one language is required to refuse a +# round trip that names none. # # What that proves: the two serialisations really do carry the same thing. # test-abstraction-db.sh counts fields and compares them; this reconstructs @@ -749,37 +797,70 @@ $SPINE_BIN --show-abstraction --ocda-db --skip-output \ --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1 COUNT=0 +DBS=0 FAILURES=0 -# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp inside it -# (pod/<doc>/media/abstraction/<uid>.ssp) +# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp files inside +# it (pod/<doc>/media/abstraction/<uid>.<lang>.ssp). +# +# A database holds every language of its document, so one database answers +# for as many .ssp files as the document has languages, and each has to be +# asked for by name: --lang says which language to emit. With no language +# named a database holding more than one refuses to choose, which is checked +# too - a round trip that quietly emitted the first language would pass +# against the first .ssp and say nothing about the other nine. for db in "$OUT_DIR"/pod/*.ocda.db; do [ -f "$db" ] || continue - base=$(basename "$db" .ocda.db) - ssp=$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1) - COUNT=$((COUNT + 1)) - if [ ! -f "$ssp" ]; then - echo "MISSING .ssp for $base" + DBS=$((DBS + 1)) + stem=$(basename "$db" .ocda.db) + # the languages come from the .ssp files rather than from the database, so + # that this script needs no sqlite3: each .ssp is a language the database + # has to be able to re-emit. A language in the database with no .ssp beside + # it is the other direction, and test-abstraction-db.sh checks that. + langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \ + | sed "s|.*/$stem\.||; s|\.ssp$||" | sort) + if [ -z "$langs" ]; then + echo "MISSING .ssp for $stem" FAILURES=$((FAILURES + 1)) continue fi - if ! $SPINE_BIN --db-round-trip="$db" > "$OUT_DIR/from-db.ssp" 2> "$OUT_DIR/from-db.err"; then - echo "ERROR: $base (spine exited non-zero)" - head -3 "$OUT_DIR/from-db.err" - FAILURES=$((FAILURES + 1)) - continue - fi - if ! cmp -s "$ssp" "$OUT_DIR/from-db.ssp"; then - echo "DIFFERS: $base ($(diff "$ssp" "$OUT_DIR/from-db.ssp" | grep -c '^[<>]') lines)" - diff --unified=1 "$ssp" "$OUT_DIR/from-db.ssp" | head -12 - echo " ..." - FAILURES=$((FAILURES + 1)) + n_langs=$(printf '%s\n' "$langs" | grep -c .) + for lang in $langs; do + base="$stem.$lang" + ssp="$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1)" + COUNT=$((COUNT + 1)) + if [ ! -f "$ssp" ]; then + echo "MISSING .ssp for $base" + FAILURES=$((FAILURES + 1)) + continue + fi + if ! $SPINE_BIN --db-round-trip="$db" --lang="$lang" \ + > "$OUT_DIR/from-db.ssp" 2> "$OUT_DIR/from-db.err"; then + echo "ERROR: $base (spine exited non-zero)" + head -3 "$OUT_DIR/from-db.err" + FAILURES=$((FAILURES + 1)) + continue + fi + if ! cmp -s "$ssp" "$OUT_DIR/from-db.ssp"; then + echo "DIFFERS: $base ($(diff "$ssp" "$OUT_DIR/from-db.ssp" | grep -c '^[<>]') lines)" + diff --unified=1 "$ssp" "$OUT_DIR/from-db.ssp" | head -12 + echo " ..." + FAILURES=$((FAILURES + 1)) + fi + done + # more than one language and none named: it must refuse rather than pick + if [ "$n_langs" -gt 1 ]; then + if $SPINE_BIN --db-round-trip="$db" > /dev/null 2> /dev/null; then + echo "$stem" + echo " ACCEPTED a round trip with no language named, holding $n_langs" + FAILURES=$((FAILURES + 1)) + fi fi done rm -rf "$OUT_DIR" if [ "$FAILURES" -eq 0 ]; then - echo "PASS: all $COUNT databases re-emit their document's .ssp exactly" + echo "PASS: $DBS databases re-emit all $COUNT of their documents' .ssp exactly" exit 0 else echo "FAIL: $FAILURES of $COUNT do not" |
