diff options
| -rw-r--r-- | org/tests_for_document_abstraction_shell_scripts.org | 161 | ||||
| -rwxr-xr-x | test/run-tests.sh | 2 | ||||
| -rwxr-xr-x | test/test-abstraction-db-roundtrip.sh | 76 | ||||
| -rwxr-xr-x | test/test-abstraction-db.sh | 79 |
4 files changed, 240 insertions, 78 deletions
diff --git a/org/tests_for_document_abstraction_shell_scripts.org b/org/tests_for_document_abstraction_shell_scripts.org index e321286..0f41ca6 100644 --- a/org/tests_for_document_abstraction_shell_scripts.org +++ b/org/tests_for_document_abstraction_shell_scripts.org @@ -30,7 +30,7 @@ # SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \ # ./test/run-tests.sh ./bin/spine-ldc # -# sqlite3 is needed by two of the four. If it is not on PATH the script says +# sqlite3 is needed by one of the five. If it is not on PATH the script says # so and re-runs itself inside a nix shell that has it, so this works: # SpinePOD=... ./test/run-tests.sh ./bin/spine-ldc # @@ -533,23 +533,38 @@ lev4_subtoc|object_subtoc report() { echo " MISMATCH $1: .ssp $2, db $3"; } -# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), the -# database beside it (pod/<uid>.ocda.db) +# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), one +# per language, and the database beside it (pod/<uid>.ocda.db), one per +# document holding every language of it. So the .ssp name carries the +# language and the database name does not: <uid>.<lang>.ssp beside +# <uid>.ocda.db, and inside the database the language's rows are the ones +# whose doc_id is the id documents gives that language. for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do [ -f "$ssp" ] || continue base=$(basename "$ssp" .ssp) - db="$OUT_DIR/pod/$base.ocda.db" + lang="${base##*.}" + stem="${base%.*}" + db="$OUT_DIR/pod/$stem.ocda.db" if [ ! -f "$db" ]; then echo "MISSING db for $base" FAILURES=$((FAILURES + 1)) continue fi + did=$(sqlite3 "$db" "select id from documents where lang='$lang';" 2>/dev/null || echo '') + if [ -z "$did" ]; then + echo "MISSING $lang in $(basename "$db"); it holds:" \ + "$(sqlite3 "$db" "select group_concat(lang,' ') from documents;" 2>/dev/null)" + FAILURES=$((FAILURES + 1)) + continue + fi + # the objects of this one language, for the child tables to key on + mine="object_id in (select id from objects where doc_id=$did)" DOCS=$((DOCS + 1)) doc_fail=0 # object population s_objs=$(grep -c '^\[' "$ssp" || true) - d_objs=$(sqlite3 "$db" "select count(*) from objects;" 2>/dev/null || echo ERR) + d_objs=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did;" 2>/dev/null || echo ERR) if [ "$s_objs" != "$d_objs" ]; then echo "$base"; report "objects" "$s_objs" "$d_objs"; doc_fail=1 fi @@ -557,7 +572,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do # per section for sec in head toc body endnotes glossary bibliography bookindex blurb; do s_sec=$(awk -v s="@$sec {" '$0==s{f=1;next} /^\}$/{f=0} f&&/^\[/{n++} END{print n+0}' "$ssp") - d_sec=$(sqlite3 "$db" "select count(*) from objects where section='$sec';" 2>/dev/null || echo ERR) + d_sec=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and section='$sec';" 2>/dev/null || echo ERR) if [ "$s_sec" != "$d_sec" ]; then [ $doc_fail -eq 0 ] && echo "$base" report "section $sec" "$s_sec" "$d_sec"; doc_fail=1 @@ -568,10 +583,10 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do echo "$CHECKS" | while IFS='|' read -r prop pred; do [ -z "$prop" ] && continue case "$prop" in - ,*": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;; - ,*) s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;; + *": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;; + *) s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;; esac - d_n=$(sqlite3 "$db" "select count(*) from objects where $pred;" 2>/dev/null || echo ERR) + d_n=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and ($pred);" 2>/dev/null || echo ERR) if [ "$s_n" != "$d_n" ]; then echo "$base: MISMATCH .$prop: .ssp $s_n, db $d_n" fi @@ -587,7 +602,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do echo "$LISTS" | while IFS='|' read -r prop table; do [ -z "$prop" ] && continue s_n=$(grep -c "^\.$prop: " "$ssp" || true) - d_n=$(sqlite3 "$db" "select count(*) from $table;" 2>/dev/null || echo ERR) + d_n=$(sqlite3 "$db" "select count(*) from $table where $mine;" 2>/dev/null || echo ERR) if [ "$s_n" != "$d_n" ]; then echo "$base: MISMATCH .$prop: .ssp $s_n, $table $d_n" fi @@ -599,10 +614,15 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do fi rm -f "$OUT_DIR/.lists.$$" - # every image the abstraction referred to and found is carried in the file - want=$(sqlite3 "$db" "select count(distinct name) from object_images where missing = 0;" 2>/dev/null || echo ERR) + # every image the abstraction referred to and found is carried in the file. + # the files table is shared: the languages of a document refer to the same + # images and one copy serves them all, so what is counted here is the names + # this language referred to and the stored files that answer them. + want=$(sqlite3 "$db" "select count(distinct name) from object_images + where missing = 0 and $mine;" 2>/dev/null || echo ERR) have=$(sqlite3 "$db" "select count(*) from files f where f.role='image' - and exists (select 1 from object_images i where i.name = f.name);" 2>/dev/null || echo ERR) + and exists (select 1 from object_images i where i.name = f.name + and i.missing = 0 and i.$mine);" 2>/dev/null || echo ERR) if [ "$want" != "$have" ]; then [ $doc_fail -eq 0 ] && echo "$base" report "images carried" "$want referenced" "$have stored"; doc_fail=1 @@ -610,9 +630,11 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do # the text itself, on the first and last citable object for pick in "min" "max"; do - ocn=$(sqlite3 "$db" "select $pick(ocn) from objects where ocn > 0 and section='body';" 2>/dev/null || echo "") + ocn=$(sqlite3 "$db" "select $pick(ocn) from objects + where doc_id=$did and ocn > 0 and section='body';" 2>/dev/null || echo "") [ -z "$ocn" ] && continue - sqlite3 "$db" "select text from objects where ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true + sqlite3 "$db" "select text from objects + where doc_id=$did and ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true awk -v o="[$ocn] " 'index($0,o)==1{f=1;next} f&&/^$/{exit} f&&/^\| /{print substr($0,3)}' \ "$ssp" > "$OUT_DIR/.ssptxt.$$" if ! diff -q "$OUT_DIR/.dbtxt.$$" "$OUT_DIR/.ssptxt.$$" > /dev/null 2>&1; then @@ -638,8 +660,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do } ' "$ssp" | sort > "$OUT_DIR/.sspmeta.$$" sqlite3 -separator "$(printf '\t')" "$db" \ - "select key, value from metadata where key not in - ('schema.name','schema.version','source.filename','source.ssp_digest');" \ + "select key, value from metadata where doc_id=$did and key not in + ('source.filename','source.ssp_digest');" \ 2>/dev/null | sort > "$OUT_DIR/.dbmeta.$$" if ! diff -q "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" > /dev/null 2>&1; then [ $doc_fail -eq 0 ] && echo "$base" @@ -650,7 +672,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do rm -f "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" # the source filename, and the version the two share - sf=$(sqlite3 "$db" "select value from metadata where key='source.filename';" 2>/dev/null || echo '') + sf=$(sqlite3 "$db" "select value from metadata + where doc_id=$did and key='source.filename';" 2>/dev/null || echo '') sf_ssp=$(sed -n 's/^% Source: //p' "$ssp" | head -1) if [ "$sf" != "$sf_ssp" ]; then [ $doc_fail -eq 0 ] && echo "$base" @@ -658,7 +681,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do fi sv_ssp=$(sed -n 's/^% SiSU Document Abstraction v//p' "$ssp" | head -1) # the file says what it is - sv=$(sqlite3 "$db" "select value from metadata where key='schema.version';" 2>/dev/null || echo '') + sv=$(sqlite3 "$db" "select value from metadata + where doc_id is null and key='schema.version';" 2>/dev/null || echo '') if [ -z "$sv" ]; then [ $doc_fail -eq 0 ] && echo "$base" echo " MISSING metadata schema.version"; doc_fail=1 @@ -670,10 +694,31 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do FAILURES=$((FAILURES + doc_fail)) done +# one database per document, holding every language of it: the languages the +# documents table names are exactly the languages there are .ssp files for. +# A per language loop cannot see this - it checks the languages it finds and +# would not notice one the database had gained or lost - so it is checked per +# database, once, here. +DBS=0 +for db in $(find "$OUT_DIR/pod" -maxdepth 1 -name "*.ocda.db" 2>/dev/null | sort); do + [ -f "$db" ] || continue + DBS=$((DBS + 1)) + stem=$(basename "$db" .ocda.db) + d_langs=$(sqlite3 "$db" "select lang from documents order by lang;" 2>/dev/null \ + | tr '\n' ' ' | sed 's/ $//') + s_langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \ + | sed "s|.*/$stem\.||; s|\.ssp$||" | sort | tr '\n' ' ' | sed 's/ $//') + if [ "$d_langs" != "$s_langs" ]; then + echo "$stem" + echo " MISMATCH languages: db '$d_langs', .ssp files '$s_langs'" + FAILURES=$((FAILURES + 1)) + fi +done + rm -rf "$OUT_DIR" if [ "$FAILURES" -eq 0 ]; then - echo "PASS: .ssp and .ocda.db agree for all $DOCS documents" + echo "PASS: .ssp and .ocda.db agree for all $DOCS documents in $DBS databases" exit 0 else echo "FAIL: $FAILURES document(s) disagree" @@ -691,7 +736,10 @@ fi # # Reads each <doc>.ocda.db back into the abstraction, emits it as # .ssp, and requires the result to be byte identical to the .ssp written -# from the same document. +# from the same document. A database holds every language of its document, +# so each language is asked for by name (--lang) and held against its own +# .ssp, and a database of more than one language is required to refuse a +# round trip that names none. # # What that proves: the two serialisations really do carry the same thing. # test-abstraction-db.sh counts fields and compares them; this reconstructs @@ -749,37 +797,70 @@ $SPINE_BIN --show-abstraction --ocda-db --skip-output \ --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1 COUNT=0 +DBS=0 FAILURES=0 -# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp inside it -# (pod/<doc>/media/abstraction/<uid>.ssp) +# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp files inside +# it (pod/<doc>/media/abstraction/<uid>.<lang>.ssp). +# +# A database holds every language of its document, so one database answers +# for as many .ssp files as the document has languages, and each has to be +# asked for by name: --lang says which language to emit. With no language +# named a database holding more than one refuses to choose, which is checked +# too - a round trip that quietly emitted the first language would pass +# against the first .ssp and say nothing about the other nine. for db in "$OUT_DIR"/pod/*.ocda.db; do [ -f "$db" ] || continue - base=$(basename "$db" .ocda.db) - ssp=$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1) - COUNT=$((COUNT + 1)) - if [ ! -f "$ssp" ]; then - echo "MISSING .ssp for $base" + DBS=$((DBS + 1)) + stem=$(basename "$db" .ocda.db) + # the languages come from the .ssp files rather than from the database, so + # that this script needs no sqlite3: each .ssp is a language the database + # has to be able to re-emit. A language in the database with no .ssp beside + # it is the other direction, and test-abstraction-db.sh checks that. + langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \ + | sed "s|.*/$stem\.||; s|\.ssp$||" | sort) + if [ -z "$langs" ]; then + echo "MISSING .ssp for $stem" FAILURES=$((FAILURES + 1)) continue fi - if ! $SPINE_BIN --db-round-trip="$db" > "$OUT_DIR/from-db.ssp" 2> "$OUT_DIR/from-db.err"; then - echo "ERROR: $base (spine exited non-zero)" - head -3 "$OUT_DIR/from-db.err" - FAILURES=$((FAILURES + 1)) - continue - fi - if ! cmp -s "$ssp" "$OUT_DIR/from-db.ssp"; then - echo "DIFFERS: $base ($(diff "$ssp" "$OUT_DIR/from-db.ssp" | grep -c '^[<>]') lines)" - diff --unified=1 "$ssp" "$OUT_DIR/from-db.ssp" | head -12 - echo " ..." - FAILURES=$((FAILURES + 1)) + n_langs=$(printf '%s\n' "$langs" | grep -c .) + for lang in $langs; do + base="$stem.$lang" + ssp="$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1)" + COUNT=$((COUNT + 1)) + if [ ! -f "$ssp" ]; then + echo "MISSING .ssp for $base" + FAILURES=$((FAILURES + 1)) + continue + fi + if ! $SPINE_BIN --db-round-trip="$db" --lang="$lang" \ + > "$OUT_DIR/from-db.ssp" 2> "$OUT_DIR/from-db.err"; then + echo "ERROR: $base (spine exited non-zero)" + head -3 "$OUT_DIR/from-db.err" + FAILURES=$((FAILURES + 1)) + continue + fi + if ! cmp -s "$ssp" "$OUT_DIR/from-db.ssp"; then + echo "DIFFERS: $base ($(diff "$ssp" "$OUT_DIR/from-db.ssp" | grep -c '^[<>]') lines)" + diff --unified=1 "$ssp" "$OUT_DIR/from-db.ssp" | head -12 + echo " ..." + FAILURES=$((FAILURES + 1)) + fi + done + # more than one language and none named: it must refuse rather than pick + if [ "$n_langs" -gt 1 ]; then + if $SPINE_BIN --db-round-trip="$db" > /dev/null 2> /dev/null; then + echo "$stem" + echo " ACCEPTED a round trip with no language named, holding $n_langs" + FAILURES=$((FAILURES + 1)) + fi fi done rm -rf "$OUT_DIR" if [ "$FAILURES" -eq 0 ]; then - echo "PASS: all $COUNT databases re-emit their document's .ssp exactly" + echo "PASS: $DBS databases re-emit all $COUNT of their documents' .ssp exactly" exit 0 else echo "FAIL: $FAILURES of $COUNT do not" diff --git a/test/run-tests.sh b/test/run-tests.sh index 1c8ac07..2f27892 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -7,7 +7,7 @@ # SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \ # ./test/run-tests.sh ./bin/spine-ldc # -# sqlite3 is needed by two of the four. If it is not on PATH the script says +# sqlite3 is needed by one of the five. If it is not on PATH the script says # so and re-runs itself inside a nix shell that has it, so this works: # SpinePOD=... ./test/run-tests.sh ./bin/spine-ldc # diff --git a/test/test-abstraction-db-roundtrip.sh b/test/test-abstraction-db-roundtrip.sh index 338f375..0ec7847 100755 --- a/test/test-abstraction-db-roundtrip.sh +++ b/test/test-abstraction-db-roundtrip.sh @@ -3,7 +3,10 @@ # # Reads each <doc>.ocda.db back into the abstraction, emits it as # .ssp, and requires the result to be byte identical to the .ssp written -# from the same document. +# from the same document. A database holds every language of its document, +# so each language is asked for by name (--lang) and held against its own +# .ssp, and a database of more than one language is required to refuse a +# round trip that names none. # # What that proves: the two serialisations really do carry the same thing. # test-abstraction-db.sh counts fields and compares them; this reconstructs @@ -61,37 +64,70 @@ $SPINE_BIN --show-abstraction --ocda-db --skip-output \ --output="$OUT_DIR" "$SAMPLES_DIR"/* 2>&1 | tail -1 COUNT=0 +DBS=0 FAILURES=0 -# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp inside it -# (pod/<doc>/media/abstraction/<uid>.ssp) +# the database sits beside the pod (pod/<uid>.ocda.db), the .ssp files inside +# it (pod/<doc>/media/abstraction/<uid>.<lang>.ssp). +# +# A database holds every language of its document, so one database answers +# for as many .ssp files as the document has languages, and each has to be +# asked for by name: --lang says which language to emit. With no language +# named a database holding more than one refuses to choose, which is checked +# too - a round trip that quietly emitted the first language would pass +# against the first .ssp and say nothing about the other nine. for db in "$OUT_DIR"/pod/*.ocda.db; do [ -f "$db" ] || continue - base=$(basename "$db" .ocda.db) - ssp=$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1) - COUNT=$((COUNT + 1)) - if [ ! -f "$ssp" ]; then - echo "MISSING .ssp for $base" + DBS=$((DBS + 1)) + stem=$(basename "$db" .ocda.db) + # the languages come from the .ssp files rather than from the database, so + # that this script needs no sqlite3: each .ssp is a language the database + # has to be able to re-emit. A language in the database with no .ssp beside + # it is the other direction, and test-abstraction-db.sh checks that. + langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \ + | sed "s|.*/$stem\.||; s|\.ssp$||" | sort) + if [ -z "$langs" ]; then + echo "MISSING .ssp for $stem" FAILURES=$((FAILURES + 1)) continue fi - if ! $SPINE_BIN --db-round-trip="$db" > "$OUT_DIR/from-db.ssp" 2> "$OUT_DIR/from-db.err"; then - echo "ERROR: $base (spine exited non-zero)" - head -3 "$OUT_DIR/from-db.err" - FAILURES=$((FAILURES + 1)) - continue - fi - if ! cmp -s "$ssp" "$OUT_DIR/from-db.ssp"; then - echo "DIFFERS: $base ($(diff "$ssp" "$OUT_DIR/from-db.ssp" | grep -c '^[<>]') lines)" - diff --unified=1 "$ssp" "$OUT_DIR/from-db.ssp" | head -12 - echo " ..." - FAILURES=$((FAILURES + 1)) + n_langs=$(printf '%s\n' "$langs" | grep -c .) + for lang in $langs; do + base="$stem.$lang" + ssp="$(find "$OUT_DIR/pod" -name "$base.ssp" 2>/dev/null | head -1)" + COUNT=$((COUNT + 1)) + if [ ! -f "$ssp" ]; then + echo "MISSING .ssp for $base" + FAILURES=$((FAILURES + 1)) + continue + fi + if ! $SPINE_BIN --db-round-trip="$db" --lang="$lang" \ + > "$OUT_DIR/from-db.ssp" 2> "$OUT_DIR/from-db.err"; then + echo "ERROR: $base (spine exited non-zero)" + head -3 "$OUT_DIR/from-db.err" + FAILURES=$((FAILURES + 1)) + continue + fi + if ! cmp -s "$ssp" "$OUT_DIR/from-db.ssp"; then + echo "DIFFERS: $base ($(diff "$ssp" "$OUT_DIR/from-db.ssp" | grep -c '^[<>]') lines)" + diff --unified=1 "$ssp" "$OUT_DIR/from-db.ssp" | head -12 + echo " ..." + FAILURES=$((FAILURES + 1)) + fi + done + # more than one language and none named: it must refuse rather than pick + if [ "$n_langs" -gt 1 ]; then + if $SPINE_BIN --db-round-trip="$db" > /dev/null 2> /dev/null; then + echo "$stem" + echo " ACCEPTED a round trip with no language named, holding $n_langs" + FAILURES=$((FAILURES + 1)) + fi fi done rm -rf "$OUT_DIR" if [ "$FAILURES" -eq 0 ]; then - echo "PASS: all $COUNT databases re-emit their document's .ssp exactly" + echo "PASS: $DBS databases re-emit all $COUNT of their documents' .ssp exactly" exit 0 else echo "FAIL: $FAILURES of $COUNT do not" diff --git a/test/test-abstraction-db.sh b/test/test-abstraction-db.sh index 6a0e4e6..2fac414 100755 --- a/test/test-abstraction-db.sh +++ b/test/test-abstraction-db.sh @@ -121,23 +121,38 @@ lev4_subtoc|object_subtoc report() { echo " MISMATCH $1: .ssp $2, db $3"; } -# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), the -# database beside it (pod/<uid>.ocda.db) +# the .ssp is written into the pod tree (pod/<doc>/media/abstraction/), one +# per language, and the database beside it (pod/<uid>.ocda.db), one per +# document holding every language of it. So the .ssp name carries the +# language and the database name does not: <uid>.<lang>.ssp beside +# <uid>.ocda.db, and inside the database the language's rows are the ones +# whose doc_id is the id documents gives that language. for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do [ -f "$ssp" ] || continue base=$(basename "$ssp" .ssp) - db="$OUT_DIR/pod/$base.ocda.db" + lang="${base##*.}" + stem="${base%.*}" + db="$OUT_DIR/pod/$stem.ocda.db" if [ ! -f "$db" ]; then echo "MISSING db for $base" FAILURES=$((FAILURES + 1)) continue fi + did=$(sqlite3 "$db" "select id from documents where lang='$lang';" 2>/dev/null || echo '') + if [ -z "$did" ]; then + echo "MISSING $lang in $(basename "$db"); it holds:" \ + "$(sqlite3 "$db" "select group_concat(lang,' ') from documents;" 2>/dev/null)" + FAILURES=$((FAILURES + 1)) + continue + fi + # the objects of this one language, for the child tables to key on + mine="object_id in (select id from objects where doc_id=$did)" DOCS=$((DOCS + 1)) doc_fail=0 # object population s_objs=$(grep -c '^\[' "$ssp" || true) - d_objs=$(sqlite3 "$db" "select count(*) from objects;" 2>/dev/null || echo ERR) + d_objs=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did;" 2>/dev/null || echo ERR) if [ "$s_objs" != "$d_objs" ]; then echo "$base"; report "objects" "$s_objs" "$d_objs"; doc_fail=1 fi @@ -145,7 +160,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do # per section for sec in head toc body endnotes glossary bibliography bookindex blurb; do s_sec=$(awk -v s="@$sec {" '$0==s{f=1;next} /^\}$/{f=0} f&&/^\[/{n++} END{print n+0}' "$ssp") - d_sec=$(sqlite3 "$db" "select count(*) from objects where section='$sec';" 2>/dev/null || echo ERR) + d_sec=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and section='$sec';" 2>/dev/null || echo ERR) if [ "$s_sec" != "$d_sec" ]; then [ $doc_fail -eq 0 ] && echo "$base" report "section $sec" "$s_sec" "$d_sec"; doc_fail=1 @@ -159,7 +174,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do *": true") s_n=$(grep -c "^\.$prop\$" "$ssp" || true) ;; *) s_n=$(grep -c "^\.$prop: " "$ssp" || true) ;; esac - d_n=$(sqlite3 "$db" "select count(*) from objects where $pred;" 2>/dev/null || echo ERR) + d_n=$(sqlite3 "$db" "select count(*) from objects where doc_id=$did and ($pred);" 2>/dev/null || echo ERR) if [ "$s_n" != "$d_n" ]; then echo "$base: MISMATCH .$prop: .ssp $s_n, db $d_n" fi @@ -175,7 +190,7 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do echo "$LISTS" | while IFS='|' read -r prop table; do [ -z "$prop" ] && continue s_n=$(grep -c "^\.$prop: " "$ssp" || true) - d_n=$(sqlite3 "$db" "select count(*) from $table;" 2>/dev/null || echo ERR) + d_n=$(sqlite3 "$db" "select count(*) from $table where $mine;" 2>/dev/null || echo ERR) if [ "$s_n" != "$d_n" ]; then echo "$base: MISMATCH .$prop: .ssp $s_n, $table $d_n" fi @@ -187,10 +202,15 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do fi rm -f "$OUT_DIR/.lists.$$" - # every image the abstraction referred to and found is carried in the file - want=$(sqlite3 "$db" "select count(distinct name) from object_images where missing = 0;" 2>/dev/null || echo ERR) + # every image the abstraction referred to and found is carried in the file. + # the files table is shared: the languages of a document refer to the same + # images and one copy serves them all, so what is counted here is the names + # this language referred to and the stored files that answer them. + want=$(sqlite3 "$db" "select count(distinct name) from object_images + where missing = 0 and $mine;" 2>/dev/null || echo ERR) have=$(sqlite3 "$db" "select count(*) from files f where f.role='image' - and exists (select 1 from object_images i where i.name = f.name);" 2>/dev/null || echo ERR) + and exists (select 1 from object_images i where i.name = f.name + and i.missing = 0 and i.$mine);" 2>/dev/null || echo ERR) if [ "$want" != "$have" ]; then [ $doc_fail -eq 0 ] && echo "$base" report "images carried" "$want referenced" "$have stored"; doc_fail=1 @@ -198,9 +218,11 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do # the text itself, on the first and last citable object for pick in "min" "max"; do - ocn=$(sqlite3 "$db" "select $pick(ocn) from objects where ocn > 0 and section='body';" 2>/dev/null || echo "") + ocn=$(sqlite3 "$db" "select $pick(ocn) from objects + where doc_id=$did and ocn > 0 and section='body';" 2>/dev/null || echo "") [ -z "$ocn" ] && continue - sqlite3 "$db" "select text from objects where ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true + sqlite3 "$db" "select text from objects + where doc_id=$did and ocn=$ocn and section='body';" > "$OUT_DIR/.dbtxt.$$" 2>/dev/null || true awk -v o="[$ocn] " 'index($0,o)==1{f=1;next} f&&/^$/{exit} f&&/^\| /{print substr($0,3)}' \ "$ssp" > "$OUT_DIR/.ssptxt.$$" if ! diff -q "$OUT_DIR/.dbtxt.$$" "$OUT_DIR/.ssptxt.$$" > /dev/null 2>&1; then @@ -226,8 +248,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do } ' "$ssp" | sort > "$OUT_DIR/.sspmeta.$$" sqlite3 -separator "$(printf '\t')" "$db" \ - "select key, value from metadata where key not in - ('schema.name','schema.version','source.filename','source.ssp_digest');" \ + "select key, value from metadata where doc_id=$did and key not in + ('source.filename','source.ssp_digest');" \ 2>/dev/null | sort > "$OUT_DIR/.dbmeta.$$" if ! diff -q "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" > /dev/null 2>&1; then [ $doc_fail -eq 0 ] && echo "$base" @@ -238,7 +260,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do rm -f "$OUT_DIR/.sspmeta.$$" "$OUT_DIR/.dbmeta.$$" # the source filename, and the version the two share - sf=$(sqlite3 "$db" "select value from metadata where key='source.filename';" 2>/dev/null || echo '') + sf=$(sqlite3 "$db" "select value from metadata + where doc_id=$did and key='source.filename';" 2>/dev/null || echo '') sf_ssp=$(sed -n 's/^% Source: //p' "$ssp" | head -1) if [ "$sf" != "$sf_ssp" ]; then [ $doc_fail -eq 0 ] && echo "$base" @@ -246,7 +269,8 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do fi sv_ssp=$(sed -n 's/^% SiSU Document Abstraction v//p' "$ssp" | head -1) # the file says what it is - sv=$(sqlite3 "$db" "select value from metadata where key='schema.version';" 2>/dev/null || echo '') + sv=$(sqlite3 "$db" "select value from metadata + where doc_id is null and key='schema.version';" 2>/dev/null || echo '') if [ -z "$sv" ]; then [ $doc_fail -eq 0 ] && echo "$base" echo " MISSING metadata schema.version"; doc_fail=1 @@ -258,10 +282,31 @@ for ssp in $(find "$OUT_DIR/pod" -name "*.ssp" 2>/dev/null | sort); do FAILURES=$((FAILURES + doc_fail)) done +# one database per document, holding every language of it: the languages the +# documents table names are exactly the languages there are .ssp files for. +# A per language loop cannot see this - it checks the languages it finds and +# would not notice one the database had gained or lost - so it is checked per +# database, once, here. +DBS=0 +for db in $(find "$OUT_DIR/pod" -maxdepth 1 -name "*.ocda.db" 2>/dev/null | sort); do + [ -f "$db" ] || continue + DBS=$((DBS + 1)) + stem=$(basename "$db" .ocda.db) + d_langs=$(sqlite3 "$db" "select lang from documents order by lang;" 2>/dev/null \ + | tr '\n' ' ' | sed 's/ $//') + s_langs=$(find "$OUT_DIR/pod" -name "$stem.*.ssp" 2>/dev/null \ + | sed "s|.*/$stem\.||; s|\.ssp$||" | sort | tr '\n' ' ' | sed 's/ $//') + if [ "$d_langs" != "$s_langs" ]; then + echo "$stem" + echo " MISMATCH languages: db '$d_langs', .ssp files '$s_langs'" + FAILURES=$((FAILURES + 1)) + fi +done + rm -rf "$OUT_DIR" if [ "$FAILURES" -eq 0 ]; then - echo "PASS: .ssp and .ocda.db agree for all $DOCS documents" + echo "PASS: .ssp and .ocda.db agree for all $DOCS documents in $DBS databases" exit 0 else echo "FAIL: $FAILURES document(s) disagree" |