1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
|
#!/usr/bin/env sh
# run-tests.sh
#
# Runs the abstraction tests in sequence and reports one line each.
#
# Usage:
# SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \
# ./test/run-tests.sh ./bin/spine-ldc
#
# Two outside tools are needed: sqlite3 by one script and epubcheck by
# another. If either is not on PATH the harness re-runs itself inside a nix
# shell that has both, so this works as it stands:
# SpinePOD=... ./test/run-tests.sh ./bin/spine-ldc
#
# Exit codes:
# 0 every test ran and passed
# n the number of tests that ran and failed
# 2 the suite did not run in full: a set-up problem here (no $SpinePOD,
# no binary, a missing tool) or a script exiting 2 for one of its own.
# Reported as INCOMPLETE rather than PASS, because nothing should read
# "it passed" out of a check that never ran
#
# Order, and what depends on what:
#
# 1 test-abstraction-ssp.sh the abstraction has not changed
# 2 test-abstraction-ssp-roundtrip.sh the .ssp can be read back whole
# 3 test-abstraction-db.sh the two serialisations agree
# 4 test-abstraction-db-roundtrip.sh the .db can be read back whole
# 5 test-ocn-alignment.sh the languages agree on their ocns
# 6 test-acceptance-db-to-pod.sh a database gives its document back
# 7 test-epub-validity.sh the epubs a reader can actually open
#
# None of them calls another, and each can be run on its own. The order
# matters only for reading the result:
#
# - 1 is first because it is the one that says whether the abstraction
# itself moved. If it fails, the others are answering a different
# question than you think, and its failure is the one to look at.
# - 2 reads the committed reference set, so it is only a statement about
# the current binary if 1 passes. Run together, they say: the abstraction
# is unchanged, and this binary can read that abstraction back.
# - 3 and 4 generate both artefacts themselves and depend on nothing
# committed. Since the database is built from the .ssp, they are checks
# on the reader rather than on two independent writers.
# - 5 is about the sample documents rather than about a serialisation: it
# asserts that spine reports the two known divergences in live-manual's
# translations and reports nothing about the other documents. It will
# fail if those translations are mended, which is the right moment to
# hear about it.
# - 6 is the acceptance test for the document source claim, and the widest
# of them: it builds a document from its pod and again from the database
# that build produced, and requires the two output trees to be identical.
# Where 3 and 4 check one serialisation against another, this checks a
# whole document through the whole pipeline twice. If it fails and the
# others pass, what broke is the carrying or the materialising rather
# than the abstraction.
set -e
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SPINE_BIN="${1:-}"
if [ -z "$SpinePOD" ]; then
echo "ERROR: \$SpinePOD is not set." >&2
echo " e.g. SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod" >&2
exit 2
fi
# the tools the scripts need, and the shell that provides them.
#
# Both are checked, not just sqlite3. Checking one and providing two meant
# that a shell with sqlite3 already on PATH - a dev shell, or any machine
# with sqlite installed - did not re-exec, and test-epub-validity.sh then
# could not find epubcheck and reported a set-up problem that read as a
# failing test.
_missing=""
command -v sqlite3 > /dev/null 2>&1 || _missing="$_missing sqlite3"
command -v epubcheck > /dev/null 2>&1 || _missing="$_missing epubcheck"
if [ -n "$_missing" ]; then
if [ -n "$SpineTestsReExeced" ]; then
echo "ERROR: still missing$_missing inside the nix shell." >&2
exit 2
fi
if command -v nix > /dev/null 2>&1; then
echo "not on PATH:$_missing - re-running inside nix shell (sqlite, epubcheck)"
SpineTestsReExeced=1 exec nix shell "nixpkgs#sqlite" "nixpkgs#epubcheck" \
-c "$0" "$@"
fi
echo "ERROR: missing$_missing and nix is not available." >&2
exit 2
fi
TESTS="test-abstraction-ssp.sh
test-abstraction-ssp-roundtrip.sh
test-abstraction-db.sh
test-abstraction-db-roundtrip.sh
test-ocn-alignment.sh
test-acceptance-db-to-pod.sh
test-epub-validity.sh"
PASSED=0
FAILED=0
SETUP=0
RESULTS=""
LOG="${TMPDIR:-/tmp}/spine-run-tests.$$"
for t in $TESTS; do
printf '%s\n' "── $t"
# the status has to be the test's, not the last command of a pipeline:
# "cmd | sed" reports sed's status, which is always 0, and a harness that
# cannot fail is worse than no harness
# "|| _rc=$?" and not a bare call: set -e is on, and a bare non-zero
# would end the harness at the first failing test instead of running the
# rest and summarising
_rc=0
sh "$SCRIPT_DIR/$t" $SPINE_BIN > "$LOG" 2>&1 || _rc=$?
sed 's/^/ /' "$LOG"
# every script uses 2 for "could not run" and 1 for "ran and failed", and
# the two must not read alike: "FAIL test-epub-validity.sh" says the epubs
# are invalid, where exit 2 says nobody checked them
case "$_rc" in
0) PASSED=$((PASSED + 1)); RESULTS="$RESULTS
ok $t" ;;
2) SETUP=$((SETUP + 1)); RESULTS="$RESULTS
SET-UP $t (could not run; nothing was checked)" ;;
*) FAILED=$((FAILED + 1)); RESULTS="$RESULTS
FAIL $t" ;;
esac
done
rm -f "$LOG"
echo ""
echo "summary:$RESULTS"
echo ""
_total=$((PASSED + FAILED + SETUP))
if [ "$FAILED" -eq 0 ] && [ "$SETUP" -eq 0 ]; then
echo "PASS: $PASSED of $_total"
exit 0
elif [ "$FAILED" -eq 0 ]; then
# nothing failed, but the suite did not run in full, and saying PASS here
# would claim more than was checked
echo "INCOMPLETE: $PASSED of $_total ran, $SETUP could not"
exit 2
else
echo "FAIL: $FAILED of $_total"
exit "$FAILED"
fi
|
