#!/usr/bin/env sh
# run-tests.sh
#
# Runs the abstraction tests in sequence and reports one line each.
#
# Usage:
#   SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod \
#     ./test/run-tests.sh ./bin/spine-ldc
#
# Two outside tools are needed: sqlite3 by one script and epubcheck by
# another. If either is not on PATH the harness re-runs itself inside a nix
# shell that has both, so this works as it stands:
#   SpinePOD=... ./test/run-tests.sh ./bin/spine-ldc
#
# Exit codes:
#   0  every test ran and passed
#   n  the number of tests that ran and failed
#   2  the suite did not run in full: a set-up problem here (no $SpinePOD,
#      no binary, a missing tool) or a script exiting 2 for one of its own.
#      Reported as INCOMPLETE rather than PASS, because nothing should read
#      "it passed" out of a check that never ran
#
# Order, and what depends on what:
#
#   1 test-abstraction-ssp.sh            the abstraction has not changed
#   2 test-run-order.sh                  nor does it depend on the run
#   3 test-abstraction-ssp-roundtrip.sh  the .ssp can be read back whole
#   4 test-abstraction-db.sh             the two serialisations agree
#   5 test-abstraction-db-roundtrip.sh   the .db can be read back whole
#   6 test-ocn-alignment.sh              the languages agree on their ocns
#   7 test-acceptance-db-to-pod.sh       a database gives its document back
#   8 test-epub-validity.sh              the epubs a reader can actually open
#
# None of them calls another, and each can be run on its own. The order
# matters only for reading the result:
#
#   - 1 is first because it is the one that says whether the abstraction
#     itself moved. If it fails, the others are answering a different
#     question than you think, and its failure is the one to look at.
#   - 2 says that a document abstracted alone is the document abstracted in
#     company. 1 only ever runs the whole sample set, so an abstraction
#     that depended on what else was in the run passed it; this is the
#     check that would have caught that, and the one that says an artefact
#     can be verified on its own.
#   - 3 reads the committed reference set, so it is only a statement about
#     the current binary if 1 passes. Run together, they say: the abstraction
#     is unchanged, and this binary can read that abstraction back.
#   - 4 and 5 generate both artefacts themselves and depend on nothing
#     committed. Since the database is built from the .ssp, they are checks
#     on the reader rather than on two independent writers.
#   - 6 is about the sample documents rather than about a serialisation: it
#     asserts that spine reports the two known divergences in live-manual's
#     translations and reports nothing about the other documents. It will
#     fail if those translations are mended, which is the right moment to
#     hear about it.
#   - 7 is the acceptance test for the document source claim, and the widest
#     of them: it builds a document from its pod and again from the database
#     that build produced, and requires the two output trees to be identical.
#     Where 4 and 5 check one serialisation against another, this checks a
#     whole document through the whole pipeline twice. If it fails and the
#     others pass, what broke is the carrying or the materialising rather
#     than the abstraction.

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SPINE_BIN="${1:-}"

if [ -z "$SpinePOD" ]; then
  echo "ERROR: \$SpinePOD is not set." >&2
  echo "  e.g. SpinePOD=../../markup/sisudoc-spine-samples/markup/pod-samples/pod" >&2
  exit 2
fi

# the tools the scripts need, and the shell that provides them.
#
# Both are checked, not just sqlite3. Checking one and providing two meant
# that a shell with sqlite3 already on PATH - a dev shell, or any machine
# with sqlite installed - did not re-exec, and test-epub-validity.sh then
# could not find epubcheck and reported a set-up problem that read as a
# failing test.
_missing=""
command -v sqlite3   > /dev/null 2>&1 || _missing="$_missing sqlite3"
command -v epubcheck > /dev/null 2>&1 || _missing="$_missing epubcheck"
if [ -n "$_missing" ]; then
  if [ -n "$SpineTestsReExeced" ]; then
    echo "ERROR: still missing$_missing inside the nix shell." >&2
    exit 2
  fi
  if command -v nix > /dev/null 2>&1; then
    echo "not on PATH:$_missing - re-running inside nix shell (sqlite, epubcheck)"
    SpineTestsReExeced=1 exec nix shell "nixpkgs#sqlite" "nixpkgs#epubcheck" \
      -c "$0" "$@"
  fi
  echo "ERROR: missing$_missing and nix is not available." >&2
  exit 2
fi

TESTS="test-abstraction-ssp.sh
test-run-order.sh
test-abstraction-ssp-roundtrip.sh
test-abstraction-db.sh
test-abstraction-db-roundtrip.sh
test-ocn-alignment.sh
test-acceptance-db-to-pod.sh
test-epub-validity.sh"

PASSED=0
FAILED=0
SETUP=0
RESULTS=""
LOG="${TMPDIR:-/tmp}/spine-run-tests.$$"

for t in $TESTS; do
  printf '%s\n' "── $t"
  # the status has to be the test's, not the last command of a pipeline:
  # "cmd | sed" reports sed's status, which is always 0, and a harness that
  # cannot fail is worse than no harness
  # "|| _rc=$?" and not a bare call: set -e is on, and a bare non-zero
  # would end the harness at the first failing test instead of running the
  # rest and summarising
  _rc=0
  sh "$SCRIPT_DIR/$t" $SPINE_BIN > "$LOG" 2>&1 || _rc=$?
  sed 's/^/   /' "$LOG"
  # every script uses 2 for "could not run" and 1 for "ran and failed", and
  # the two must not read alike: "FAIL test-epub-validity.sh" says the epubs
  # are invalid, where exit 2 says nobody checked them
  case "$_rc" in
    0) PASSED=$((PASSED + 1)); RESULTS="$RESULTS
  ok      $t" ;;
    2) SETUP=$((SETUP + 1));   RESULTS="$RESULTS
  SET-UP  $t   (could not run; nothing was checked)" ;;
    *) FAILED=$((FAILED + 1)); RESULTS="$RESULTS
  FAIL    $t" ;;
  esac
done
rm -f "$LOG"

echo ""
echo "summary:$RESULTS"
echo ""
_total=$((PASSED + FAILED + SETUP))
if [ "$FAILED" -eq 0 ] && [ "$SETUP" -eq 0 ]; then
  echo "PASS: $PASSED of $_total"
  exit 0
elif [ "$FAILED" -eq 0 ]; then
  # nothing failed, but the suite did not run in full, and saying PASS here
  # would claim more than was checked
  echo "INCOMPLETE: $PASSED of $_total ran, $SETUP could not"
  exit 2
else
  echo "FAIL: $FAILED of $_total"
  exit "$FAILED"
fi