flake devshell: issues with diffoscope & datasetteHEADmain

flake devshell issues with a couple of nixpkgs:

diffoscope pinned to nixos-26.05
  made available in dsh-production-post
  - datasette is useful for viewing sisudoc spine .ocda.db files without
    the need to write any sql.

datasette pinned to nixos-26.05 and patched asgi-csrf
  made available in dsh-production & dsh-production-post
  - diffs two epubs, two odts or two pdfs by content rather (than as
    bytes) (useful when output moves and cmp only says "differ")

datasette broken in nixpkgs (both in stable nixos-26.05 and unstable)
nix refuses to evaluate it, fix with patched asgi-csrf
(used nixos-26.05)

python-multipart 0.0.26 removed FormParser's FileClass argument, which
asgi-csrf passes (simonw/asgi-csrf#38, open), and 26.05 carries 0.0.29.

A nixpkgs-stable input (nixos-26.05, by its channel url, locked), and
from it, for datasette alone, a python package set in which:

- asgi-csrf hands its own ErrorOnWrite(...).write to on_file, which
  python-multipart still calls as each file part ends, in place of
  FileClass: a file arriving before the csrf token is still refused,
  and asgi-csrf's 48 tests pass, the multipart ones included;
- datasette skips test_max_csv_mb, which streams a large csv under the
  query time limit and is interrupted on a loaded builder.

datasette is made available in dsh-production and dsh-production-post.
datasette 0.65.2 serves an .ocda.db, its tables, views, json api, sql
and facets.

(assisted by Claude-Code)

2 files changed, 397 insertions, 1 deletions

diff --git a/org/config_nix.org b/org/config_nix.org
index cea53f2..146c9ea 100644
--- a/org/config_nix.org
+++ b/org/config_nix.org
@@ -16,12 +16,390 @@
 * nix :nix:
 ** flake.nix :flake:
 
+flake devshell, issues with diffoscope and with datasette
+
+diffoscope sorted by pinning for the time being to stable branch
+
+nix devshell: fix datasette with patched asgi-csrf
+
+datasette is useful for viewing sisudoc spine .ocda.db files without the need to
+write any sql.
+
+datasette broken in nixpkgs (both in stable nixos-26.05 and unstable) nix
+refuses to evaluate it, fix with patched asgi-csrf (used nixos-26.05)
+
+python-multipart 0.0.26 removed FormParser's FileClass argument, which asgi-csrf
+passes (simonw/asgi-csrf#38, open), and 26.05 carries 0.0.29.
+
+A nixpkgs-stable input (nixos-26.05, by its channel url, locked), and from it,
+for datasette alone, a python package set in which:
+
+- asgi-csrf hands its own ErrorOnWrite(...).write to on_file, which
+  python-multipart still calls as each file part ends, in place of FileClass: a
+  file arriving before the csrf token is still refused, and asgi-csrf's 48 tests
+  pass, the multipart ones included;
+- datasette skips test_max_csv_mb, which streams a large csv under the query
+  time limit and is interrupted on a loaded builder.
+
+datasette is made available in dsh-production and dsh-production-post. datasette
+0.65.2 serves an .ocda.db, its tables, views, json api, sql and facets.
+
 #+HEADER: :tangle ../flake.nix
 #+HEADER: :noweb yes
 #+BEGIN_SRC nix
 {
   description = "a sisu like parser & document generator";
   inputs.nixpkgs.url = "<<nixpkgs_unstable>>";
+  inputs.nixpkgs-stable.url = "<<nixpkgs_stable>>";
+  outputs = {
+    self,
+    nixpkgs,
+    ...
+  }@inputs: let
+    pname = "spine";
+    version = "<<spine_version>>";
+    supportedSystems = ["x86_64-linux"]; # [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ];
+    forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
+    nixpkgsFor = forAllSystems (system: import nixpkgs {inherit system;}); # nixpkgs instantiated for supported system types
+    localOverlay = (final: prev: {
+      ldc = prev.callPackage ./nix-overlays/ldc {  };                           # -> ok <<ldc_version>>
+      # dmd overlay: revert to GCC14; (current) GCC 15 introduced nullptr in its headers, and DMD's ImportC parser needs update to handle it, monitor and update:
+      dmd = prev.callPackage ./nix-overlays/dmd { stdenv = prev.gcc14Stdenv; }; # -> ok <<dmd_version>>
+      dub = prev.callPackage ./nix-overlays/dub {  };                           # -> ? <<dub_version>>
+      dtools = prev.callPackage ./nix-overlays/dtools {  };                     # -> ok <<dtools_version>>
+      #gdc = prev.callPackage ./nix-overlays/gdc {  };
+    });
+    pkgsForSystem = system: import nixpkgs {
+      overlays = [
+        localOverlay
+      ];
+      inherit system;
+    };
+    mkSpine = { pkgs, compilerPkg, compilerBin, buildType }: pkgs.stdenv.mkDerivation {
+      inherit pname version;
+      meta.mainProgram = "spine";
+      executable = true; # not used
+      src = self;
+      buildInputs = [ pkgs.sqlite pkgs.zstd.out ];
+      nativeBuildInputs = [ pkgs.dub compilerPkg pkgs.gnumake ];
+      preBuild = ''
+        export HOME=$TMPDIR
+      '';
+      buildPhase = ''
+        runHook preBuild
+        buildCMD="dub run --cache=local --compiler=$(type -P ${compilerBin}) --build=${buildType} --combined --skip-registry=all"
+        echo $buildCMD
+        $buildCMD
+        echo $buildCMD
+        runHook postBuild
+      '';
+      checkPhase = ''
+        runHook preCheck
+        dub test --combined --skip-registry=all
+        runHook postCheck
+      '';
+      installPhase = ''
+        runHook preInstall
+        mkdir -p $out/bin
+        install -m755 ./bin/spine $out/bin/spine
+        runHook postInstall
+      '';
+      postInstall = ''
+        echo "❯❯ ./result/bin/spine -v --source --pod --text --epub --html --html-link-pdf --html-link-curate --html-link-markup --curate --output=./OUTPUT_TEST ../sisu-spine-samples/markup/pod-samples/pod/*";
+        echo `ls -la $out/bin/spine`
+        echo "❯❯ spine-v${version} (rev: ${self.shortRev or "unknown"})"
+        $out/bin/spine -v
+      '';
+    };
+    shellHook = ''
+      #export Date=`date "+%Y%m%d"`
+      ## set local values in .envrc-local (or here if you must)
+      echo '❯❯ nix build';
+      ## ImPure Nix environment
+      SpineGitVer=`git describe | sed "s/^[a-z_-]\+\([0-9.]\+\)/\1/" | sed "s/\([^-]*-g\)/r\1/" | sed "s/-/./g"` && \
+      SpineGitBranch=`git branch --show-current` && \
+      echo "❯❯ spine-v${version} - ($SpineGitBranch: $SpineGitVer)"
+      ## Pure Nix environment - version info from flake inputs
+      #echo "❯❯ spine-v${version} (rev: ${self.shortRev or "unknown"})"
+      echo '❯❯ $SpineBIN -v --source --pod --text --epub --html --html-link-pdf --html-link-curate --html-link-markup --curate --output=$SpineOUT  $SpinePOD/*';
+      echo "❯❯ $SpineBIN -v --source --pod --text --epub --html --html-link-pdf --html-link-curate --html-link-markup --curate --output=$SpineOUT  $SpinePOD/*";
+      echo '❯❯ nix flake update && nix flake check && nix flake show';
+    '';
+    mkDevShell = { pkgs, name, compiler, extraPackages ? [] }: pkgs.mkShell {
+      inherit name shellHook;
+      packages = [
+        pkgs.direnv
+        pkgs.nix-direnv
+        compiler
+        pkgs.dub
+        pkgs.dtools
+        pkgs.gnumake
+        pkgs.sqlite
+        pkgs.zstd
+        pkgs.gnugrep
+        pkgs.gnused
+        pkgs.ripgrep
+      ] ++ extraPackages;
+    };
+  in {
+    packages = forAllSystems (system: let
+      pkgs-ovl = pkgsForSystem system;
+      pkgs-nix = nixpkgsFor.${system};
+    in {
+      default = self.packages.${system}.spine-nixpkgs-ldc;
+      spine-nixpkgs-dmd = mkSpine {
+        pkgs = pkgs-nix;
+        # nixpkgs dmd: revert to GCC14 (same nullptr/ImportC fix as overlay dmd)
+        compilerPkg = pkgs-nix.dmd.override { stdenv = pkgs-nix.gcc14Stdenv; };
+        #compilerPkg = pkgs-nix.dmd;
+        compilerBin = "dmd";
+        buildType = "dmd";
+      };
+      spine-nixpkgs-ldc = mkSpine {
+        pkgs = pkgs-nix;
+        compilerPkg = pkgs-nix.ldc;
+        compilerBin = "ldmd2";
+        buildType = "ldmd2";
+      };
+      spine-overlay-dmd = mkSpine {
+        pkgs = pkgs-ovl;
+        compilerPkg = pkgs-ovl.dmd;
+        compilerBin = "dmd";
+        buildType = "dmd";
+      };
+      spine-overlay-ldc = mkSpine {
+        pkgs = pkgs-ovl;
+        compilerPkg = pkgs-ovl.ldc;
+        compilerBin = "ldmd2";
+        buildType = "ldmd2";
+      };
+      #spine-overlay-gdc = mkSpine {
+      #  pkgs = pkgs-ovl;
+      #  compilerPkg = pkgs-ovl.gdc;
+      #  compilerBin = "gdc";
+      #  buildType = "gdc";
+      #};
+    });
+    apps = forAllSystems (system: {
+      default = {
+        type = "app";
+        program = "${self.packages.${system}.default}/bin/spine";
+      };
+    });
+    devShells = forAllSystems (system: let
+      pkgs-ovl = pkgsForSystem system;
+      pkgs-nix = nixpkgsFor.${system};
+      # datasette, from the stable channel. It is held back both in stable
+      # and in unstable by asgi-csrf 0.11, marked broken because
+      # python-multipart 0.0.26 dropped FormParser's FileClass argument
+      # (simonw/asgi-csrf#38, open). asgi-csrf used it only to refuse a file
+      # arriving before the csrf token, by a file class whose write raises;
+      # given instead as on_file, which python-multipart still calls as each
+      # file ends, that write refuses it the same way, and asgi-csrf's own
+      # tests then pass.
+      pkgs-stable = import inputs.nixpkgs-stable {
+        inherit system;
+        overlays = [ (final: prev: {
+          pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ (pyfinal: pyprev: {
+            asgi-csrf = pyprev.asgi-csrf.overridePythonAttrs (old: {
+              postPatch = (old.postPatch or "") + ''
+                substituteInPlace asgi_csrf.py \
+                  --replace-fail "lambda: None," "ErrorOnWrite(None, None, None).write," \
+                  --replace-fail "FileClass=ErrorOnWrite," ""
+              '';
+              meta = old.meta // { broken = false; };
+            });
+            # unbuilt by hydra while asgi-csrf was broken; this one test streams
+            # a large csv under the query time limit, and is interrupted on a
+            # loaded builder (sqlite "interrupted", a 500): timing, not datasette
+            datasette = pyprev.datasette.overridePythonAttrs (old: {
+              disabledTests = (old.disabledTests or [ ]) ++ [ "test_max_csv_mb" ];
+            });
+          }) ];
+        }) ];
+      };
+    in {
+      default = self.devShells.${system}.dsh-overlay;
+      #default = import ./shell.nix {pkgs = pkgs-nix; flake = self;};
+      dsh-overlay = mkDevShell {
+        pkgs = pkgs-ovl;
+        # "spine-<<spine_version>> base dev shell, ldc-<<ldc_version>>, dub-<<dub_version>> - dtools-<<dtools_version>>";
+        name = "spine-${version} dev shell (overlay ldc)";
+        compiler = pkgs-ovl.ldc;
+      };
+      dsh-nixpkgs-dmd = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell (nixpkgs dmd)";
+        # nixpkgs dmd: revert to GCC14 (same nullptr/ImportC fix as overlay dmd)
+        compiler = pkgs-nix.dmd.override { stdenv = pkgs-nix.gcc14Stdenv; };
+        #compiler = pkgs-nix.dmd;
+      };
+      dsh-nixpkgs-ldc = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell (nixpkgs ldc)";
+        compiler = pkgs-nix.ldc;
+      };
+      dsh-overlay-dmd = mkDevShell {
+        pkgs = pkgs-ovl;
+        # "spine-<<spine_version>> base dev shell, dmd-<<dmd_version>>, dub-<<dub_version>> - dtools-<<dtools_version>>";
+        name = "spine-${version} dev shell (overlay dmd)";
+        compiler = pkgs-ovl.dmd;
+      };
+      dsh-overlay-ldc = mkDevShell {
+        pkgs = pkgs-ovl;
+        # "spine-<<spine_version>> base dev shell, ldc-<<ldc_version>>, dub-<<dub_version>> - dtools-<<dtools_version>>";
+        name = "spine-${version} dev shell (overlay ldc)";
+        compiler = pkgs-ovl.ldc;
+      };
+      dsh-epub = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell for epub output";
+        compiler = pkgs-nix.ldc;
+        extraPackages = with pkgs-nix; [
+          libxml2
+          html-tidy
+          xmlstarlet
+          epubcheck
+          ebook_tools
+          epr
+          sigil
+          calibre #(suite includes: ebook-viewer)
+          koreader
+          foliate
+        ];
+      };
+      dsh-html = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell for html output";
+        compiler = pkgs-nix.ldc;
+        extraPackages = with pkgs-nix; [
+          # ❯❯ text-mode web browsers
+          elinks
+          #links2
+          lynx
+          w3m
+          # ❯❯ light graphical
+          #dillo
+        ];
+      };
+      dsh-odf = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell for epub output";
+        compiler = pkgs-nix.ldc;
+        extraPackages = with pkgs-nix; [
+          jing
+        ];
+      };
+      dsh-typst-pdf = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell for latex & pdf output";
+        compiler = pkgs-nix.ldc;
+        extraPackages = with pkgs-nix; [
+          typst
+          #typstyle
+          #typst-live
+          #typstwriter
+        ];
+      };
+      dsh-latex-pdf = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell for latex & pdf output";
+        compiler = pkgs-nix.ldc;
+        extraPackages = with pkgs-nix; [
+          source-sans-pro
+          source-serif-pro
+          source-code-pro
+          texliveFull
+        ];
+      };
+      dsh-sqlite = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell for sqlite3 output";
+        compiler = pkgs-nix.ldc;
+        extraPackages = with pkgs-nix; [
+          #sqlite
+          sqlite-interactive
+          litecli
+          visidata
+          sqlitebrowser
+          letos
+          dbeaver-bin
+        ];
+      };
+      dsh-translations = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell for internationalization, po4a";
+        compiler = pkgs-nix.ldc;
+        extraPackages = with pkgs-nix; [
+          po4a
+          gettext
+        ];
+      };
+      dsh-production = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell: production run and its checks";
+        compiler = pkgs-nix.ldc;
+        extraPackages = with pkgs-nix; [
+          # the suite shells out to these two; without them run-tests.sh
+          # re-execs itself into a nix shell that has them
+          sqlite-interactive   # readline cli; base sqlite has none
+          pkgs-stable.datasette # serves an .ocda.db as a browsable, queryable web page with a json api, one command, no configuration
+          epubcheck
+          unzip                # containers: every one spine writes is a zip
+          # xml and html by eye and by check
+          libxml2              # xmllint
+          xmlstarlet           # xml sel -t -v
+          html-tidy            # the scroll html, which epubcheck never sees
+          # pdf
+          poppler-utils        # pdfinfo, pdftotext
+          typst
+          #texliveFull          # NOT texlive.combined.scheme-full: deprecated,
+          file                 # what a file is
+        ];
+      };
+      dsh-production-post = mkDevShell {
+        pkgs = pkgs-nix;
+        name = "spine-${version} dev shell: production run and its checks";
+        compiler = pkgs-nix.ldc;
+        extraPackages = with pkgs-nix; [
+          # the suite shells out to these two; without them run-tests.sh
+          # re-execs itself into a nix shell that has them
+          sqlite-interactive   # readline cli; base sqlite has none
+          pkgs-stable.datasette # serves an .ocda.db as a browsable, queryable web page with a json api, one command, no configuration
+          letos
+          epubcheck
+          pkgs-stable.diffoscope           # diffs two epubs, two odts or two pdfs by content rather (than as bytes) (useful when output moves and cmp only says "differ")
+          unzip                # containers: every one spine writes is a zip
+          # xml and html by eye and by check
+          libxml2              # xmllint
+          xmlstarlet           # xml sel -t -v
+          html-tidy            # the scroll html, which epubcheck never sees
+          xmlstarlet
+          ebook_tools
+          #jing
+          # pdf
+          poppler-utils        # pdfinfo, pdftotext
+          typst
+          #texliveFull          # NOT texlive.combined.scheme-full: deprecated,
+          file                 # what a file is
+          coreutils            # sha256sum and numfmt guaranteed, for checking a digests.txt line by hand
+          po4a
+          gettext
+        ];
+      };
+    });
+  };
+}
+#+END_SRC
+
+*** reinstate this flake.nix once datasette builds again without intervention
+
+#+HEADER: :noweb yes
+#+BEGIN_SRC nix
+{
+  description = "a sisu like parser & document generator";
+  inputs.nixpkgs.url = "<<nixpkgs_unstable>>";
+  #inputs.nixpkgs-stable.url = "<<nixpkgs_stable>>";
   outputs = {
     self,
     nixpkgs,
@@ -306,7 +684,7 @@
           # the suite shells out to these two; without them run-tests.sh
           # re-execs itself into a nix shell that has them
           sqlite-interactive   # readline cli; base sqlite has none
-          #datasette            # serves an .ocda.db as a browsable, queryable web page with a json api, one command, no configuration
+          # datasette            # serves an .ocda.db as a browsable, queryable web page with a json api, one command, no configuration
           letos
           epubcheck
           diffoscope           # diffs two epubs, two odts or two pdfs by content rather (than as bytes) (useful when output moves and cmp only says "differ")
@@ -1073,6 +1451,7 @@ postInstall:
 #+END_SRC
 
 * nixpkgs source
+** unstable
 
 #+NAME: nixpkgs_unstable
 #+HEADER: :noweb yes
@@ -1080,4 +1459,12 @@ postInstall:
 <<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_unstable()>>
 #+END_SRC
 
+** stable
+
+#+NAME: nixpkgs_stable
+#+HEADER: :noweb yes
+#+BEGIN_SRC emacs-lisp
+<<./sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org:nixpkgs_stable()>>
+#+END_SRC
+
 * __END__
diff --git a/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org b/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org
index 59936d2..347779d 100644
--- a/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org
+++ b/org/sisudoc_spine_version_info_and_doc_header_including_copyright_and_license.org
@@ -978,6 +978,8 @@ git+file:///home/ralph/grotto/repo/git.repo/projects/doc-reform/packages/nix-fla
 - github:NixOS/nixpkgs/nixpkgs-unstable
 - https://channels.nixos.org/nixpkgs-unstable/nixexprs.tar.zst
 
+** nipkgs unstable
+
 #+NAME: nixpkgs_unstable
 #+BEGIN_SRC org
 https://channels.nixos.org/nixpkgs-unstable/nixexprs.tar.zst
@@ -986,3 +988,10 @@ https://channels.nixos.org/nixpkgs-unstable/nixexprs.tar.zst
 #+BEGIN_SRC org
 github:NixOS/nixpkgs/nixpkgs-unstable
 #+END_SRC
+
+** nipkgs stable
+
+#+NAME: nixpkgs_stable
+#+BEGIN_SRC org
+https://channels.nixos.org/nixos-26.05/nixexprs.tar.zst
+#+END_SRC