diff --git a/core/src/file.sld b/core/src/file.sld index 7212c73..0a83c80 100644 --- a/core/src/file.sld +++ b/core/src/file.sld @@ -5,7 +5,8 @@ (zilch magic) (zilch nix binproto) (zilch nix daemon) (zilch nix drv) (zilch zexpr) (chicken base) (chicken format) (srfi 128) (srfi 132) (srfi 146) (srfi 151)) - (export zfile zsymlink zdir + (export zfile zsymlink zdir + z-file? z-directory? z-symlink? zfile->store) (begin diff --git a/lang/ninja/src/nixpkgs.sld b/lang/ninja/src/nixpkgs.sld index fb28516..5032a2a 100644 --- a/lang/ninja/src/nixpkgs.sld +++ b/lang/ninja/src/nixpkgs.sld @@ -5,13 +5,15 @@ (zilch lang ninja) (zilch lang ninja build) (zilch lang ninja config) (zilch magic) (zilch nixpkgs) (zilch vfs) + (zilch file) (zilch nix drv) (zilch zexpr) - (srfi 132) (srfi 146) (srfi 152)) + (srfi 128) (srfi 132) (srfi 146) (srfi 152)) (export setup-ninja-environment - build-nixpkgs-drv-reproducibly) + build-nixpkgs-drv-reproducibly + determine-data-flow) (begin (define coreutils (cdr (assoc "out" (nixpkgs "coreutils")))) @@ -233,4 +235,119 @@ `(("buildCommand" . ,postbuild-builder)) placeholders) (map car placeholders)) - export-depfile)))) + export-depfile)) + + ;; Build the derivation, but with stubbed out header and .so files. + ;; This is used to determine the dataflow, to make cross-project incremental + ;; builds work. + ;; Returns a single SRFI-146 mapping, containing keys of shape `(output . (dir . name))`, + ;; and values either a zexpr-y store path, or a pair `(marker . )` + (define (determine-data-flow conf) + (define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf)) + + (define edges (built-edge-phony-inputs (cdr (edge-ref "all")))) + + (define fix-placeholders-command "") + (define copy-in-place-command "") + + ; Append the necessary commands to fix up the fake store paths post-install but pre-everything-else. + (for-each + (lambda (plc) + (set! fix-placeholders-command + (string-append fix-placeholders-command + "zilchFixPlaceholder \"" (cdr plc) "\" \"$" (car plc) "\"\n")) + (set! copy-in-place-command + (string-append copy-in-place-command + "cp -rf ../out" (cdr plc) " \"$" (car plc) "\" || mkdir \"$" (car plc) "\"\n"))) + placeholders) + + ; Turn the original `src` and `build` into a known store path. + (define realised-store (store-path-devirtualise configured-drv)) + + (define make-all-placeholder-files + (string-join (map (lambda (v) (string-append "zilchMakeFile \"" v "\"\n")) edges) "\n")) + + ; Prepare the post-build builder. This puts everything in its place and runs the post-build phases from the original .drv. + (define postbuild-builder + #~,(string-append + "zilchMakeFile() {\n" + "mkdir -p bdir/build/$(dirname \"$1\")\n" + "rm bdir/build/\"$1\" || true\n" + "echo \"ZILCH MARKER FILE ->$1\" > bdir/build/\"$1\"\n" + "}\n" + "zilchPlace() {\n" + "cd $NIX_BUILD_TOP; cp -rf --no-preserve=ownership " #$realised-store " bdir\n" + "chmod ugo+rw -R bdir\n" + "(cd " #$realised-store "/src; find . -type f '(' -name '*.h' -o -name '*.hh' -o -name '*.so' ')') | while read f; do zilchMakeFile \"../src/$f\"; done\n" + make-all-placeholder-files + "cd bdir/build\n" + "}\n" + "mesonBuildDir=$NIX_BUILD_TOP/bdir/build cmakeBuildDir=$NIX_BUILD_TOP/bdir/build cmakeDir=\"$NIX_BUILD_TOP/bdir/src/${cmakeDir:-.}\"\n" + "zilchFixPlaceholder() {\n" + " find ../out -type f -exec sed -i -e \"s|$1|$2|g\" \"{}\" \";\" || exit 1\n" + " find \"../out\" -type l | while read link; do\n" + " target=\"$(readlink \"$link\")\"; rewritten=\"$(printf \"%s\" \"$target\" | sed -e \"s|$1|$2|g\")\"\n" + " rm \"$link\" && ln -s \"$rewritten\" \"$link\" || exit 1\n" + " done\n" + "}\n" + "zilchFixup() {\n" + fix-placeholders-command + copy-in-place-command + "}\n" + "zilchMesonInstall() {\n" + " runHook preInstall\n" + " local flagsArray=()\n" + " if [[ -n \"$mesonInstallTags\" ]]; then\n" + " flagsArray+=(\"--tags\" \"$(concatStringsSep \",\" mesonInstallTags)\")\n" + " fi\n" + " concatTo flagsArray mesonInstallFlags mesonInstallFlagsArray\n" + " DESTDIR=$NIX_BUILD_TOP/bdir/out meson install --no-rebuild \"${flagsArray[@]}\"\n" + " zilchFixup\n" + " runHook postInstall\n" + "}\n" + "zilchCmakeInstall() {\n" + " runHook preInstall\n" + " DESTDIR=$NIX_BUILD_TOP/bdir/out cmake --install .\n" + " zilchFixup\n" + " runHook postInstall\n" + "}\n" + "if [[ $(type -t mesonInstallPhase) == function ]]; then\n" + " installPhase=zilchMesonInstall; checkPhase=mesonCheckPhase\n" + "else\n" + " installPhase=zilchCmakeInstall; checkPhase=ninjaCheckPhase\n" + "fi\n" + "phases=\"zilchPlace checkPhase ${preInstallPhases[*]:-} installPhase ${preFixupPhases[*]:-} fixupPhase installCheckPhase ${preDistPhases[*]:-} distPhase ${postPhases[*]:-}\"\n" + "for curPhase in ${phases[*]}; do runPhase \"$curPhase\"; done\n")) + + ; Patch the original .drv to run the postbuild-builder command. + (define patched-drv + (patch-drv initial-drv + (append + `(("buildCommand" . ,postbuild-builder)) + placeholders) + (map car placeholders))) + (define (get-file-marker fptr) + (call-with-port (store-path-open fptr) + (lambda (p) + (define header (read-string 20 p)) + (and (string=? header "ZILCH MARKER FILE ->") + (let ((str (read-string 99999 p))) + (string-copy str 0 (- (string-length str) 1))))))) + (define output (mapping (make-default-comparator))) + (define (process-output name-data-pair) + (define name (car name-data-pair)) + (define store-path (cdr name-data-pair)) + (define vfs (vfs-from-store store-path)) + (mapping-for-each + (lambda (path val) + ; TODO(puck): this depends on vfs-from-store internals + (define is-file (z-file? val)) + (define marker (and is-file (get-file-marker val))) + (if marker + (set! output (mapping-set! output (cons name path) (cons 'marker marker))) + (set! output (mapping-set! output (cons name path) val)))) + (vfs-contents vfs))) + (for-each process-output patched-drv) + output))) + + ; TODO(puck): for each output, do the necessary dance of figuring out where it came from. read first N bytes, compare, then do the thing. output a big alist and do the dataflow dance?