puck/zilch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

(zilch nixpkgs): add dataflow tracking support

Browse source 
Dataflow tracking runs the installPhase of a derivation, and then
does its best to track build input -> store path output pathways.
This allows for substituting the installPhase of a derivation as much
as possible with relatively quick store path logic, allowing for cross-
project build input tracking. 

Change-Id: I6a6a69642530b32edbc2c521a3f584703731b6e1
This commit is contained in:
puck 2025-07-26 15:42:17 +00:00
parent 612ee6fe83
commit feeb14eed5
2 changed files with 122 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

1
core/src/file.sld
View file

@ -6,6 +6,7 @@
(chicken base) (chicken format)
(srfi 128) (srfi 132) (srfi 146) (srfi 151))
(export zfile zsymlink zdir
z-file? z-directory? z-symlink?
zfile->store)
(begin

123
lang/ninja/src/nixpkgs.sld
View file

@ -5,13 +5,15 @@
(zilch lang ninja) (zilch lang ninja build)
(zilch lang ninja config)
(zilch magic) (zilch nixpkgs) (zilch vfs)
(zilch file)
(zilch nix drv)
(zilch zexpr)
(srfi 132) (srfi 146) (srfi 152))
(srfi 128) (srfi 132) (srfi 146) (srfi 152))
(export
setup-ninja-environment
build-nixpkgs-drv-reproducibly)
build-nixpkgs-drv-reproducibly
determine-data-flow)
(begin
(define coreutils (cdr (assoc "out" (nixpkgs "coreutils"))))
@ -233,4 +235,119 @@
`(("buildCommand" . ,postbuild-builder))
placeholders)
(map car placeholders))
export-depfile))))
export-depfile))
;; Build the derivation, but with stubbed out header and .so files.
;; This is used to determine the dataflow, to make cross-project incremental
;; builds work.
;; Returns a single SRFI-146 mapping, containing keys of shape `(output . (dir . name))`,
;; and values either a zexpr-y store path, or a pair `(marker . <name of the build output>)`
(define (determine-data-flow conf)
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf))
(define edges (built-edge-phony-inputs (cdr (edge-ref "all"))))
(define fix-placeholders-command "")
(define copy-in-place-command "")
; Append the necessary commands to fix up the fake store paths post-install but pre-everything-else.
(for-each
(lambda (plc)
(set! fix-placeholders-command
(string-append fix-placeholders-command
"zilchFixPlaceholder \"" (cdr plc) "\" \"$" (car plc) "\"\n"))
(set! copy-in-place-command
(string-append copy-in-place-command
"cp -rf ../out" (cdr plc) " \"$" (car plc) "\" || mkdir \"$" (car plc) "\"\n")))
placeholders)
; Turn the original `src` and `build` into a known store path.
(define realised-store (store-path-devirtualise configured-drv))
(define make-all-placeholder-files
(string-join (map (lambda (v) (string-append "zilchMakeFile \"" v "\"\n")) edges) "\n"))
; Prepare the post-build builder. This puts everything in its place and runs the post-build phases from the original .drv.
(define postbuild-builder
#~,(string-append
"zilchMakeFile() {\n"
"mkdir -p bdir/build/$(dirname \"$1\")\n"
"rm bdir/build/\"$1\" || true\n"
"echo \"ZILCH MARKER FILE ->$1\" > bdir/build/\"$1\"\n"
"}\n"
"zilchPlace() {\n"
"cd $NIX_BUILD_TOP; cp -rf --no-preserve=ownership " #$realised-store " bdir\n"
"chmod ugo+rw -R bdir\n"
"(cd " #$realised-store "/src; find . -type f '(' -name '*.h' -o -name '*.hh' -o -name '*.so' ')') | while read f; do zilchMakeFile \"../src/$f\"; done\n"
make-all-placeholder-files
"cd bdir/build\n"
"}\n"
"mesonBuildDir=$NIX_BUILD_TOP/bdir/build cmakeBuildDir=$NIX_BUILD_TOP/bdir/build cmakeDir=\"$NIX_BUILD_TOP/bdir/src/${cmakeDir:-.}\"\n"
"zilchFixPlaceholder() {\n"
" find ../out -type f -exec sed -i -e \"s|$1|$2|g\" \"{}\" \";\" || exit 1\n"
" find \"../out\" -type l | while read link; do\n"
" target=\"$(readlink \"$link\")\"; rewritten=\"$(printf \"%s\" \"$target\" | sed -e \"s|$1|$2|g\")\"\n"
" rm \"$link\" && ln -s \"$rewritten\" \"$link\" || exit 1\n"
" done\n"
"}\n"
"zilchFixup() {\n"
fix-placeholders-command
copy-in-place-command
"}\n"
"zilchMesonInstall() {\n"
" runHook preInstall\n"
" local flagsArray=()\n"
" if [[ -n \"$mesonInstallTags\" ]]; then\n"
" flagsArray+=(\"--tags\" \"$(concatStringsSep \",\" mesonInstallTags)\")\n"
" fi\n"
" concatTo flagsArray mesonInstallFlags mesonInstallFlagsArray\n"
" DESTDIR=$NIX_BUILD_TOP/bdir/out meson install --no-rebuild \"${flagsArray[@]}\"\n"
" zilchFixup\n"
" runHook postInstall\n"
"}\n"
"zilchCmakeInstall() {\n"
" runHook preInstall\n"
" DESTDIR=$NIX_BUILD_TOP/bdir/out cmake --install .\n"
" zilchFixup\n"
" runHook postInstall\n"
"}\n"
"if [[ $(type -t mesonInstallPhase) == function ]]; then\n"
" installPhase=zilchMesonInstall; checkPhase=mesonCheckPhase\n"
"else\n"
" installPhase=zilchCmakeInstall; checkPhase=ninjaCheckPhase\n"
"fi\n"
"phases=\"zilchPlace checkPhase ${preInstallPhases[*]:-} installPhase ${preFixupPhases[*]:-} fixupPhase installCheckPhase ${preDistPhases[*]:-} distPhase ${postPhases[*]:-}\"\n"
"for curPhase in ${phases[*]}; do runPhase \"$curPhase\"; done\n"))
; Patch the original .drv to run the postbuild-builder command.
(define patched-drv
(patch-drv initial-drv
(append
`(("buildCommand" . ,postbuild-builder))
placeholders)
(map car placeholders)))
(define (get-file-marker fptr)
(call-with-port (store-path-open fptr)
(lambda (p)
(define header (read-string 20 p))
(and (string=? header "ZILCH MARKER FILE ->")
(let ((str (read-string 99999 p)))
(string-copy str 0 (- (string-length str) 1)))))))
(define output (mapping (make-default-comparator)))
(define (process-output name-data-pair)
(define name (car name-data-pair))
(define store-path (cdr name-data-pair))
(define vfs (vfs-from-store store-path))
(mapping-for-each
(lambda (path val)
; TODO(puck): this depends on vfs-from-store internals
(define is-file (z-file? val))
(define marker (and is-file (get-file-marker val)))
(if marker
(set! output (mapping-set! output (cons name path) (cons 'marker marker)))
(set! output (mapping-set! output (cons name path) val))))
(vfs-contents vfs)))
(for-each process-output patched-drv)
output)))
; TODO(puck): for each output, do the necessary dance of figuring out where it came from. read first N bytes, compare, then do the thing. output a big alist and do the dataflow dance?