puck/zilch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

(zilch lang ninja): Support rewriting derivation inputs

Browse source 
This lets incremental Ninja builds depend on other incremental Ninja
builds.

Change-Id: I6a6a6964ef300cae2e180970048c8a4881c88e19
This commit is contained in:
puck 2025-07-26 15:42:17 +00:00
parent c1c606ee0b
commit 0023f3def8
4 changed files with 267 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

17
cli/zilch-ninja.scm
View file

@ -58,24 +58,29 @@
(when (and (ninja-build-config-depfile-path config) (file-exists? (ninja-build-config-depfile-path config)))
(set-ninja-build-config-depfile! config (alist->mapping (make-default-comparator) (call-with-input-file (ninja-build-config-depfile-path config) read))))
(define (export-depfiles finalized-drv)
(define config (finalized-drv-config finalized-drv))
(when (ninja-build-config-depfile-path config)
(call-with-output-file (ninja-build-config-depfile-path config) (lambda (p) (write (mapping->alist (finalized-drv-depfile finalized-drv)) p))))
(for-each (lambda (v) (export-depfiles (force (drv-rewrite-finalized-drv v)))) (finalized-drv-rewritten-drvs finalized-drv)))
(cond
((string=? (car args) "source")
(let*-values
(((_ configured-drv _ _ _ _) (setup-ninja-environment config))
(((_ configured-drv _ _ _ _ _) (setup-ninja-environment config '()))
((realised) (store-path-realised configured-drv))
((path) (if (null? (cdr args)) "src" (cadr args))))
(system* (string-append "cp -rf --no-preserve=ownership " realised "/src " (qs path)))
(system* (string-append "chmod -R u+rw " (qs path)))))
((string=? (car args) "build")
(if (null? (cdr args))
(let-values (((built export-depfile) (build-nixpkgs-drv-reproducibly config)))
(let ((output (build-nixpkgs-drv-reproducibly config)))
(for-each
(lambda (output-and-path)
(store-path-realised (cdr output-and-path))
(printf "~A\t-> ~S\n" (car output-and-path) (cdr output-and-path)))
built)
(when (ninja-build-config-depfile-path config)
(call-with-output-file (ninja-build-config-depfile-path config) (lambda (p) (write (mapping->alist (export-depfile)) p)))))
(finalized-drv-output-store-paths output))
(export-depfiles output))
(let-values (((_ _ _ edge-ref defaults _) (setup-ninja-environment config)))
(for-each
(lambda (target)
@ -84,7 +89,7 @@
(cdr args)))))
((string=? (car args) "diff")
(let*-values
(((_ configured-drv _ _ _ _) (setup-ninja-environment config))
(((_ configured-drv _ _ _ _ _) (setup-ninja-environment config '()))
((realised) (store-path-realised configured-drv))
((path) (or source "src")))
(process-execute "git" (list "diff" "--no-index" "--" (string-append realised "/src") path))))

80
lang/ninja/src/build.sld
View file

@ -6,7 +6,8 @@
(zilch nix drv) (zilch nix path)
(zilch nixpkgs) (zilch zexpr) (zilch vfs)
(chicken format)
(srfi 128) (srfi 146) (srfi 152)
(srfi 26) (srfi 128) (srfi 146) (srfi 152) (srfi 207)
(zilch lib rewrite)
(zilch lang ninja) (zilch lang ninja config) (zilch lang ninja depfile))
(export
@ -21,6 +22,56 @@
(define patchelf (cdr (assoc "out" (nixpkgs "patchelf"))))
(define llvm-bintools (cdr (assoc "out" (nixpkgs-eval "llvmPackages_latest.bintools-unwrapped"))))
(define (replace-input drv input-drv new-outputs)
(when (store-path? drv) (set! drv (store-path-drv drv)))
(define output-names (map car (derivation-outputs drv)))
(define new-env (list))
(define input-drv-pathhash (string-copy (derivation-path input-drv) (+ 1 (string-length "/nix/store")) (+ 33 (string-length "/nix/store"))))
(define input-drv-name (derivation-name input-drv))
; For each output of the to-remove .drv, make up to two rewrites:
; the placeholder (/[base32 hex]) -> the corresponding output in new-outputs
; the old output path -> the corresponding output in new-outputs
(define all-rewrites '())
(for-each
(lambda (output-names)
(define new-path (string->utf8 (cdr (assoc (car output-names) new-outputs))))
(define is-floating (eq? (derivation-output-hash (cdr output-names)) 'floating))
(unless is-floating
(set! all-rewrites
(cons
(cons (string->utf8 (derivation-output-path (cdr output-names))) new-path)
all-rewrites)))
(set! all-rewrites
(cons
(cons (string->utf8 (make-upstream-output-placeholder input-drv-pathhash input-drv-name (car output-names))) new-path)
all-rewrites)))
(derivation-outputs input-drv))
(for-each
(lambda (kv)
(cond
; TODO(puck): don't strip (dis)allowed(References/Requisites)?
((member (car kv) '("allowedReferences" "disallowedReferences" "allowedRequisites" "disallowedRequisites") string=?)
#f)
((member (car kv) output-names string=?)
#f)
(else
(set! new-env (cons (cons (car kv) (rewrite-bytevector-or-string (cdr kv) all-rewrites)) new-env)))))
(derivation-env drv))
(define is-ca-derivation (symbol? (derivation-output-hash (cdar (derivation-outputs drv)))))
(define out-drv
((if is-ca-derivation make-ca-derivation make-input-addressed-derivation) (derivation-name drv)
(derivation-system drv)
(derivation-input-drvs drv)
(derivation-input-src drv)
(cons (rewrite-bytevector-or-string (derivation-builder drv) all-rewrites) (map (lambda (v) (rewrite-bytevector-or-string v all-rewrites)) (derivation-args drv)))
new-env
(map car (derivation-outputs drv))))
(map (lambda (v) (make-store-path out-drv (car v) #f)) (derivation-outputs out-drv)))
;; Represents a single built `<build-edge>`.
;;
;; - `edge`: The `<build-edge>` that this `<built-edge>` represents.
@ -39,6 +90,16 @@
(lib-placeholder built-edge-lib-placeholder)
(phony-inputs built-edge-phony-inputs))
;; Represents the entire build environment.
;;
;; - `config`:
;; - `vfs`: The base file system, containing all files that are unchanging. Expected to be mounted at `/build/bdir`.
;; - `header-files`: A list of all "header files". When an edge has a depfile, it is built assuming all files in this list that said edge uses are in this list.
;; (string list)
;; - `build-dir`: The build dir, relative to the root of the vfs.
;; - `parsed-depfiles`: A mapping of (primary) edge output to the list of depnedencies in the depfile.
;; - `collected-deps`: A rebuild of `parsed-depfiles`, filled in as edges are realised.
;; - `secondary-vfs-setup`: A shell script (with $COREUTILS) that sets up any secondary VFS paths.
(define-record-type <build-env>
(make-build-env config vfs header-files build-dir parsed-depfiles collected-deps secondary-vfs-setup)
build-env?
@ -78,6 +139,9 @@
"zilch-ninja"
(string-map (lambda (c) (if (is-valid-store-path-char c) c #\-)) (if (> (string-length str) 128) (string-copy str 0 128) str))))
; `edges`: mapping of relative-path to (input-store-path . <promise of <built-edge>>)
; where input-store-path is 'base (part of base vfs), 'phony (follow built-edge-phony-inputs), or a store path.
; depfile-data is a list of all dependency file based entries; and should be appended to the list of inputs.
(define (inner-derivation-for-edge env edges current-edge resolved depfile-data)
(when (build-rule-rspfile resolved) (error "rspfile not yet supported" current-edge))
(define copy-input-files "")
@ -97,9 +161,8 @@
(define input-edge (and input (force (cdr input))))
(cond
; if input-file is 'base, this is part of the base vfs; we don't filter that right now.
; if input-file is 'base, this is part of the base vfs; we don't filter that.
((eq? input-file 'base) #f)
((and (pair? input-file) (eq? (car input-file) 'base-vfs)) #f)
; Phony rule; pass through the inputs literally.
((eq? input-file 'phony) (for-each append-file (built-edge-phony-inputs input-edge)))
@ -258,6 +321,11 @@
;; Returns a derivation that runs the command for this edge,
;; inside a Nix derivation with the correct inputs.
;;
;; This differentiates from inner-derivation-for-edge by handling the two possible instantiations of the derivation.
;; When a build fails because of header files, it will rerun with the full set of header files.
;;
;; TODO(puck): "header files" is a bad name for this concept?
(define (derivation-for-edge env edges current-edge)
(define resolved (build-edge-resolved current-edge))
(define result #f)
@ -346,7 +414,7 @@
;; handle this Ninja file.
;; - `relative-to`: The directory in which the Ninja file was found. Used to
;; resolve relative references in the Ninja file.
;; - `secondary-roots`: A list of (store-path-map finalized base-vfs-store-path-map (overlay-path-0 . promise-to-store-path) ... . (overlay-path-n . promise-to-store-path)) alists. i _guess_.
;; - `secondary-roots`: A list of <drv-rewrite> records.
(define (process-ninja-file file conf relative-to secondary-roots)
(unless (or (string=? relative-to "") (string-suffix? "/" relative-to)) (set! relative-to (string-append relative-to "/")))
@ -356,7 +424,7 @@
#~,(string-append
"$COREUTILS/cp -rf --no-preserve=ownership " #$v " " k "; "
"$COREUTILS/chmod ugo+rw -R " k "; "))
(define all-rewrites (apply append (map (lambda (v) (mapping-map->list setup-secondary-vfs (car (cddr v)))) secondary-roots)))
(define all-rewrites (apply append (map (lambda (v) (mapping-map->list setup-secondary-vfs (drv-rewrite-vfs-bases v))) secondary-roots)))
(define secondary-vfs-setup #~,(apply string-append #$all-rewrites))
(define path-to-vfs
@ -451,7 +519,7 @@
(lambda (path-promise-pair)
(set! edges (mapping-set! edges (car path-promise-pair) (cons (cdr path-promise-pair) #f)))
(set-build-env-header-files! env (cons (car path-promise-pair) (build-env-header-files env))))
(cdr (cddr secondary-root))))
(drv-rewrite-overlay-paths secondary-root)))
secondary-roots)
; Finish VFS filtering by taking out everything under build/meson-private.

34
lang/ninja/src/config.sld
View file

@ -11,17 +11,22 @@
ninja-build-config-environment ninja-build-config-environment-drv
ninja-build-config-root-dir ninja-build-config-patches ninja-build-config-targets
ninja-build-config-override-source ninja-build-config-depfile ninja-build-config-depfile-path
ninja-build-config-disallow-elide
ninja-build-config-disallow-elide ninja-build-config-rewrites
set-ninja-build-config-root-dir! set-ninja-build-config-environment! set-ninja-build-config-depfile!
<drv-rewrite>
make-drv-rewrite drv-rewrite?
drv-rewrite-source-paths drv-rewrite-finalized-drv
drv-rewrite-vfs-bases drv-rewrite-overlay-paths
parse-ninja-config)
(begin
;; Represents a parsed Ninja build configuration.
;; See `parse-ninja-config` for the definition of these fields.
(define-record-type <ninja-build-config>
(make-ninja-build-config environment environment-drv root-dir patches targets override-source depfile depfile-path disallow-elide)
(make-ninja-build-config environment environment-drv root-dir patches targets override-source depfile depfile-path disallow-elide rewrites)
ninja-build-config?
(environment ninja-build-config-environment set-ninja-build-config-environment!)
(environment-drv ninja-build-config-environment-drv set-ninja-build-config-environment-drv!)
@ -31,7 +36,8 @@
(override-source ninja-build-config-override-source set-ninja-build-config-override-source!)
(depfile ninja-build-config-depfile set-ninja-build-config-depfile!)
(depfile-path ninja-build-config-depfile-path set-ninja-build-config-depfile-path!)
(disallow-elide ninja-build-config-disallow-elide set-ninja-build-config-disallow-elide!))
(disallow-elide ninja-build-config-disallow-elide set-ninja-build-config-disallow-elide!)
(rewrites ninja-build-config-rewrites set-ninja-build-config-rewrites!))
(define (parse-config-inner conf data)
(cond
@ -88,6 +94,11 @@
(list-val (if (list? val) val (list val))))
(set-ninja-build-config-targets! conf (append list-val (ninja-build-config-targets conf))))
(parse-config-inner conf (cddr data)))
((#:rewrite)
(let*
((val (list-ref data 1)))
(set-ninja-build-config-rewrites! conf (cons (cons (car val) (parse-ninja-config (cdr val))) (ninja-build-config-rewrites conf))))
(parse-config-inner conf (cddr data)))
(else (error (string-append "Unknown directive " (keyword->string (car data)) " parsing Zilch Ninja config")))))))
;; Parses a Zilch Ninja configuration file.
@ -118,7 +129,22 @@
;; incorrect.
;; - `target: "foo"`/`targets: '("foo" "bar")`: Build these targets instead
;; of the default specified in the Ninja file.
;; - `rewrite: ("<derivation name>" . <nested config>): A nested Zilch
;; Ninja configuration to swap out the input derivation with equivalent
;; name.
(define (parse-ninja-config config)
(unless (list? config)
(error "expected Zilch Ninja config to be a list"))
(parse-config-inner (make-ninja-build-config #f #f #f '() #f #f #f #f #f) config))))
(parse-config-inner (make-ninja-build-config #f #f #f '() #f #f #f #f #f '()) config))
;; `source-paths`: mapping of original store path to virtual path.
;; `finalized-drv`: promise of a <finalized-drv>
;; `vfs-bases`: mapping of virtualised path to base VFS
;; `overlay-paths`: alist of virtualised path to promise of a store path
(define-record-type <drv-rewrite>
(make-drv-rewrite source-paths finalized-drv vfs-bases overlay-paths)
drv-rewrite?
(source-paths drv-rewrite-source-paths)
(finalized-drv drv-rewrite-finalized-drv)
(vfs-bases drv-rewrite-vfs-bases)
(overlay-paths drv-rewrite-overlay-paths))))

167
lang/ninja/src/nixpkgs.sld
View file

@ -6,18 +6,44 @@
(zilch lang ninja config)
(zilch magic) (zilch nixpkgs) (zilch vfs)
(zilch file)
(zilch nix drv)
(zilch nix drv) (zilch nix hash)
(zilch lib hash) (zilch lib rewrite)
(zilch zexpr)
(srfi 128) (srfi 132) (srfi 146) (srfi 152))
(srfi 26) (srfi 128) (srfi 132) (srfi 146) (srfi 152) (srfi 207))
(export
setup-ninja-environment
build-nixpkgs-drv-reproducibly
determine-data-flow)
determine-data-flow
virtual-path-for make-virtual-path
<finalized-drv> finalized-drv?
finalized-drv-output-store-paths finalized-drv-depfile
finalized-drv-rewritten-drvs finalized-drv-config)
(begin
(define coreutils (cdr (assoc "out" (nixpkgs "coreutils"))))
(define rewrite-hooks
(string-append
"__zilch_rewrite_sed=\"\"\n"
"echo \"$__zilch_rewrites\" | while read -d ' ' rewrite_from; do\n"
" read -d ' ' rewrite_to\n"
" __zilch_rewrite_sed=\"$__zilch_rewrite_sed s|$rewrite_from|$rewrite_to|g\"\n"
"done || true\n"
"echo \"$__zilch_rewrites\" | while read -d ' ' rewrite_from; do\n"
" read -d ' ' rewrite_to\n"
" cp --no-preserve=ownership -rf \"$rewrite_from\" \"$rewrite_to\"\n"
" chmod -R ugo+rw \"$rewrite_to\"\n"
" find \"$rewrite_to\" -type f -exec sed -i -e \"$__zilch_rewrite_sed\" \"{}\" \";\" || exit 1\n"
" find \"$rewrite_to\" -type l | while read link; do\n"
" target=\"$(readlink \"$link\")\"; rewritten=\"$(printf \"%s\" \"$target\" | sed -e \"$__zilch_rewrite_sed\")\"\n"
" rm \"$link\" && ln -s \"$rewritten\" \"$link\" || exit 1\n"
" done\n"
"done || true\n"
"unset __zilch_rewrites __zilch_rewrite_sed\n"))
; Shellcode to run instead of the default stdenv genericBuild();
; This takes the source and configuration of our derivation and prepares
; a proper environment to build inside.
@ -38,7 +64,7 @@
; Patch the passed in .drv by appending to the environment
; and changing the list of outputs.
(define (patch-drv drv append-env outputs)
(define (patch-drv drv append-env outputs rewrites)
(define ctx (zexp-unwrap (zexp (zexp-unquote append-env))))
(define new-env (list))
(for-each
@ -47,7 +73,7 @@
(cond
(mem (set! new-env (cons mem new-env)))
((member (car kv) '("allowedReferences" "disallowedReferences" "allowedRequisites" "disallowedRequisites")) #f)
(else (set! new-env (cons kv new-env)))))
(else (set! new-env (cons (cons (car kv) (rewrite-bytevector-or-string (cdr kv) rewrites)) new-env)))))
(derivation-env drv))
(for-each
(lambda (kv)
@ -71,6 +97,23 @@
(define base-placeholder "zilchplaceholderdonotuseanywhere-")
;; Generate a placeholder for a virtualised path.
;; This is similar to `make-placeholder`, but outputs
;; a string that is of the shape `/nix/store/{invalid hash}-{discriminator}`,
;; where the hash starts with `tz`, to make it invalid base32.
(define (make-virtual-path discriminator)
(define data (string-append "zilch!" discriminator))
(define hash (as-base32 (hash-compress (sha256 (string->utf8 data)))))
(string-set! hash 0 #\t)
(string-set! hash 1 #\z)
(string-append "/nix/store/" hash "-" discriminator))
;; Generate a placeholder for a virtualised path, like `make-virtual-path`,
;; but based on the input store path string.
(define (virtual-path-for store-path)
(define index (string-contains store-path "-"))
(make-virtual-path (string-copy store-path (+ 1 index))))
; Create a placeholder store path that is unique
(define (make-fake-store-path name output)
(when (> (string-length output) 32)
@ -78,6 +121,17 @@
(string-append "/nix/store/" output (string-copy base-placeholder (string-length output)) name (if (string=? output "out") "" (string-append "-" output))))
(define-record-type <finalized-drv>
(make-finalized-drv output-store-paths export-depfile rewritten-drvs config)
finalized-drv?
(output-store-paths finalized-drv-output-store-paths)
(export-depfile finalized-drv-export-depfile)
(rewritten-drvs finalized-drv-rewritten-drvs)
(config finalized-drv-config))
(define (finalized-drv-depfile drv)
((finalized-drv-export-depfile drv)))
;; Takes a `<ninja-build-config>` representing a Nixpkgs derivation, and
;; preprocesses the derivation such that it can be reconstituted once Zilch
;; has taken over the Ninja build requirements.
@ -86,7 +140,7 @@
;; encodes many specific parts that are unlikely to be useful by external
;; parties.
;;
;; Returns 6 values:
;; Returns 7 values:
;;
;; - The initial Nixpkgs derivation as `<derivation>`
;; - The Nixpkgs derivation after running all phases up to and including
@ -94,7 +148,8 @@
;; - An alist of output names to placeholder store paths
;; - The `edge-ref`, `defaults`, and `export-depfile` values from calling
;; `process-ninja-file`
(define (setup-ninja-environment conf)
;; - A list of store path -> store path rewrites.
(define (setup-ninja-environment conf secondary-roots)
(define initial-drv (ninja-build-config-environment-drv conf))
(when (store-path? initial-drv)
(set! initial-drv (store-path-drv initial-drv)))
@ -103,17 +158,31 @@
(define existing-env (ninja-build-config-environment conf))
(define rewrites (list))
(define rewrite-extra "")
(for-each
(lambda (secondary-root)
(mapping-for-each
(lambda (orig-path virtual-path)
(set! rewrites (cons (cons (string->utf8 orig-path) (string->utf8 virtual-path)) rewrites))
(set! rewrite-extra (string-append rewrite-extra orig-path " " virtual-path "\n")))
(drv-rewrite-source-paths secondary-root)))
secondary-roots)
; Override output environment variables with our placeholders.
(set-ninja-build-config-environment! conf #~,(map (lambda (v) (or (assoc (car v) placeholders) v)) #$existing-env))
(set-ninja-build-config-environment! conf #~,(map (lambda (v) (or (assoc (car v) placeholders) (cons (car v) (rewrite-bytevector-or-string (cdr v) rewrites)))) #$existing-env))
; Take the initially requested .drv, replace its buildCommand, and set a single ("zilch_out") output path.
(define configured-drv
(cdar
(patch-drv initial-drv
(append
`(("buildCommand" . ,configure-builder))
`(("buildCommand" . ,configure-builder)
("addInputsHook" . ,rewrite-hooks)
("__zilch_rewrites" . ,rewrite-extra))
placeholders)
'("zilch_out"))))
'("zilch_out")
rewrites)))
; This VFS contains two directories: `src` (source tree) and `build` (Ninja build files).
(define configured-vfs (vfs-from-store configured-drv))
@ -141,10 +210,49 @@
(read-ninja-file (read-file-at-path "build.ninja") read-file-at-path))
; Process the build.ninja file.
(define-values (edge-ref defaults export-depfile) (process-ninja-file ninja-file conf "build"))
(define-values (edge-ref defaults export-depfile) (process-ninja-file ninja-file conf "build" secondary-roots))
(values initial-drv configured-drv placeholders edge-ref defaults export-depfile))
(values initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites))
(define (process-secondary-root parent-conf secondary-root)
(define conf (cdr secondary-root))
(define drv-name (car secondary-root))
(define matching-drv #f)
(let loop ((drv-list (derivation-input-drvs (ninja-build-config-environment-drv parent-conf))))
(cond
((null? drv-list) (error "Cannot find matching derivation for rewrite" drv-name))
((string=? (derivation-name (caar drv-list)) drv-name)
(set! matching-drv (caar drv-list)))
(else (loop (cdr drv-list)))))
(define (disc-name out-name)
(if (string=? out-name "out") drv-name (string-append drv-name "-" out-name)))
(define future-rewrites (map (lambda (a) (cons (derivation-output-path (cdr a)) (make-virtual-path (disc-name (car a))))) (derivation-outputs matching-drv)))
(define output-map (alist->mapping (make-default-comparator) (map (lambda (a) (cons (car a) (make-virtual-path (disc-name (car a))))) (derivation-outputs matching-drv))))
; secondary-root is (name . #<ninja-build-config>)
(define its-secondary-roots (map (cute process-secondary-root conf <>) (ninja-build-config-rewrites conf)))
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites) (setup-ninja-environment conf its-secondary-roots))
(define vfses (mapping-map (lambda (k v) (values k (mapping (make-default-comparator)))) (make-default-comparator) output-map))
(define (transform-obj obj)
(cond
((and (pair? obj) (eq? (car obj) 'marker))
(let ((obj-edge (edge-ref (cdr obj))))
(or (and (cdr obj-edge) (built-edge-lib-placeholder (cdr obj-edge)))
(car obj-edge))))
(else obj)))
(define extra-paths '())
(mapping-for-each
(lambda (key obj)
(define base-store-path (mapping-ref output-map (car key)))
(define full-path (string-append base-store-path "/" (if (string=? "" (cadr key)) (cddr key) (string-append (cadr key) "/" (cddr key)))))
(if (or (string-suffix? ".hh" (cddr key)) (string-suffix? ".h" (cddr key)) (string-suffix? ".hpp" (cddr key)))
(set! extra-paths (cons (cons full-path (transform-obj obj)) extra-paths))
(set! vfses (mapping-set! vfses (car key) (mapping-set! (mapping-ref vfses (car key)) (cdr key) (transform-obj obj))))))
(determine-data-flow conf))
(set! vfses (mapping-map/monotone! (lambda (k v) (values k (vfs-to-store (make-vfs v)))) (make-default-comparator) vfses))
(define finalized (delay (finalize-drv conf its-secondary-roots initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites)))
(make-drv-rewrite output-map finalized vfses extra-paths))
;; Takes a `<ninja-build-config>` representing a Nixpkgs derivation, and
;; build it using Zilch.
;;
@ -156,8 +264,12 @@
;; necessary inputs generated from the depfile data. This can be stored in
;; a file for later rebuilds.
(define (build-nixpkgs-drv-reproducibly conf)
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf))
(define secondary-roots (map (cute process-secondary-root conf <>) (ninja-build-config-rewrites conf)))
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites) (setup-ninja-environment conf secondary-roots))
(finalize-drv conf secondary-roots initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites))
;; "Finalizes" a previously setup Ninja environment, running the install and fixup hooks.
(define (finalize-drv conf secondary-roots initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites)
; Build all store paths necessary for installing. This assumes Meson.
(define preinstall-state (force (built-edge-out-drv (cdr (edge-ref "all")))))
@ -182,6 +294,20 @@
; This is necessary because these paths may be CA, and we can't guarantee daemon support for that past this point.
(define realised-built (store-path-devirtualise preinstall-state))
; Take each output store path, then copy it over to the expected place
; then later run a fixup to the original store path at placeholder state
(define prepared-store-paths (list))
(define rewrite-placeholder-fixes (list))
(define (fixup-single-secondary-root single-drv-item)
(define drv-name (virtual-path-for (store-path-path single-drv-item)))
(set! prepared-store-paths (cons #~,(string-append "cp -rf " #$single-drv-item " " drv-name "\n") prepared-store-paths))
(set! rewrite-placeholder-fixes (cons #~,(string-append "zilchFixPlaceholder " drv-name " " #$single-drv-item "\n") rewrite-placeholder-fixes)))
(define (prepare-secondary-root secondary-root)
; alist of output name -> store path
(define realised (finalized-drv-output-store-paths (force (drv-rewrite-finalized-drv secondary-root))))
(map (lambda (v) (fixup-single-secondary-root (cdr v))) realised))
(for-each prepare-secondary-root secondary-roots)
; Prepare the post-build builder. This puts everything in its place and runs the post-build phases from the original .drv.
(define postbuild-builder
#~,(string-append
@ -201,6 +327,7 @@
"}\n"
"zilchFixup() {\n"
fix-placeholders-command
#$(apply string-append #$rewrite-placeholder-fixes)
copy-in-place-command
"}\n"
"zilchMesonInstall() {\n"
@ -229,13 +356,15 @@
"for curPhase in ${phases[*]}; do runPhase \"$curPhase\"; done\n"))
; Patch the original .drv to run the postbuild-builder command.
(values
(define store-paths
(patch-drv initial-drv
(append
`(("buildCommand" . ,postbuild-builder))
`(("buildCommand" . ,postbuild-builder)
("addInputsHook" . ,#~,(apply string-append #$prepared-store-paths)))
placeholders)
(map car placeholders))
export-depfile))
(map car placeholders)
rewrites))
(make-finalized-drv store-paths export-depfile secondary-roots conf))
;; Build the derivation, but with stubbed out header and .so files.
;; This is used to determine the dataflow, to make cross-project incremental
@ -243,7 +372,7 @@
;; Returns a single SRFI-146 mapping, containing keys of shape `(output . (dir . name))`,
;; and values either a zexpr-y store path, or a pair `(marker . <name of the build output>)`
(define (determine-data-flow conf)
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf))
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites) (setup-ninja-environment conf '()))
(define edges (built-edge-phony-inputs (cdr (edge-ref "all"))))
@ -325,7 +454,7 @@
(append
`(("buildCommand" . ,postbuild-builder))
placeholders)
(map car placeholders)))
(map car placeholders) '()))
(define (get-file-marker fptr)
(call-with-port (store-path-open fptr)
(lambda (p)