From 0023f3def810a0e1d2e94c2fc7e04137fdb2148f Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Sat, 26 Jul 2025 15:42:17 +0000 Subject: [PATCH] (zilch lang ninja): Support rewriting derivation inputs This lets incremental Ninja builds depend on other incremental Ninja builds. Change-Id: I6a6a6964ef300cae2e180970048c8a4881c88e19 --- cli/zilch-ninja.scm | 17 ++-- lang/ninja/src/build.sld | 84 ++++++++++++++++-- lang/ninja/src/config.sld | 34 +++++++- lang/ninja/src/nixpkgs.sld | 171 ++++++++++++++++++++++++++++++++----- 4 files changed, 267 insertions(+), 39 deletions(-) diff --git a/cli/zilch-ninja.scm b/cli/zilch-ninja.scm index 55bef44..250c3fd 100644 --- a/cli/zilch-ninja.scm +++ b/cli/zilch-ninja.scm @@ -58,24 +58,29 @@ (when (and (ninja-build-config-depfile-path config) (file-exists? (ninja-build-config-depfile-path config))) (set-ninja-build-config-depfile! config (alist->mapping (make-default-comparator) (call-with-input-file (ninja-build-config-depfile-path config) read)))) +(define (export-depfiles finalized-drv) + (define config (finalized-drv-config finalized-drv)) + (when (ninja-build-config-depfile-path config) + (call-with-output-file (ninja-build-config-depfile-path config) (lambda (p) (write (mapping->alist (finalized-drv-depfile finalized-drv)) p)))) + (for-each (lambda (v) (export-depfiles (force (drv-rewrite-finalized-drv v)))) (finalized-drv-rewritten-drvs finalized-drv))) + (cond ((string=? (car args) "source") (let*-values - (((_ configured-drv _ _ _ _) (setup-ninja-environment config)) + (((_ configured-drv _ _ _ _ _) (setup-ninja-environment config '())) ((realised) (store-path-realised configured-drv)) ((path) (if (null? (cdr args)) "src" (cadr args)))) (system* (string-append "cp -rf --no-preserve=ownership " realised "/src " (qs path))) (system* (string-append "chmod -R u+rw " (qs path))))) ((string=? (car args) "build") (if (null? (cdr args)) - (let-values (((built export-depfile) (build-nixpkgs-drv-reproducibly config))) + (let ((output (build-nixpkgs-drv-reproducibly config))) (for-each (lambda (output-and-path) (store-path-realised (cdr output-and-path)) (printf "~A\t-> ~S\n" (car output-and-path) (cdr output-and-path))) - built) - (when (ninja-build-config-depfile-path config) - (call-with-output-file (ninja-build-config-depfile-path config) (lambda (p) (write (mapping->alist (export-depfile)) p))))) + (finalized-drv-output-store-paths output)) + (export-depfiles output)) (let-values (((_ _ _ edge-ref defaults _) (setup-ninja-environment config))) (for-each (lambda (target) @@ -84,7 +89,7 @@ (cdr args))))) ((string=? (car args) "diff") (let*-values - (((_ configured-drv _ _ _ _) (setup-ninja-environment config)) + (((_ configured-drv _ _ _ _ _) (setup-ninja-environment config '())) ((realised) (store-path-realised configured-drv)) ((path) (or source "src"))) (process-execute "git" (list "diff" "--no-index" "--" (string-append realised "/src") path)))) diff --git a/lang/ninja/src/build.sld b/lang/ninja/src/build.sld index 92d8e96..397895d 100644 --- a/lang/ninja/src/build.sld +++ b/lang/ninja/src/build.sld @@ -6,7 +6,8 @@ (zilch nix drv) (zilch nix path) (zilch nixpkgs) (zilch zexpr) (zilch vfs) (chicken format) - (srfi 128) (srfi 146) (srfi 152) + (srfi 26) (srfi 128) (srfi 146) (srfi 152) (srfi 207) + (zilch lib rewrite) (zilch lang ninja) (zilch lang ninja config) (zilch lang ninja depfile)) (export @@ -21,6 +22,56 @@ (define patchelf (cdr (assoc "out" (nixpkgs "patchelf")))) (define llvm-bintools (cdr (assoc "out" (nixpkgs-eval "llvmPackages_latest.bintools-unwrapped")))) + (define (replace-input drv input-drv new-outputs) + (when (store-path? drv) (set! drv (store-path-drv drv))) + (define output-names (map car (derivation-outputs drv))) + (define new-env (list)) + + (define input-drv-pathhash (string-copy (derivation-path input-drv) (+ 1 (string-length "/nix/store")) (+ 33 (string-length "/nix/store")))) + (define input-drv-name (derivation-name input-drv)) + + ; For each output of the to-remove .drv, make up to two rewrites: + ; the placeholder (/[base32 hex]) -> the corresponding output in new-outputs + ; the old output path -> the corresponding output in new-outputs + (define all-rewrites '()) + (for-each + (lambda (output-names) + (define new-path (string->utf8 (cdr (assoc (car output-names) new-outputs)))) + (define is-floating (eq? (derivation-output-hash (cdr output-names)) 'floating)) + (unless is-floating + (set! all-rewrites + (cons + (cons (string->utf8 (derivation-output-path (cdr output-names))) new-path) + all-rewrites))) + (set! all-rewrites + (cons + (cons (string->utf8 (make-upstream-output-placeholder input-drv-pathhash input-drv-name (car output-names))) new-path) + all-rewrites))) + (derivation-outputs input-drv)) + + (for-each + (lambda (kv) + (cond + ; TODO(puck): don't strip (dis)allowed(References/Requisites)? + ((member (car kv) '("allowedReferences" "disallowedReferences" "allowedRequisites" "disallowedRequisites") string=?) + #f) + ((member (car kv) output-names string=?) + #f) + (else + (set! new-env (cons (cons (car kv) (rewrite-bytevector-or-string (cdr kv) all-rewrites)) new-env))))) + (derivation-env drv)) + (define is-ca-derivation (symbol? (derivation-output-hash (cdar (derivation-outputs drv))))) + (define out-drv + ((if is-ca-derivation make-ca-derivation make-input-addressed-derivation) (derivation-name drv) + (derivation-system drv) + (derivation-input-drvs drv) + (derivation-input-src drv) + (cons (rewrite-bytevector-or-string (derivation-builder drv) all-rewrites) (map (lambda (v) (rewrite-bytevector-or-string v all-rewrites)) (derivation-args drv))) + new-env + (map car (derivation-outputs drv)))) + (map (lambda (v) (make-store-path out-drv (car v) #f)) (derivation-outputs out-drv))) + + ;; Represents a single built ``. ;; ;; - `edge`: The `` that this `` represents. @@ -38,7 +89,17 @@ (out-drv built-edge-out-drv) (lib-placeholder built-edge-lib-placeholder) (phony-inputs built-edge-phony-inputs)) - + + ;; Represents the entire build environment. + ;; + ;; - `config`: + ;; - `vfs`: The base file system, containing all files that are unchanging. Expected to be mounted at `/build/bdir`. + ;; - `header-files`: A list of all "header files". When an edge has a depfile, it is built assuming all files in this list that said edge uses are in this list. + ;; (string list) + ;; - `build-dir`: The build dir, relative to the root of the vfs. + ;; - `parsed-depfiles`: A mapping of (primary) edge output to the list of depnedencies in the depfile. + ;; - `collected-deps`: A rebuild of `parsed-depfiles`, filled in as edges are realised. + ;; - `secondary-vfs-setup`: A shell script (with $COREUTILS) that sets up any secondary VFS paths. (define-record-type (make-build-env config vfs header-files build-dir parsed-depfiles collected-deps secondary-vfs-setup) build-env? @@ -64,7 +125,7 @@ (else (set! part-stack (cons part part-stack))))) parts) (string-join (reverse part-stack) "/")) - + (define (is-valid-store-path-char c) (or (and (char>=? c #\0) (char<=? c #\9)) @@ -78,6 +139,9 @@ "zilch-ninja" (string-map (lambda (c) (if (is-valid-store-path-char c) c #\-)) (if (> (string-length str) 128) (string-copy str 0 128) str)))) + ; `edges`: mapping of relative-path to (input-store-path . >) + ; where input-store-path is 'base (part of base vfs), 'phony (follow built-edge-phony-inputs), or a store path. + ; depfile-data is a list of all dependency file based entries; and should be appended to the list of inputs. (define (inner-derivation-for-edge env edges current-edge resolved depfile-data) (when (build-rule-rspfile resolved) (error "rspfile not yet supported" current-edge)) (define copy-input-files "") @@ -97,9 +161,8 @@ (define input-edge (and input (force (cdr input)))) (cond - ; if input-file is 'base, this is part of the base vfs; we don't filter that right now. + ; if input-file is 'base, this is part of the base vfs; we don't filter that. ((eq? input-file 'base) #f) - ((and (pair? input-file) (eq? (car input-file) 'base-vfs)) #f) ; Phony rule; pass through the inputs literally. ((eq? input-file 'phony) (for-each append-file (built-edge-phony-inputs input-edge))) @@ -258,6 +321,11 @@ ;; Returns a derivation that runs the command for this edge, ;; inside a Nix derivation with the correct inputs. + ;; + ;; This differentiates from inner-derivation-for-edge by handling the two possible instantiations of the derivation. + ;; When a build fails because of header files, it will rerun with the full set of header files. + ;; + ;; TODO(puck): "header files" is a bad name for this concept? (define (derivation-for-edge env edges current-edge) (define resolved (build-edge-resolved current-edge)) (define result #f) @@ -346,7 +414,7 @@ ;; handle this Ninja file. ;; - `relative-to`: The directory in which the Ninja file was found. Used to ;; resolve relative references in the Ninja file. - ;; - `secondary-roots`: A list of (store-path-map finalized base-vfs-store-path-map (overlay-path-0 . promise-to-store-path) ... . (overlay-path-n . promise-to-store-path)) alists. i _guess_. + ;; - `secondary-roots`: A list of records. (define (process-ninja-file file conf relative-to secondary-roots) (unless (or (string=? relative-to "") (string-suffix? "/" relative-to)) (set! relative-to (string-append relative-to "/"))) @@ -356,7 +424,7 @@ #~,(string-append "$COREUTILS/cp -rf --no-preserve=ownership " #$v " " k "; " "$COREUTILS/chmod ugo+rw -R " k "; ")) - (define all-rewrites (apply append (map (lambda (v) (mapping-map->list setup-secondary-vfs (car (cddr v)))) secondary-roots))) + (define all-rewrites (apply append (map (lambda (v) (mapping-map->list setup-secondary-vfs (drv-rewrite-vfs-bases v))) secondary-roots))) (define secondary-vfs-setup #~,(apply string-append #$all-rewrites)) (define path-to-vfs @@ -451,7 +519,7 @@ (lambda (path-promise-pair) (set! edges (mapping-set! edges (car path-promise-pair) (cons (cdr path-promise-pair) #f))) (set-build-env-header-files! env (cons (car path-promise-pair) (build-env-header-files env)))) - (cdr (cddr secondary-root)))) + (drv-rewrite-overlay-paths secondary-root))) secondary-roots) ; Finish VFS filtering by taking out everything under build/meson-private. diff --git a/lang/ninja/src/config.sld b/lang/ninja/src/config.sld index 530a621..d10133c 100644 --- a/lang/ninja/src/config.sld +++ b/lang/ninja/src/config.sld @@ -11,9 +11,14 @@ ninja-build-config-environment ninja-build-config-environment-drv ninja-build-config-root-dir ninja-build-config-patches ninja-build-config-targets ninja-build-config-override-source ninja-build-config-depfile ninja-build-config-depfile-path - ninja-build-config-disallow-elide + ninja-build-config-disallow-elide ninja-build-config-rewrites set-ninja-build-config-root-dir! set-ninja-build-config-environment! set-ninja-build-config-depfile! + + + make-drv-rewrite drv-rewrite? + drv-rewrite-source-paths drv-rewrite-finalized-drv + drv-rewrite-vfs-bases drv-rewrite-overlay-paths parse-ninja-config) @@ -21,7 +26,7 @@ ;; Represents a parsed Ninja build configuration. ;; See `parse-ninja-config` for the definition of these fields. (define-record-type - (make-ninja-build-config environment environment-drv root-dir patches targets override-source depfile depfile-path disallow-elide) + (make-ninja-build-config environment environment-drv root-dir patches targets override-source depfile depfile-path disallow-elide rewrites) ninja-build-config? (environment ninja-build-config-environment set-ninja-build-config-environment!) (environment-drv ninja-build-config-environment-drv set-ninja-build-config-environment-drv!) @@ -31,7 +36,8 @@ (override-source ninja-build-config-override-source set-ninja-build-config-override-source!) (depfile ninja-build-config-depfile set-ninja-build-config-depfile!) (depfile-path ninja-build-config-depfile-path set-ninja-build-config-depfile-path!) - (disallow-elide ninja-build-config-disallow-elide set-ninja-build-config-disallow-elide!)) + (disallow-elide ninja-build-config-disallow-elide set-ninja-build-config-disallow-elide!) + (rewrites ninja-build-config-rewrites set-ninja-build-config-rewrites!)) (define (parse-config-inner conf data) (cond @@ -88,6 +94,11 @@ (list-val (if (list? val) val (list val)))) (set-ninja-build-config-targets! conf (append list-val (ninja-build-config-targets conf)))) (parse-config-inner conf (cddr data))) + ((#:rewrite) + (let* + ((val (list-ref data 1))) + (set-ninja-build-config-rewrites! conf (cons (cons (car val) (parse-ninja-config (cdr val))) (ninja-build-config-rewrites conf)))) + (parse-config-inner conf (cddr data))) (else (error (string-append "Unknown directive " (keyword->string (car data)) " parsing Zilch Ninja config"))))))) ;; Parses a Zilch Ninja configuration file. @@ -118,7 +129,22 @@ ;; incorrect. ;; - `target: "foo"`/`targets: '("foo" "bar")`: Build these targets instead ;; of the default specified in the Ninja file. + ;; - `rewrite: ("" . ): A nested Zilch + ;; Ninja configuration to swap out the input derivation with equivalent + ;; name. (define (parse-ninja-config config) (unless (list? config) (error "expected Zilch Ninja config to be a list")) - (parse-config-inner (make-ninja-build-config #f #f #f '() #f #f #f #f #f) config)))) + (parse-config-inner (make-ninja-build-config #f #f #f '() #f #f #f #f #f '()) config)) + + ;; `source-paths`: mapping of original store path to virtual path. + ;; `finalized-drv`: promise of a + ;; `vfs-bases`: mapping of virtualised path to base VFS + ;; `overlay-paths`: alist of virtualised path to promise of a store path + (define-record-type + (make-drv-rewrite source-paths finalized-drv vfs-bases overlay-paths) + drv-rewrite? + (source-paths drv-rewrite-source-paths) + (finalized-drv drv-rewrite-finalized-drv) + (vfs-bases drv-rewrite-vfs-bases) + (overlay-paths drv-rewrite-overlay-paths)))) diff --git a/lang/ninja/src/nixpkgs.sld b/lang/ninja/src/nixpkgs.sld index f49672d..1b7f3f4 100644 --- a/lang/ninja/src/nixpkgs.sld +++ b/lang/ninja/src/nixpkgs.sld @@ -6,18 +6,44 @@ (zilch lang ninja config) (zilch magic) (zilch nixpkgs) (zilch vfs) (zilch file) - (zilch nix drv) + (zilch nix drv) (zilch nix hash) + (zilch lib hash) (zilch lib rewrite) (zilch zexpr) - (srfi 128) (srfi 132) (srfi 146) (srfi 152)) + (srfi 26) (srfi 128) (srfi 132) (srfi 146) (srfi 152) (srfi 207)) (export setup-ninja-environment build-nixpkgs-drv-reproducibly - determine-data-flow) + determine-data-flow + virtual-path-for make-virtual-path + + finalized-drv? + finalized-drv-output-store-paths finalized-drv-depfile + finalized-drv-rewritten-drvs finalized-drv-config) + (begin (define coreutils (cdr (assoc "out" (nixpkgs "coreutils")))) + (define rewrite-hooks + (string-append + "__zilch_rewrite_sed=\"\"\n" + "echo \"$__zilch_rewrites\" | while read -d ' ' rewrite_from; do\n" + " read -d ' ' rewrite_to\n" + " __zilch_rewrite_sed=\"$__zilch_rewrite_sed s|$rewrite_from|$rewrite_to|g\"\n" + "done || true\n" + "echo \"$__zilch_rewrites\" | while read -d ' ' rewrite_from; do\n" + " read -d ' ' rewrite_to\n" + " cp --no-preserve=ownership -rf \"$rewrite_from\" \"$rewrite_to\"\n" + " chmod -R ugo+rw \"$rewrite_to\"\n" + " find \"$rewrite_to\" -type f -exec sed -i -e \"$__zilch_rewrite_sed\" \"{}\" \";\" || exit 1\n" + " find \"$rewrite_to\" -type l | while read link; do\n" + " target=\"$(readlink \"$link\")\"; rewritten=\"$(printf \"%s\" \"$target\" | sed -e \"$__zilch_rewrite_sed\")\"\n" + " rm \"$link\" && ln -s \"$rewritten\" \"$link\" || exit 1\n" + " done\n" + "done || true\n" + "unset __zilch_rewrites __zilch_rewrite_sed\n")) + ; Shellcode to run instead of the default stdenv genericBuild(); ; This takes the source and configuration of our derivation and prepares ; a proper environment to build inside. @@ -38,7 +64,7 @@ ; Patch the passed in .drv by appending to the environment ; and changing the list of outputs. - (define (patch-drv drv append-env outputs) + (define (patch-drv drv append-env outputs rewrites) (define ctx (zexp-unwrap (zexp (zexp-unquote append-env)))) (define new-env (list)) (for-each @@ -47,7 +73,7 @@ (cond (mem (set! new-env (cons mem new-env))) ((member (car kv) '("allowedReferences" "disallowedReferences" "allowedRequisites" "disallowedRequisites")) #f) - (else (set! new-env (cons kv new-env))))) + (else (set! new-env (cons (cons (car kv) (rewrite-bytevector-or-string (cdr kv) rewrites)) new-env))))) (derivation-env drv)) (for-each (lambda (kv) @@ -70,6 +96,23 @@ (map (lambda (l) (cons (car l) (make-store-path reprocessed (car l) #f))) (derivation-outputs reprocessed))) (define base-placeholder "zilchplaceholderdonotuseanywhere-") + + ;; Generate a placeholder for a virtualised path. + ;; This is similar to `make-placeholder`, but outputs + ;; a string that is of the shape `/nix/store/{invalid hash}-{discriminator}`, + ;; where the hash starts with `tz`, to make it invalid base32. + (define (make-virtual-path discriminator) + (define data (string-append "zilch!" discriminator)) + (define hash (as-base32 (hash-compress (sha256 (string->utf8 data))))) + (string-set! hash 0 #\t) + (string-set! hash 1 #\z) + (string-append "/nix/store/" hash "-" discriminator)) + + ;; Generate a placeholder for a virtualised path, like `make-virtual-path`, + ;; but based on the input store path string. + (define (virtual-path-for store-path) + (define index (string-contains store-path "-")) + (make-virtual-path (string-copy store-path (+ 1 index)))) ; Create a placeholder store path that is unique (define (make-fake-store-path name output) @@ -77,7 +120,18 @@ (set! output (string-copy output 0 32))) (string-append "/nix/store/" output (string-copy base-placeholder (string-length output)) name (if (string=? output "out") "" (string-append "-" output)))) - + + (define-record-type + (make-finalized-drv output-store-paths export-depfile rewritten-drvs config) + finalized-drv? + (output-store-paths finalized-drv-output-store-paths) + (export-depfile finalized-drv-export-depfile) + (rewritten-drvs finalized-drv-rewritten-drvs) + (config finalized-drv-config)) + + (define (finalized-drv-depfile drv) + ((finalized-drv-export-depfile drv))) + ;; Takes a `` representing a Nixpkgs derivation, and ;; preprocesses the derivation such that it can be reconstituted once Zilch ;; has taken over the Ninja build requirements. @@ -86,7 +140,7 @@ ;; encodes many specific parts that are unlikely to be useful by external ;; parties. ;; - ;; Returns 6 values: + ;; Returns 7 values: ;; ;; - The initial Nixpkgs derivation as `` ;; - The Nixpkgs derivation after running all phases up to and including @@ -94,7 +148,8 @@ ;; - An alist of output names to placeholder store paths ;; - The `edge-ref`, `defaults`, and `export-depfile` values from calling ;; `process-ninja-file` - (define (setup-ninja-environment conf) + ;; - A list of store path -> store path rewrites. + (define (setup-ninja-environment conf secondary-roots) (define initial-drv (ninja-build-config-environment-drv conf)) (when (store-path? initial-drv) (set! initial-drv (store-path-drv initial-drv))) @@ -102,18 +157,32 @@ (define placeholders (map (lambda (output-and-info) (cons (car output-and-info) (make-fake-store-path name (car output-and-info)))) (derivation-outputs initial-drv))) (define existing-env (ninja-build-config-environment conf)) + + (define rewrites (list)) + (define rewrite-extra "") + (for-each + (lambda (secondary-root) + (mapping-for-each + (lambda (orig-path virtual-path) + (set! rewrites (cons (cons (string->utf8 orig-path) (string->utf8 virtual-path)) rewrites)) + (set! rewrite-extra (string-append rewrite-extra orig-path " " virtual-path "\n"))) + (drv-rewrite-source-paths secondary-root))) + secondary-roots) ; Override output environment variables with our placeholders. - (set-ninja-build-config-environment! conf #~,(map (lambda (v) (or (assoc (car v) placeholders) v)) #$existing-env)) + (set-ninja-build-config-environment! conf #~,(map (lambda (v) (or (assoc (car v) placeholders) (cons (car v) (rewrite-bytevector-or-string (cdr v) rewrites)))) #$existing-env)) ; Take the initially requested .drv, replace its buildCommand, and set a single ("zilch_out") output path. (define configured-drv (cdar (patch-drv initial-drv (append - `(("buildCommand" . ,configure-builder)) + `(("buildCommand" . ,configure-builder) + ("addInputsHook" . ,rewrite-hooks) + ("__zilch_rewrites" . ,rewrite-extra)) placeholders) - '("zilch_out")))) + '("zilch_out") + rewrites))) ; This VFS contains two directories: `src` (source tree) and `build` (Ninja build files). (define configured-vfs (vfs-from-store configured-drv)) @@ -141,10 +210,49 @@ (read-ninja-file (read-file-at-path "build.ninja") read-file-at-path)) ; Process the build.ninja file. - (define-values (edge-ref defaults export-depfile) (process-ninja-file ninja-file conf "build")) + (define-values (edge-ref defaults export-depfile) (process-ninja-file ninja-file conf "build" secondary-roots)) - (values initial-drv configured-drv placeholders edge-ref defaults export-depfile)) - + (values initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites)) + + (define (process-secondary-root parent-conf secondary-root) + (define conf (cdr secondary-root)) + (define drv-name (car secondary-root)) + + (define matching-drv #f) + (let loop ((drv-list (derivation-input-drvs (ninja-build-config-environment-drv parent-conf)))) + (cond + ((null? drv-list) (error "Cannot find matching derivation for rewrite" drv-name)) + ((string=? (derivation-name (caar drv-list)) drv-name) + (set! matching-drv (caar drv-list))) + (else (loop (cdr drv-list))))) + (define (disc-name out-name) + (if (string=? out-name "out") drv-name (string-append drv-name "-" out-name))) + + (define future-rewrites (map (lambda (a) (cons (derivation-output-path (cdr a)) (make-virtual-path (disc-name (car a))))) (derivation-outputs matching-drv))) + (define output-map (alist->mapping (make-default-comparator) (map (lambda (a) (cons (car a) (make-virtual-path (disc-name (car a))))) (derivation-outputs matching-drv)))) + ; secondary-root is (name . #) + (define its-secondary-roots (map (cute process-secondary-root conf <>) (ninja-build-config-rewrites conf))) + (define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites) (setup-ninja-environment conf its-secondary-roots)) + (define vfses (mapping-map (lambda (k v) (values k (mapping (make-default-comparator)))) (make-default-comparator) output-map)) + (define (transform-obj obj) + (cond + ((and (pair? obj) (eq? (car obj) 'marker)) + (let ((obj-edge (edge-ref (cdr obj)))) + (or (and (cdr obj-edge) (built-edge-lib-placeholder (cdr obj-edge))) + (car obj-edge)))) + (else obj))) + (define extra-paths '()) + (mapping-for-each + (lambda (key obj) + (define base-store-path (mapping-ref output-map (car key))) + (define full-path (string-append base-store-path "/" (if (string=? "" (cadr key)) (cddr key) (string-append (cadr key) "/" (cddr key))))) + (if (or (string-suffix? ".hh" (cddr key)) (string-suffix? ".h" (cddr key)) (string-suffix? ".hpp" (cddr key))) + (set! extra-paths (cons (cons full-path (transform-obj obj)) extra-paths)) + (set! vfses (mapping-set! vfses (car key) (mapping-set! (mapping-ref vfses (car key)) (cdr key) (transform-obj obj)))))) + (determine-data-flow conf)) + (set! vfses (mapping-map/monotone! (lambda (k v) (values k (vfs-to-store (make-vfs v)))) (make-default-comparator) vfses)) + (define finalized (delay (finalize-drv conf its-secondary-roots initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites))) + (make-drv-rewrite output-map finalized vfses extra-paths)) ;; Takes a `` representing a Nixpkgs derivation, and ;; build it using Zilch. ;; @@ -156,8 +264,12 @@ ;; necessary inputs generated from the depfile data. This can be stored in ;; a file for later rebuilds. (define (build-nixpkgs-drv-reproducibly conf) - (define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf)) + (define secondary-roots (map (cute process-secondary-root conf <>) (ninja-build-config-rewrites conf))) + (define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites) (setup-ninja-environment conf secondary-roots)) + (finalize-drv conf secondary-roots initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites)) + ;; "Finalizes" a previously setup Ninja environment, running the install and fixup hooks. + (define (finalize-drv conf secondary-roots initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites) ; Build all store paths necessary for installing. This assumes Meson. (define preinstall-state (force (built-edge-out-drv (cdr (edge-ref "all"))))) @@ -182,6 +294,20 @@ ; This is necessary because these paths may be CA, and we can't guarantee daemon support for that past this point. (define realised-built (store-path-devirtualise preinstall-state)) + ; Take each output store path, then copy it over to the expected place + ; then later run a fixup to the original store path at placeholder state + (define prepared-store-paths (list)) + (define rewrite-placeholder-fixes (list)) + (define (fixup-single-secondary-root single-drv-item) + (define drv-name (virtual-path-for (store-path-path single-drv-item))) + (set! prepared-store-paths (cons #~,(string-append "cp -rf " #$single-drv-item " " drv-name "\n") prepared-store-paths)) + (set! rewrite-placeholder-fixes (cons #~,(string-append "zilchFixPlaceholder " drv-name " " #$single-drv-item "\n") rewrite-placeholder-fixes))) + (define (prepare-secondary-root secondary-root) + ; alist of output name -> store path + (define realised (finalized-drv-output-store-paths (force (drv-rewrite-finalized-drv secondary-root)))) + (map (lambda (v) (fixup-single-secondary-root (cdr v))) realised)) + (for-each prepare-secondary-root secondary-roots) + ; Prepare the post-build builder. This puts everything in its place and runs the post-build phases from the original .drv. (define postbuild-builder #~,(string-append @@ -201,6 +327,7 @@ "}\n" "zilchFixup() {\n" fix-placeholders-command + #$(apply string-append #$rewrite-placeholder-fixes) copy-in-place-command "}\n" "zilchMesonInstall() {\n" @@ -229,13 +356,15 @@ "for curPhase in ${phases[*]}; do runPhase \"$curPhase\"; done\n")) ; Patch the original .drv to run the postbuild-builder command. - (values + (define store-paths (patch-drv initial-drv (append - `(("buildCommand" . ,postbuild-builder)) + `(("buildCommand" . ,postbuild-builder) + ("addInputsHook" . ,#~,(apply string-append #$prepared-store-paths))) placeholders) - (map car placeholders)) - export-depfile)) + (map car placeholders) + rewrites)) + (make-finalized-drv store-paths export-depfile secondary-roots conf)) ;; Build the derivation, but with stubbed out header and .so files. ;; This is used to determine the dataflow, to make cross-project incremental @@ -243,7 +372,7 @@ ;; Returns a single SRFI-146 mapping, containing keys of shape `(output . (dir . name))`, ;; and values either a zexpr-y store path, or a pair `(marker . )` (define (determine-data-flow conf) - (define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf)) + (define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites) (setup-ninja-environment conf '())) (define edges (built-edge-phony-inputs (cdr (edge-ref "all")))) @@ -325,7 +454,7 @@ (append `(("buildCommand" . ,postbuild-builder)) placeholders) - (map car placeholders))) + (map car placeholders) '())) (define (get-file-marker fptr) (call-with-port (store-path-open fptr) (lambda (p)