(zilch lang ninja): Support rewriting derivation inputs
This lets incremental Ninja builds depend on other incremental Ninja builds. Change-Id: I6a6a6964ef300cae2e180970048c8a4881c88e19
This commit is contained in:
parent
c1c606ee0b
commit
0023f3def8
4 changed files with 267 additions and 39 deletions
|
|
@ -6,7 +6,8 @@
|
|||
(zilch nix drv) (zilch nix path)
|
||||
(zilch nixpkgs) (zilch zexpr) (zilch vfs)
|
||||
(chicken format)
|
||||
(srfi 128) (srfi 146) (srfi 152)
|
||||
(srfi 26) (srfi 128) (srfi 146) (srfi 152) (srfi 207)
|
||||
(zilch lib rewrite)
|
||||
(zilch lang ninja) (zilch lang ninja config) (zilch lang ninja depfile))
|
||||
|
||||
(export
|
||||
|
|
@ -21,6 +22,56 @@
|
|||
(define patchelf (cdr (assoc "out" (nixpkgs "patchelf"))))
|
||||
(define llvm-bintools (cdr (assoc "out" (nixpkgs-eval "llvmPackages_latest.bintools-unwrapped"))))
|
||||
|
||||
(define (replace-input drv input-drv new-outputs)
|
||||
(when (store-path? drv) (set! drv (store-path-drv drv)))
|
||||
(define output-names (map car (derivation-outputs drv)))
|
||||
(define new-env (list))
|
||||
|
||||
(define input-drv-pathhash (string-copy (derivation-path input-drv) (+ 1 (string-length "/nix/store")) (+ 33 (string-length "/nix/store"))))
|
||||
(define input-drv-name (derivation-name input-drv))
|
||||
|
||||
; For each output of the to-remove .drv, make up to two rewrites:
|
||||
; the placeholder (/[base32 hex]) -> the corresponding output in new-outputs
|
||||
; the old output path -> the corresponding output in new-outputs
|
||||
(define all-rewrites '())
|
||||
(for-each
|
||||
(lambda (output-names)
|
||||
(define new-path (string->utf8 (cdr (assoc (car output-names) new-outputs))))
|
||||
(define is-floating (eq? (derivation-output-hash (cdr output-names)) 'floating))
|
||||
(unless is-floating
|
||||
(set! all-rewrites
|
||||
(cons
|
||||
(cons (string->utf8 (derivation-output-path (cdr output-names))) new-path)
|
||||
all-rewrites)))
|
||||
(set! all-rewrites
|
||||
(cons
|
||||
(cons (string->utf8 (make-upstream-output-placeholder input-drv-pathhash input-drv-name (car output-names))) new-path)
|
||||
all-rewrites)))
|
||||
(derivation-outputs input-drv))
|
||||
|
||||
(for-each
|
||||
(lambda (kv)
|
||||
(cond
|
||||
; TODO(puck): don't strip (dis)allowed(References/Requisites)?
|
||||
((member (car kv) '("allowedReferences" "disallowedReferences" "allowedRequisites" "disallowedRequisites") string=?)
|
||||
#f)
|
||||
((member (car kv) output-names string=?)
|
||||
#f)
|
||||
(else
|
||||
(set! new-env (cons (cons (car kv) (rewrite-bytevector-or-string (cdr kv) all-rewrites)) new-env)))))
|
||||
(derivation-env drv))
|
||||
(define is-ca-derivation (symbol? (derivation-output-hash (cdar (derivation-outputs drv)))))
|
||||
(define out-drv
|
||||
((if is-ca-derivation make-ca-derivation make-input-addressed-derivation) (derivation-name drv)
|
||||
(derivation-system drv)
|
||||
(derivation-input-drvs drv)
|
||||
(derivation-input-src drv)
|
||||
(cons (rewrite-bytevector-or-string (derivation-builder drv) all-rewrites) (map (lambda (v) (rewrite-bytevector-or-string v all-rewrites)) (derivation-args drv)))
|
||||
new-env
|
||||
(map car (derivation-outputs drv))))
|
||||
(map (lambda (v) (make-store-path out-drv (car v) #f)) (derivation-outputs out-drv)))
|
||||
|
||||
|
||||
;; Represents a single built `<build-edge>`.
|
||||
;;
|
||||
;; - `edge`: The `<build-edge>` that this `<built-edge>` represents.
|
||||
|
|
@ -38,7 +89,17 @@
|
|||
(out-drv built-edge-out-drv)
|
||||
(lib-placeholder built-edge-lib-placeholder)
|
||||
(phony-inputs built-edge-phony-inputs))
|
||||
|
||||
|
||||
;; Represents the entire build environment.
|
||||
;;
|
||||
;; - `config`:
|
||||
;; - `vfs`: The base file system, containing all files that are unchanging. Expected to be mounted at `/build/bdir`.
|
||||
;; - `header-files`: A list of all "header files". When an edge has a depfile, it is built assuming all files in this list that said edge uses are in this list.
|
||||
;; (string list)
|
||||
;; - `build-dir`: The build dir, relative to the root of the vfs.
|
||||
;; - `parsed-depfiles`: A mapping of (primary) edge output to the list of depnedencies in the depfile.
|
||||
;; - `collected-deps`: A rebuild of `parsed-depfiles`, filled in as edges are realised.
|
||||
;; - `secondary-vfs-setup`: A shell script (with $COREUTILS) that sets up any secondary VFS paths.
|
||||
(define-record-type <build-env>
|
||||
(make-build-env config vfs header-files build-dir parsed-depfiles collected-deps secondary-vfs-setup)
|
||||
build-env?
|
||||
|
|
@ -64,7 +125,7 @@
|
|||
(else (set! part-stack (cons part part-stack)))))
|
||||
parts)
|
||||
(string-join (reverse part-stack) "/"))
|
||||
|
||||
|
||||
(define (is-valid-store-path-char c)
|
||||
(or
|
||||
(and (char>=? c #\0) (char<=? c #\9))
|
||||
|
|
@ -78,6 +139,9 @@
|
|||
"zilch-ninja"
|
||||
(string-map (lambda (c) (if (is-valid-store-path-char c) c #\-)) (if (> (string-length str) 128) (string-copy str 0 128) str))))
|
||||
|
||||
; `edges`: mapping of relative-path to (input-store-path . <promise of <built-edge>>)
|
||||
; where input-store-path is 'base (part of base vfs), 'phony (follow built-edge-phony-inputs), or a store path.
|
||||
; depfile-data is a list of all dependency file based entries; and should be appended to the list of inputs.
|
||||
(define (inner-derivation-for-edge env edges current-edge resolved depfile-data)
|
||||
(when (build-rule-rspfile resolved) (error "rspfile not yet supported" current-edge))
|
||||
(define copy-input-files "")
|
||||
|
|
@ -97,9 +161,8 @@
|
|||
(define input-edge (and input (force (cdr input))))
|
||||
|
||||
(cond
|
||||
; if input-file is 'base, this is part of the base vfs; we don't filter that right now.
|
||||
; if input-file is 'base, this is part of the base vfs; we don't filter that.
|
||||
((eq? input-file 'base) #f)
|
||||
((and (pair? input-file) (eq? (car input-file) 'base-vfs)) #f)
|
||||
|
||||
; Phony rule; pass through the inputs literally.
|
||||
((eq? input-file 'phony) (for-each append-file (built-edge-phony-inputs input-edge)))
|
||||
|
|
@ -258,6 +321,11 @@
|
|||
|
||||
;; Returns a derivation that runs the command for this edge,
|
||||
;; inside a Nix derivation with the correct inputs.
|
||||
;;
|
||||
;; This differentiates from inner-derivation-for-edge by handling the two possible instantiations of the derivation.
|
||||
;; When a build fails because of header files, it will rerun with the full set of header files.
|
||||
;;
|
||||
;; TODO(puck): "header files" is a bad name for this concept?
|
||||
(define (derivation-for-edge env edges current-edge)
|
||||
(define resolved (build-edge-resolved current-edge))
|
||||
(define result #f)
|
||||
|
|
@ -346,7 +414,7 @@
|
|||
;; handle this Ninja file.
|
||||
;; - `relative-to`: The directory in which the Ninja file was found. Used to
|
||||
;; resolve relative references in the Ninja file.
|
||||
;; - `secondary-roots`: A list of (store-path-map finalized base-vfs-store-path-map (overlay-path-0 . promise-to-store-path) ... . (overlay-path-n . promise-to-store-path)) alists. i _guess_.
|
||||
;; - `secondary-roots`: A list of <drv-rewrite> records.
|
||||
(define (process-ninja-file file conf relative-to secondary-roots)
|
||||
(unless (or (string=? relative-to "") (string-suffix? "/" relative-to)) (set! relative-to (string-append relative-to "/")))
|
||||
|
||||
|
|
@ -356,7 +424,7 @@
|
|||
#~,(string-append
|
||||
"$COREUTILS/cp -rf --no-preserve=ownership " #$v " " k "; "
|
||||
"$COREUTILS/chmod ugo+rw -R " k "; "))
|
||||
(define all-rewrites (apply append (map (lambda (v) (mapping-map->list setup-secondary-vfs (car (cddr v)))) secondary-roots)))
|
||||
(define all-rewrites (apply append (map (lambda (v) (mapping-map->list setup-secondary-vfs (drv-rewrite-vfs-bases v))) secondary-roots)))
|
||||
(define secondary-vfs-setup #~,(apply string-append #$all-rewrites))
|
||||
|
||||
(define path-to-vfs
|
||||
|
|
@ -451,7 +519,7 @@
|
|||
(lambda (path-promise-pair)
|
||||
(set! edges (mapping-set! edges (car path-promise-pair) (cons (cdr path-promise-pair) #f)))
|
||||
(set-build-env-header-files! env (cons (car path-promise-pair) (build-env-header-files env))))
|
||||
(cdr (cddr secondary-root))))
|
||||
(drv-rewrite-overlay-paths secondary-root)))
|
||||
secondary-roots)
|
||||
|
||||
; Finish VFS filtering by taking out everything under build/meson-private.
|
||||
|
|
|
|||
|
|
@ -11,9 +11,14 @@
|
|||
ninja-build-config-environment ninja-build-config-environment-drv
|
||||
ninja-build-config-root-dir ninja-build-config-patches ninja-build-config-targets
|
||||
ninja-build-config-override-source ninja-build-config-depfile ninja-build-config-depfile-path
|
||||
ninja-build-config-disallow-elide
|
||||
ninja-build-config-disallow-elide ninja-build-config-rewrites
|
||||
|
||||
set-ninja-build-config-root-dir! set-ninja-build-config-environment! set-ninja-build-config-depfile!
|
||||
|
||||
<drv-rewrite>
|
||||
make-drv-rewrite drv-rewrite?
|
||||
drv-rewrite-source-paths drv-rewrite-finalized-drv
|
||||
drv-rewrite-vfs-bases drv-rewrite-overlay-paths
|
||||
|
||||
parse-ninja-config)
|
||||
|
||||
|
|
@ -21,7 +26,7 @@
|
|||
;; Represents a parsed Ninja build configuration.
|
||||
;; See `parse-ninja-config` for the definition of these fields.
|
||||
(define-record-type <ninja-build-config>
|
||||
(make-ninja-build-config environment environment-drv root-dir patches targets override-source depfile depfile-path disallow-elide)
|
||||
(make-ninja-build-config environment environment-drv root-dir patches targets override-source depfile depfile-path disallow-elide rewrites)
|
||||
ninja-build-config?
|
||||
(environment ninja-build-config-environment set-ninja-build-config-environment!)
|
||||
(environment-drv ninja-build-config-environment-drv set-ninja-build-config-environment-drv!)
|
||||
|
|
@ -31,7 +36,8 @@
|
|||
(override-source ninja-build-config-override-source set-ninja-build-config-override-source!)
|
||||
(depfile ninja-build-config-depfile set-ninja-build-config-depfile!)
|
||||
(depfile-path ninja-build-config-depfile-path set-ninja-build-config-depfile-path!)
|
||||
(disallow-elide ninja-build-config-disallow-elide set-ninja-build-config-disallow-elide!))
|
||||
(disallow-elide ninja-build-config-disallow-elide set-ninja-build-config-disallow-elide!)
|
||||
(rewrites ninja-build-config-rewrites set-ninja-build-config-rewrites!))
|
||||
|
||||
(define (parse-config-inner conf data)
|
||||
(cond
|
||||
|
|
@ -88,6 +94,11 @@
|
|||
(list-val (if (list? val) val (list val))))
|
||||
(set-ninja-build-config-targets! conf (append list-val (ninja-build-config-targets conf))))
|
||||
(parse-config-inner conf (cddr data)))
|
||||
((#:rewrite)
|
||||
(let*
|
||||
((val (list-ref data 1)))
|
||||
(set-ninja-build-config-rewrites! conf (cons (cons (car val) (parse-ninja-config (cdr val))) (ninja-build-config-rewrites conf))))
|
||||
(parse-config-inner conf (cddr data)))
|
||||
(else (error (string-append "Unknown directive " (keyword->string (car data)) " parsing Zilch Ninja config")))))))
|
||||
|
||||
;; Parses a Zilch Ninja configuration file.
|
||||
|
|
@ -118,7 +129,22 @@
|
|||
;; incorrect.
|
||||
;; - `target: "foo"`/`targets: '("foo" "bar")`: Build these targets instead
|
||||
;; of the default specified in the Ninja file.
|
||||
;; - `rewrite: ("<derivation name>" . <nested config>): A nested Zilch
|
||||
;; Ninja configuration to swap out the input derivation with equivalent
|
||||
;; name.
|
||||
(define (parse-ninja-config config)
|
||||
(unless (list? config)
|
||||
(error "expected Zilch Ninja config to be a list"))
|
||||
(parse-config-inner (make-ninja-build-config #f #f #f '() #f #f #f #f #f) config))))
|
||||
(parse-config-inner (make-ninja-build-config #f #f #f '() #f #f #f #f #f '()) config))
|
||||
|
||||
;; `source-paths`: mapping of original store path to virtual path.
|
||||
;; `finalized-drv`: promise of a <finalized-drv>
|
||||
;; `vfs-bases`: mapping of virtualised path to base VFS
|
||||
;; `overlay-paths`: alist of virtualised path to promise of a store path
|
||||
(define-record-type <drv-rewrite>
|
||||
(make-drv-rewrite source-paths finalized-drv vfs-bases overlay-paths)
|
||||
drv-rewrite?
|
||||
(source-paths drv-rewrite-source-paths)
|
||||
(finalized-drv drv-rewrite-finalized-drv)
|
||||
(vfs-bases drv-rewrite-vfs-bases)
|
||||
(overlay-paths drv-rewrite-overlay-paths))))
|
||||
|
|
|
|||
|
|
@ -6,18 +6,44 @@
|
|||
(zilch lang ninja config)
|
||||
(zilch magic) (zilch nixpkgs) (zilch vfs)
|
||||
(zilch file)
|
||||
(zilch nix drv)
|
||||
(zilch nix drv) (zilch nix hash)
|
||||
(zilch lib hash) (zilch lib rewrite)
|
||||
(zilch zexpr)
|
||||
(srfi 128) (srfi 132) (srfi 146) (srfi 152))
|
||||
(srfi 26) (srfi 128) (srfi 132) (srfi 146) (srfi 152) (srfi 207))
|
||||
|
||||
(export
|
||||
setup-ninja-environment
|
||||
build-nixpkgs-drv-reproducibly
|
||||
determine-data-flow)
|
||||
determine-data-flow
|
||||
virtual-path-for make-virtual-path
|
||||
|
||||
<finalized-drv> finalized-drv?
|
||||
finalized-drv-output-store-paths finalized-drv-depfile
|
||||
finalized-drv-rewritten-drvs finalized-drv-config)
|
||||
|
||||
|
||||
(begin
|
||||
(define coreutils (cdr (assoc "out" (nixpkgs "coreutils"))))
|
||||
|
||||
(define rewrite-hooks
|
||||
(string-append
|
||||
"__zilch_rewrite_sed=\"\"\n"
|
||||
"echo \"$__zilch_rewrites\" | while read -d ' ' rewrite_from; do\n"
|
||||
" read -d ' ' rewrite_to\n"
|
||||
" __zilch_rewrite_sed=\"$__zilch_rewrite_sed s|$rewrite_from|$rewrite_to|g\"\n"
|
||||
"done || true\n"
|
||||
"echo \"$__zilch_rewrites\" | while read -d ' ' rewrite_from; do\n"
|
||||
" read -d ' ' rewrite_to\n"
|
||||
" cp --no-preserve=ownership -rf \"$rewrite_from\" \"$rewrite_to\"\n"
|
||||
" chmod -R ugo+rw \"$rewrite_to\"\n"
|
||||
" find \"$rewrite_to\" -type f -exec sed -i -e \"$__zilch_rewrite_sed\" \"{}\" \";\" || exit 1\n"
|
||||
" find \"$rewrite_to\" -type l | while read link; do\n"
|
||||
" target=\"$(readlink \"$link\")\"; rewritten=\"$(printf \"%s\" \"$target\" | sed -e \"$__zilch_rewrite_sed\")\"\n"
|
||||
" rm \"$link\" && ln -s \"$rewritten\" \"$link\" || exit 1\n"
|
||||
" done\n"
|
||||
"done || true\n"
|
||||
"unset __zilch_rewrites __zilch_rewrite_sed\n"))
|
||||
|
||||
; Shellcode to run instead of the default stdenv genericBuild();
|
||||
; This takes the source and configuration of our derivation and prepares
|
||||
; a proper environment to build inside.
|
||||
|
|
@ -38,7 +64,7 @@
|
|||
|
||||
; Patch the passed in .drv by appending to the environment
|
||||
; and changing the list of outputs.
|
||||
(define (patch-drv drv append-env outputs)
|
||||
(define (patch-drv drv append-env outputs rewrites)
|
||||
(define ctx (zexp-unwrap (zexp (zexp-unquote append-env))))
|
||||
(define new-env (list))
|
||||
(for-each
|
||||
|
|
@ -47,7 +73,7 @@
|
|||
(cond
|
||||
(mem (set! new-env (cons mem new-env)))
|
||||
((member (car kv) '("allowedReferences" "disallowedReferences" "allowedRequisites" "disallowedRequisites")) #f)
|
||||
(else (set! new-env (cons kv new-env)))))
|
||||
(else (set! new-env (cons (cons (car kv) (rewrite-bytevector-or-string (cdr kv) rewrites)) new-env)))))
|
||||
(derivation-env drv))
|
||||
(for-each
|
||||
(lambda (kv)
|
||||
|
|
@ -70,6 +96,23 @@
|
|||
(map (lambda (l) (cons (car l) (make-store-path reprocessed (car l) #f))) (derivation-outputs reprocessed)))
|
||||
|
||||
(define base-placeholder "zilchplaceholderdonotuseanywhere-")
|
||||
|
||||
;; Generate a placeholder for a virtualised path.
|
||||
;; This is similar to `make-placeholder`, but outputs
|
||||
;; a string that is of the shape `/nix/store/{invalid hash}-{discriminator}`,
|
||||
;; where the hash starts with `tz`, to make it invalid base32.
|
||||
(define (make-virtual-path discriminator)
|
||||
(define data (string-append "zilch!" discriminator))
|
||||
(define hash (as-base32 (hash-compress (sha256 (string->utf8 data)))))
|
||||
(string-set! hash 0 #\t)
|
||||
(string-set! hash 1 #\z)
|
||||
(string-append "/nix/store/" hash "-" discriminator))
|
||||
|
||||
;; Generate a placeholder for a virtualised path, like `make-virtual-path`,
|
||||
;; but based on the input store path string.
|
||||
(define (virtual-path-for store-path)
|
||||
(define index (string-contains store-path "-"))
|
||||
(make-virtual-path (string-copy store-path (+ 1 index))))
|
||||
|
||||
; Create a placeholder store path that is unique
|
||||
(define (make-fake-store-path name output)
|
||||
|
|
@ -77,7 +120,18 @@
|
|||
(set! output (string-copy output 0 32)))
|
||||
|
||||
(string-append "/nix/store/" output (string-copy base-placeholder (string-length output)) name (if (string=? output "out") "" (string-append "-" output))))
|
||||
|
||||
|
||||
(define-record-type <finalized-drv>
|
||||
(make-finalized-drv output-store-paths export-depfile rewritten-drvs config)
|
||||
finalized-drv?
|
||||
(output-store-paths finalized-drv-output-store-paths)
|
||||
(export-depfile finalized-drv-export-depfile)
|
||||
(rewritten-drvs finalized-drv-rewritten-drvs)
|
||||
(config finalized-drv-config))
|
||||
|
||||
(define (finalized-drv-depfile drv)
|
||||
((finalized-drv-export-depfile drv)))
|
||||
|
||||
;; Takes a `<ninja-build-config>` representing a Nixpkgs derivation, and
|
||||
;; preprocesses the derivation such that it can be reconstituted once Zilch
|
||||
;; has taken over the Ninja build requirements.
|
||||
|
|
@ -86,7 +140,7 @@
|
|||
;; encodes many specific parts that are unlikely to be useful by external
|
||||
;; parties.
|
||||
;;
|
||||
;; Returns 6 values:
|
||||
;; Returns 7 values:
|
||||
;;
|
||||
;; - The initial Nixpkgs derivation as `<derivation>`
|
||||
;; - The Nixpkgs derivation after running all phases up to and including
|
||||
|
|
@ -94,7 +148,8 @@
|
|||
;; - An alist of output names to placeholder store paths
|
||||
;; - The `edge-ref`, `defaults`, and `export-depfile` values from calling
|
||||
;; `process-ninja-file`
|
||||
(define (setup-ninja-environment conf)
|
||||
;; - A list of store path -> store path rewrites.
|
||||
(define (setup-ninja-environment conf secondary-roots)
|
||||
(define initial-drv (ninja-build-config-environment-drv conf))
|
||||
(when (store-path? initial-drv)
|
||||
(set! initial-drv (store-path-drv initial-drv)))
|
||||
|
|
@ -102,18 +157,32 @@
|
|||
(define placeholders (map (lambda (output-and-info) (cons (car output-and-info) (make-fake-store-path name (car output-and-info)))) (derivation-outputs initial-drv)))
|
||||
|
||||
(define existing-env (ninja-build-config-environment conf))
|
||||
|
||||
(define rewrites (list))
|
||||
(define rewrite-extra "")
|
||||
(for-each
|
||||
(lambda (secondary-root)
|
||||
(mapping-for-each
|
||||
(lambda (orig-path virtual-path)
|
||||
(set! rewrites (cons (cons (string->utf8 orig-path) (string->utf8 virtual-path)) rewrites))
|
||||
(set! rewrite-extra (string-append rewrite-extra orig-path " " virtual-path "\n")))
|
||||
(drv-rewrite-source-paths secondary-root)))
|
||||
secondary-roots)
|
||||
|
||||
; Override output environment variables with our placeholders.
|
||||
(set-ninja-build-config-environment! conf #~,(map (lambda (v) (or (assoc (car v) placeholders) v)) #$existing-env))
|
||||
(set-ninja-build-config-environment! conf #~,(map (lambda (v) (or (assoc (car v) placeholders) (cons (car v) (rewrite-bytevector-or-string (cdr v) rewrites)))) #$existing-env))
|
||||
|
||||
; Take the initially requested .drv, replace its buildCommand, and set a single ("zilch_out") output path.
|
||||
(define configured-drv
|
||||
(cdar
|
||||
(patch-drv initial-drv
|
||||
(append
|
||||
`(("buildCommand" . ,configure-builder))
|
||||
`(("buildCommand" . ,configure-builder)
|
||||
("addInputsHook" . ,rewrite-hooks)
|
||||
("__zilch_rewrites" . ,rewrite-extra))
|
||||
placeholders)
|
||||
'("zilch_out"))))
|
||||
'("zilch_out")
|
||||
rewrites)))
|
||||
|
||||
; This VFS contains two directories: `src` (source tree) and `build` (Ninja build files).
|
||||
(define configured-vfs (vfs-from-store configured-drv))
|
||||
|
|
@ -141,10 +210,49 @@
|
|||
(read-ninja-file (read-file-at-path "build.ninja") read-file-at-path))
|
||||
|
||||
; Process the build.ninja file.
|
||||
(define-values (edge-ref defaults export-depfile) (process-ninja-file ninja-file conf "build"))
|
||||
(define-values (edge-ref defaults export-depfile) (process-ninja-file ninja-file conf "build" secondary-roots))
|
||||
|
||||
(values initial-drv configured-drv placeholders edge-ref defaults export-depfile))
|
||||
|
||||
(values initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites))
|
||||
|
||||
(define (process-secondary-root parent-conf secondary-root)
|
||||
(define conf (cdr secondary-root))
|
||||
(define drv-name (car secondary-root))
|
||||
|
||||
(define matching-drv #f)
|
||||
(let loop ((drv-list (derivation-input-drvs (ninja-build-config-environment-drv parent-conf))))
|
||||
(cond
|
||||
((null? drv-list) (error "Cannot find matching derivation for rewrite" drv-name))
|
||||
((string=? (derivation-name (caar drv-list)) drv-name)
|
||||
(set! matching-drv (caar drv-list)))
|
||||
(else (loop (cdr drv-list)))))
|
||||
(define (disc-name out-name)
|
||||
(if (string=? out-name "out") drv-name (string-append drv-name "-" out-name)))
|
||||
|
||||
(define future-rewrites (map (lambda (a) (cons (derivation-output-path (cdr a)) (make-virtual-path (disc-name (car a))))) (derivation-outputs matching-drv)))
|
||||
(define output-map (alist->mapping (make-default-comparator) (map (lambda (a) (cons (car a) (make-virtual-path (disc-name (car a))))) (derivation-outputs matching-drv))))
|
||||
; secondary-root is (name . #<ninja-build-config>)
|
||||
(define its-secondary-roots (map (cute process-secondary-root conf <>) (ninja-build-config-rewrites conf)))
|
||||
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites) (setup-ninja-environment conf its-secondary-roots))
|
||||
(define vfses (mapping-map (lambda (k v) (values k (mapping (make-default-comparator)))) (make-default-comparator) output-map))
|
||||
(define (transform-obj obj)
|
||||
(cond
|
||||
((and (pair? obj) (eq? (car obj) 'marker))
|
||||
(let ((obj-edge (edge-ref (cdr obj))))
|
||||
(or (and (cdr obj-edge) (built-edge-lib-placeholder (cdr obj-edge)))
|
||||
(car obj-edge))))
|
||||
(else obj)))
|
||||
(define extra-paths '())
|
||||
(mapping-for-each
|
||||
(lambda (key obj)
|
||||
(define base-store-path (mapping-ref output-map (car key)))
|
||||
(define full-path (string-append base-store-path "/" (if (string=? "" (cadr key)) (cddr key) (string-append (cadr key) "/" (cddr key)))))
|
||||
(if (or (string-suffix? ".hh" (cddr key)) (string-suffix? ".h" (cddr key)) (string-suffix? ".hpp" (cddr key)))
|
||||
(set! extra-paths (cons (cons full-path (transform-obj obj)) extra-paths))
|
||||
(set! vfses (mapping-set! vfses (car key) (mapping-set! (mapping-ref vfses (car key)) (cdr key) (transform-obj obj))))))
|
||||
(determine-data-flow conf))
|
||||
(set! vfses (mapping-map/monotone! (lambda (k v) (values k (vfs-to-store (make-vfs v)))) (make-default-comparator) vfses))
|
||||
(define finalized (delay (finalize-drv conf its-secondary-roots initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites)))
|
||||
(make-drv-rewrite output-map finalized vfses extra-paths))
|
||||
;; Takes a `<ninja-build-config>` representing a Nixpkgs derivation, and
|
||||
;; build it using Zilch.
|
||||
;;
|
||||
|
|
@ -156,8 +264,12 @@
|
|||
;; necessary inputs generated from the depfile data. This can be stored in
|
||||
;; a file for later rebuilds.
|
||||
(define (build-nixpkgs-drv-reproducibly conf)
|
||||
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf))
|
||||
(define secondary-roots (map (cute process-secondary-root conf <>) (ninja-build-config-rewrites conf)))
|
||||
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites) (setup-ninja-environment conf secondary-roots))
|
||||
(finalize-drv conf secondary-roots initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites))
|
||||
|
||||
;; "Finalizes" a previously setup Ninja environment, running the install and fixup hooks.
|
||||
(define (finalize-drv conf secondary-roots initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites)
|
||||
; Build all store paths necessary for installing. This assumes Meson.
|
||||
(define preinstall-state (force (built-edge-out-drv (cdr (edge-ref "all")))))
|
||||
|
||||
|
|
@ -182,6 +294,20 @@
|
|||
; This is necessary because these paths may be CA, and we can't guarantee daemon support for that past this point.
|
||||
(define realised-built (store-path-devirtualise preinstall-state))
|
||||
|
||||
; Take each output store path, then copy it over to the expected place
|
||||
; then later run a fixup to the original store path at placeholder state
|
||||
(define prepared-store-paths (list))
|
||||
(define rewrite-placeholder-fixes (list))
|
||||
(define (fixup-single-secondary-root single-drv-item)
|
||||
(define drv-name (virtual-path-for (store-path-path single-drv-item)))
|
||||
(set! prepared-store-paths (cons #~,(string-append "cp -rf " #$single-drv-item " " drv-name "\n") prepared-store-paths))
|
||||
(set! rewrite-placeholder-fixes (cons #~,(string-append "zilchFixPlaceholder " drv-name " " #$single-drv-item "\n") rewrite-placeholder-fixes)))
|
||||
(define (prepare-secondary-root secondary-root)
|
||||
; alist of output name -> store path
|
||||
(define realised (finalized-drv-output-store-paths (force (drv-rewrite-finalized-drv secondary-root))))
|
||||
(map (lambda (v) (fixup-single-secondary-root (cdr v))) realised))
|
||||
(for-each prepare-secondary-root secondary-roots)
|
||||
|
||||
; Prepare the post-build builder. This puts everything in its place and runs the post-build phases from the original .drv.
|
||||
(define postbuild-builder
|
||||
#~,(string-append
|
||||
|
|
@ -201,6 +327,7 @@
|
|||
"}\n"
|
||||
"zilchFixup() {\n"
|
||||
fix-placeholders-command
|
||||
#$(apply string-append #$rewrite-placeholder-fixes)
|
||||
copy-in-place-command
|
||||
"}\n"
|
||||
"zilchMesonInstall() {\n"
|
||||
|
|
@ -229,13 +356,15 @@
|
|||
"for curPhase in ${phases[*]}; do runPhase \"$curPhase\"; done\n"))
|
||||
|
||||
; Patch the original .drv to run the postbuild-builder command.
|
||||
(values
|
||||
(define store-paths
|
||||
(patch-drv initial-drv
|
||||
(append
|
||||
`(("buildCommand" . ,postbuild-builder))
|
||||
`(("buildCommand" . ,postbuild-builder)
|
||||
("addInputsHook" . ,#~,(apply string-append #$prepared-store-paths)))
|
||||
placeholders)
|
||||
(map car placeholders))
|
||||
export-depfile))
|
||||
(map car placeholders)
|
||||
rewrites))
|
||||
(make-finalized-drv store-paths export-depfile secondary-roots conf))
|
||||
|
||||
;; Build the derivation, but with stubbed out header and .so files.
|
||||
;; This is used to determine the dataflow, to make cross-project incremental
|
||||
|
|
@ -243,7 +372,7 @@
|
|||
;; Returns a single SRFI-146 mapping, containing keys of shape `(output . (dir . name))`,
|
||||
;; and values either a zexpr-y store path, or a pair `(marker . <name of the build output>)`
|
||||
(define (determine-data-flow conf)
|
||||
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf))
|
||||
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile rewrites) (setup-ninja-environment conf '()))
|
||||
|
||||
(define edges (built-edge-phony-inputs (cdr (edge-ref "all"))))
|
||||
|
||||
|
|
@ -325,7 +454,7 @@
|
|||
(append
|
||||
`(("buildCommand" . ,postbuild-builder))
|
||||
placeholders)
|
||||
(map car placeholders)))
|
||||
(map car placeholders) '()))
|
||||
(define (get-file-marker fptr)
|
||||
(call-with-port (store-path-open fptr)
|
||||
(lambda (p)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue