rust: various build script fixes

This commit is contained in:
puck 2025-02-12 13:12:04 +00:00
parent e42315e67a
commit 8c129e33db
12 changed files with 307 additions and 165 deletions

View file

@ -11,39 +11,9 @@
(zilch lang rust))
(export
<build-script-output>
make-build-script-output build-script-output?
build-script-output-rerun-if-changed build-script-output-rerun-if-env-changed
build-script-output-link-arg build-script-output-link-lib build-script-output-link-search
build-script-output-flags build-script-output-cfg build-script-output-check-cfg
build-script-output-env build-script-output-warning build-script-output-metadata
call-runner)
(begin
(define-record-type <build-script-output>
(make-build-script-output rerun-if-changed rerun-if-env-changed link-arg link-lib link-search flags cfg check-cfg env warning metadata)
build-script-output?
(rerun-if-changed build-script-output-rerun-if-changed set-build-script-output-rerun-if-changed!)
(rerun-if-env-changed build-script-output-rerun-if-env-changed set-build-script-output-rerun-if-env-changed!)
(link-arg build-script-output-link-arg set-build-script-output-link-arg!)
(link-lib build-script-output-link-lib set-build-script-output-link-lib!)
(link-search build-script-output-link-search set-build-script-output-link-search!)
(flags build-script-output-flags set-build-script-output-flags!)
(cfg build-script-output-cfg set-build-script-output-cfg!)
(check-cfg build-script-output-check-cfg set-build-script-output-check-cfg!)
(env build-script-output-env set-build-script-output-env!)
(warning build-script-output-warning set-build-script-output-warning!)
(metadata build-script-output-metadata set-build-script-output-metadata!))
(define-record-printer (<build-script-output> entry out)
(fprintf out "#<build-script-output changed:~S env-changed:~S flags:~S cfg:~S check-cfg:~S env:~S>"
(build-script-output-rerun-if-changed entry)
(build-script-output-rerun-if-env-changed entry)
(build-script-output-flags entry)
(build-script-output-cfg entry)
(build-script-output-check-cfg entry)
(build-script-output-env entry)))
(define linker (delay (let ((v (cdr (assoc "out" (nixpkgs "gcc"))))) #~,(string-append #$v "/bin/cc"))))
(foreign-declare "#include \"runner_source.h\"")
@ -57,76 +27,13 @@
#:edition "2021"
#:emits '(#:link #t))))
(define (parse-build-script-line line out)
;; Rewrite cargo:foo -> cargo::foo
(when (and (string-prefix? "cargo:" line) (not (string-prefix? "cargo::" line)))
(set! line (string-append "cargo::" (string-copy line 6))))
(cond
((string-prefix? "cargo::rerun-if-changed=" line)
(set-build-script-output-rerun-if-changed! out (cons (string-copy line 24) (build-script-output-rerun-if-changed out))))
((string-prefix? "cargo::rerun-if-env-changed=" line)
(set-build-script-output-rerun-if-env-changed! out (cons (string-copy line 28) (build-script-output-rerun-if-env-changed out))))
((string-prefix? "cargo::rustc-flags=" line)
(set-build-script-output-flags! out (cons (string-copy line 19) (build-script-output-flags out))))
((string-prefix? "cargo::rustc-cfg=" line)
(let* ((kv (string-copy line 17))
(splat-start (and (string-suffix? "\"" kv) (string-contains kv "=\""))))
(set-build-script-output-cfg! out
(cons
(if splat-start
(cons (string-copy kv 0 splat-start) (string-copy kv (+ splat-start 2) (- (string-length kv) 1)))
kv)
(build-script-output-cfg out)))))
((string-prefix? "cargo::rustc-check-cfg=" line)
(set-build-script-output-check-cfg! out (cons (string-copy line 23) (build-script-output-check-cfg out))))
((string-prefix? "cargo::rustc-env=" line)
(let* ((kv (string-copy line 17))
(splat-start (string-contains kv "=")))
(set-build-script-output-env! out
(cons
(cons (string-copy kv 0 splat-start) (string-copy kv (+ splat-start 1)))
(build-script-output-env out)))))
((string-prefix? "cargo::metadata=" line)
(let* ((kv (string-copy line 16))
(splat-start (string-contains kv "=")))
(set-build-script-output-metadata! out
(cons
(cons (string-copy kv 0 splat-start) (string-copy kv (+ splat-start 1)))
(build-script-output-metadata out)))))
((string-prefix? "cargo::rustc-link-search=" line)
(let* ((kv (string-copy line 25))
(splat-start (string-contains kv "=")))
(set-build-script-output-link-search! out
(cons
(if splat-start (cons (string-copy kv 0 splat-start) (string-copy kv (+ splat-start 1)))
kv)
(build-script-output-link-search out)))))
((string-prefix? "cargo::rustc-link-lib=" line)
(set-build-script-output-link-lib! out
(cons
(string-copy line 22)
(build-script-output-link-lib out))))
; TODO(puck): bad
((string-prefix? "cargo::" line)
(let* ((kv (string-copy line 7))
(splat-start (string-contains kv "=")))
(set-build-script-output-metadata! out
(cons
(cons (string-copy kv 0 splat-start) (string-copy kv (+ splat-start 1)))
(build-script-output-metadata out)))))))
;; TODO: link-arg-*, warning, others?
(define (parse-build-script-output port)
(define out (make-build-script-output '() '() '() '() '() '() '() '() '() '() '()))
(define (tick)
(define line (read-line port))
(if (eof-object? line)
out
(begin (parse-build-script-line line out) (tick))))
(tick))
(define (call-runner input-script cwd env)
(define output (store-path-for-ca-drv* "build.rs-run" "x86_64-linux" #~(#$runner-runner) #~(("script" . #$input-script) ("cwd" . #$cwd) ("OUT_DIR" . ,(make-placeholder "outdir")) . #$env) '("out" "outdir")))
(values (call-with-port (store-path-open (cdr (assoc "out" output))) parse-build-script-output) (cdr (assoc "outdir" output))))))
(define output (store-path-for-ca-drv* "build.rs-run" "x86_64-linux"
`(,runner-runner)
#~(("script" . #$input-script)
("cwd" . #$cwd)
("OUT_DIR" . ,(make-placeholder "outdir"))
. #$env)
'("out" "outdir")))
(values (cdr (assoc "outdir" output)) (cdr (assoc "out" output))))))

View file

@ -16,7 +16,7 @@
<resolved-package-build-data> make-resolved-package-build-data resolved-package-build-data?
resolved-package-build-data-dep-info resolved-package-build-data-metadata resolved-package-build-data-rlib
resolved-package-build-data-transitive-dependencies
<resolved-package> make-resolved-package resolved-package?
resolved-package-name resolved-package-version resolved-package-fs
resolved-package-cargo-target resolved-package-enabled-features resolved-package-dependencies
@ -38,12 +38,23 @@
(define linker (delay #~,(string-append #$(force gcc) "/bin/cc")))
(define pkgconfig (delay (cdr (assoc "out" (nixpkgs "pkg-config")))))
(define protobuf (delay (cdr (assoc "out" (nixpkgs "protobuf")))))
(define magic (delay (cdr (assoc "out" (nixpkgs "file")))))
(define openssl (delay (let ((data (nixpkgs "openssl"))) #~,(begin #$(cdr (assoc "out" data)) #$(cdr (assoc "dev" data))))))
(define tvix-protos (delay (vfs-to-store (vfs-from-directory "/home/nix/store/dkjgsrg8knn406qh86c3mbxpbz2rjwfy-tvix-all-protos"))))
(define (build-script-env-overrides-for-crate crate-name)
(cond
((string=? crate-name "openssl-sys")
#~(("PATH" . ,(string-append #$(force pkgconfig) "/bin:" #$(force gcc) "/bin")) ("PKG_CONFIG_PATH" . ,(string-append #$(force openssl) "/lib/pkgconfig"))))
((or (string=? crate-name "ring") (string=? crate-name "bzip2-sys") (string=? crate-name "zstd-sys") (string=? crate-name "lzma-sys") (string=? crate-name "libmimalloc-sys"))
#~(("PATH" . ,(string-append #$(force gcc) "/bin"))))
((or (string=? crate-name "tvix-castore") (string=? crate-name "tvix-store") (string=? crate-name "tvix-build"))
#~(("PATH" . ,(string-append #$(force protobuf) "/bin")) ("PROTO_ROOT" . #$(force tvix-protos))))
((or (string=? crate-name "prost-wkt-types") (string=? crate-name "nar-bridge"))
#~(("PATH" . ,(string-append #$(force protobuf) "/bin"))))
((or (string=? crate-name "magic-sys"))
#~(("NIX_LDFLAGS" . ,(string-append #$(force magic) "/lib"))))
(else '())))
@ -57,14 +68,15 @@
(selected-dependencies resolver-selected-dependencies set-resolver-selected-dependencies!))
(define-record-type <resolved-package-build-data>
(make-resolved-package-build-data dep-info metadata rlib transitive-dependencies build-script-metadata bin-flags)
(make-resolved-package-build-data dep-info metadata rlib transitive-dependencies build-script-metadata bin-flags build-script-out)
resolved-package-build-data?
(dep-info resolved-package-build-data-dep-info set-resolved-package-build-data-dep-info!)
(metadata resolved-package-build-data-metadata set-resolved-package-build-data-metadata!)
(rlib resolved-package-build-data-rlib set-resolved-package-build-data-rlib!)
(transitive-dependencies resolved-package-build-data-transitive-dependencies set-resolved-package-build-data-transitive-dependencies!)
(build-script-metadata resolved-package-build-data-build-script-metadata)
(bin-flags resolved-package-build-data-bin-flags))
(bin-flags resolved-package-build-data-bin-flags)
(build-script-out resolved-package-build-data-build-script-out))
(define-record-type <resolved-package>
(make-resolved-package name version fs cargo-target target-dependencies crate enabled-features dependencies build-data build-script)
@ -291,6 +303,8 @@
(when (list? crate-type)
(set! crate-type 'rlib))
(define buildscript-metadata '())
(define buildscript-out #f)
(define crate-links '())
(define dependency-metadata '())
(define bin-flags '())
@ -327,7 +341,7 @@
("CARGO_PKG_RUST_VERSION" . "")
("CARGO_PKG_README" . "")
("CARGO_CRATE_NAME" . ,crate-name)))
; CARGO_BIN_NAME, OUT_DIR, CARGO_BIN_EXE_*: skipping for now
; CARGO_BIN_NAME, CARGO_BIN_EXE_*: skipping for now
; CARGO_PRIMARY_PACKAGE: not sensible here
; CARGO_TARGET_TMPDIR: integration/benchmark only
; CARGO_RUSTC_CURRENT_DIR: nightly only
@ -348,23 +362,28 @@
(else out)))
(when (resolved-package-build-script resolved)
;; "build" here is a misnomer; it's handling the .drv:s.
(unless (resolved-package-build-data (resolved-package-build-script resolved))
(build-package (resolved-package-build-script resolved)))
; For each package dependency, check if it has "links" metadata.
; Crate that immediately depend on other crates that have this metadata have
; the cargo::metadata pairs passed to the build script.
(mapping-for-each
(lambda (key value)
(unless (resolved-package-build-data value)
(build-package value))
(when (cargo-crate-links (resolved-package-crate value))
(for-each
(lambda (kv)
(set! dependency-metadata (cons (cons (string-map upcase-underscore (string-append "DEP_" (cargo-crate-links (resolved-package-crate value)) "_" (car kv))) (cdr kv)) dependency-metadata)))
(resolved-package-build-data-build-script-metadata (resolved-package-build-data value)))))
; Track (link-name . build-data) for each crate in immediate dependencies that applies
(set! crate-links (cons (cons (cargo-crate-links (resolved-package-crate value)) (resolved-package-build-data value)) crate-links))))
(resolved-package-dependencies resolved))
; Collect the necessary bits, and build the build script.
(let*-values
(((build-script) (cdr (resolved-package-build-data-rlib (resolved-package-build-data (resolved-package-build-script resolved)))))
((build-script-env) (build-script-env-overrides-for-crate (cargo-crate-name (resolved-package-crate resolved))))
((rewritten-features) (map (lambda (feature) (cons (string-map upcase-underscore (string-append "CARGO_FEATURE_" feature)) "")) (resolved-package-enabled-features resolved)))
((runner-output runner-outdir)
((runner-outdir runner-outpath)
(call-runner build-script crate-root
#~(
("RUSTC" . ,(string-append #$rustc "/bin/rustc"))
@ -373,40 +392,21 @@
("OPT_LEVEL" . "0")
("PROFILE" . "debug")
("DEBUG" . "true")
("_zilch_links" . ,(string-join #$(map (lambda (kv) #~,(string-append #$(resolved-package-build-data-build-script-out (cdr kv)) ":" (car kv))) crate-links) "!"))
,@(make-cfg-values-env cfg-values '())
,@dependency-metadata
,@rewritten-features
#$@build-script-env
; TODO: OUT_DIR, NUM_JOBS, OPT_LEVEL/DEBUG/PROFILE, DEP_*
; TODO: OUT_DIR, NUM_JOBS, OPT_LEVEL/DEBUG/PROFILE
; RUSTC/RUSTDOC?, RUSTC_LINKER? and CARGO_ENCODED_RUSTFLAGS
. #$rustc-env))))
(map
(lambda (v)
(if (pair? v)
(set! params `(#:cfg ,(string-append (car v) "=\"" (cdr v) "\"") . ,params))
(set! params `(#:cfg ,v . ,params))))
(build-script-output-cfg runner-output))
(set! buildscript-metadata (build-script-output-metadata runner-output))
(for-each (lambda (kv) (set! rustc-env (cons kv rustc-env))) (build-script-output-env runner-output))
; Pass the buildscript-out data to the next crate.
(set! buildscript-out runner-outpath)
; Pass cargo::rustc-cfg and cargo::rustc-check-cfg to the crate this build script belongs to.
; Also set OUT_DIR for include! and include_str! reasons.
(let ((old-rustc-env rustc-env))
(set! rustc-env #~(("OUT_DIR" . #$runner-outdir) . #$old-rustc-env)))
(for-each
(lambda (kv) (set! bin-flags `(#:link ,kv . ,bin-flags)))
(build-script-output-link-lib runner-output))
; TODO(puck): hack to workaround lack of store path passthrough
; This should be replaced with .... $something (a dir of all build script outputs?)
(unless (or (null? build-script-env) (null? bin-flags))
(let ((v (cadr bin-flags)))
(set-cdr! bin-flags (cons #~,(begin #$build-script-env #$runner-outdir v) (cddr bin-flags)))))
(for-each
(lambda (kv) (set! bin-flags `(#:search-path ,kv . ,bin-flags)))
(build-script-output-link-search runner-output))))
; TODO(puck): check-cfg wants check-cfg everywhere
;(map
; (lambda (v)
; (set! params `(#:check-cfg ,v . ,params)))
; (build-script-output-check-cfg runner-output))))
(set! rustc-env #~(("OUT_DIR" . #$runner-outdir) ("_zilch_proc" . #$runner-outpath) . #$old-rustc-env)))))
(define params-meta params)
(define transitive-dependencies '())
@ -443,8 +443,25 @@
(car rlib)
(zsymlink (cdr rlib)))) transitive-dependencies)))
(define all-crate-features '())
(for-each
(lambda (feat)
(set! all-crate-features
(cons
(string-append "\"" (car feat) "\"")
all-crate-features)))
(cargo-crate-features (resolved-package-crate resolved)))
(set! params `(check-cfg: ,(string-append "cfg(feature, values(" (string-join all-crate-features ",") "))") . ,params))
(define inherited-build-script-out '())
(define transitive-bin-flags '())
(for-each (lambda (dep) (set! transitive-bin-flags (append (resolved-package-build-data-bin-flags (resolved-package-build-data dep)) transitive-bin-flags))) transitive-dependencies)
(for-each
(lambda (dep)
(set! transitive-bin-flags (append (resolved-package-build-data-bin-flags (resolved-package-build-data dep)) transitive-bin-flags))
(when (resolved-package-build-data-build-script-out (resolved-package-build-data dep))
(set! inherited-build-script-out (cons (resolved-package-build-data-build-script-out (resolved-package-build-data dep)) inherited-build-script-out))))
transitive-dependencies)
(unless (eq? crate-type 'rlib)
(set! params `(codegen-flags: ("linker" . ,(force linker)) . ,params)))
@ -460,10 +477,12 @@
(when (eq? crate-type 'bin)
(set! rlib-name crate-name))
(when (or (eq? crate-type 'proc-macro) (eq? crate-type 'bin))
(set! params (append transitive-bin-flags params)))
; (when (or (eq? crate-type 'proc-macro) (eq? crate-type 'bin))
; (set! params (append transitive-bin-flags params)))
(when (eq? crate-type 'bin)
(set! params `(#:codegen-flags ("rpath" . "no") . ,params)))
(set! params `(#:codegen-flags ("rpath" . "no") . ,params))
(when (string=? crate-name "tvix_cli")
(set! params `(search-path: ("native" . ,#~,(string-append #$(force magic) "/lib")) . ,params))))
; Rust is nitpicky about the filenames, fix them with copious symlinking.
(define rlib-file (cdar (apply call-rustc `(,path ,rustc-env search-path: ("dependency" . ,transitive-dependencies-rlib) emits: (link: #t) ,@bin-flags . ,params))))
@ -477,7 +496,7 @@
(set! metadata (cons rmeta-name #~,(string-append #$(zdir rmeta-name (zsymlink metadata-file)) "/" rmeta-name)))
(store-path-materialize metadata-file))
(set-resolved-package-build-data! resolved (make-resolved-package-build-data dep-info metadata rlib transitive-dependencies buildscript-metadata bin-flags))
(set-resolved-package-build-data! resolved (make-resolved-package-build-data dep-info metadata rlib transitive-dependencies buildscript-metadata bin-flags buildscript-out))
rlib-file)
(define (matches-requirements ver req)

View file

@ -11,6 +11,9 @@
(begin
(define rustc (cdr (assoc "out" (nixpkgs "rustc"))))
(define gcc (delay (cdr (assoc "out" (nixpkgs "gcc")))))
(define linker (delay #~,(string-append #$(force gcc) "/bin/cc")))
(define-record-type <rustc-emits>
(make-rustc-emits asm llvm-bc llvm-ir obj metadata link dep-info mir)
rustc-emits?
@ -87,6 +90,10 @@
((#:remap-path-prefix) (set-rustc-params-remap-path-prefix! out (cons (cadr items) (rustc-params-remap-path-prefix out))) (parse-rustc-params out (cddr items)))
(else (error "unknown rustc param" (car items))))))
(foreign-declare "#include \"rustc_wrap_source.h\"")
(define rustc_wrap-bin
#f)
(define (call-rustc input env . params)
(call-rustc-internal input env (parse-rustc-params (make-rustc-params '() '() '() '() #f #f #f #f '() '() '()) params)))
@ -136,5 +143,16 @@
(lambda (k)
(set! args (cons "--remap-path-prefix" (cons #~,(string-append #$(car k) "=" #$(cdr k)) args))))
(rustc-params-remap-path-prefix params)))
(store-path-for-ca-drv* (string-append "rustc-" (symbol->string (rustc-params-crate-type params)) "-" (rustc-params-crate-name params)) "x86_64-linux" #~(,(string-append #$rustc "/bin/rustc") . #$args) env outputs))))
(if rustc_wrap-bin
(store-path-for-ca-drv* (string-append "rustc-" (symbol->string (rustc-params-crate-type params)) "-" (rustc-params-crate-name params)) "x86_64-linux" #~(#$rustc_wrap-bin . #$args) #~(("_zilch_rustc" . ,(string-append #$rustc "/bin/rustc")) . #$env) outputs)
(store-path-for-ca-drv* (string-append "rustc-" (symbol->string (rustc-params-crate-type params)) "-" (rustc-params-crate-name params)) "x86_64-linux" #~(,(string-append #$rustc "/bin/rustc") . #$args) env outputs)))
(set! rustc_wrap-bin
(cdar
(call-rustc
(zfile (foreign-value "rustc_wrap_source" nonnull-c-string)) '()
#:codegen-flags (cons "linker" (force linker))
#:crate-type 'bin
#:crate-name "rustc_wrap"
#:edition "2021"
#:emits '(#:link #t))))))