puck/zilch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

(zilch lang rust): more fixes

Browse source
This commit is contained in:
puck 2025-03-02 20:32:05 +00:00
parent 313cca66cc
commit fb1977a45c
5 changed files with 189 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

223
lang/rust/src/resolver.sld
View file

@ -19,8 +19,9 @@
<resolved-package> make-resolved-package resolved-package?
resolved-package-name resolved-package-version resolved-package-fs
resolved-package-cargo-target resolved-package-enabled-features resolved-package-dependencies
resolved-package-build-data
resolved-package-cargo-target resolved-package-enabled-features resolved-package-pending-features
resolved-package-dependencies
resolved-package-crate resolved-package-build-data
resolver-download
resolver-resolve-nonoptional
@ -36,34 +37,24 @@
(begin
(define gcc (delay (cdr (assoc "out" (nixpkgs "gcc")))))
(define linker (delay #~,(string-append #$(force gcc) "/bin/cc")))
(define pkgconfig (delay (cdr (assoc "out" (nixpkgs "pkg-config")))))
(define protobuf (delay (cdr (assoc "out" (nixpkgs "protobuf")))))
(define magic (delay (cdr (assoc "out" (nixpkgs "file")))))
(define openssl (delay (let ((data (nixpkgs "openssl"))) #~,(begin #$(cdr (assoc "out" data)) #$(cdr (assoc "dev" data))))))
(define tvix-protos (delay (vfs-to-store (vfs-from-directory "/home/nix/store/dkjgsrg8knn406qh86c3mbxpbz2rjwfy-tvix-all-protos"))))
(define (build-script-env-overrides-for-crate crate-name is-dependency)
(cond
((and (string=? crate-name "pkg-config") is-dependency)
#~(("PATH" . ,(string-append #$(force pkgconfig) "/bin"))))
((and (string=? crate-name "openssl-sys") (not is-dependency))
#~(("PATH" . ,(string-append #$(force gcc) "/bin")) ("PKG_CONFIG_PATH" . ,(string-append #$(force openssl) "/lib/pkgconfig"))))
((and (member crate-name '("ring" "bzip2-sys" "zstd-sys" "lzma-sys" "libmimalloc-sys") string=?) (not is-dependency))
#~(("PATH" . ,(string-append #$(force gcc) "/bin"))))
((and (member crate-name '("tvix-castore" "tvix-store" "tvix-build") string=?) (not is-dependency))
#~(("PATH" . ,(string-append #$(force protobuf) "/bin")) ("PROTO_ROOT" . #$(force tvix-protos))))
((or (string=? crate-name "prost-wkt-types") (string=? crate-name "nar-bridge"))
#~(("PATH" . ,(string-append #$(force protobuf) "/bin"))))
((or (string=? crate-name "magic-sys"))
#~(("NIX_LDFLAGS" . ,(string-append #$(force magic) "/lib"))))
(else '())))
(foreign-declare "#include \"false_source.h\"")
(define cargo-stub
(delay
(cdar
(call-rustc
(zfile (foreign-value "false_source" nonnull-c-string)) '()
#:codegen-flags (cons "linker" (force linker))
#:crate-type 'bin
#:crate-name "false"
#:edition "2021"
#:emits '(#:link #t)))))
; Used to select a set of crates plus their versions.
(define-record-type <resolver>
; locked-dependencies is a mapping of package-name to a mapping of version to (version . (lockfile-entry . unpack-promise))
; selected-dependencies is a mapping of package-name to a list of (version . resolved-package)(?)
; pending-features is a mapping of (package-name . version) to a list of features
(make-resolver locked-dependencies selected-dependencies)
resolver?
(locked-dependencies resolver-locked-dependencies set-resolver-locked-dependencies!)
@ -81,7 +72,7 @@
(build-script-out resolved-package-build-data-build-script-out))
(define-record-type <resolved-package>
(make-resolved-package name version fs cargo-target target-dependencies crate enabled-features dependencies build-data build-script)
(make-resolved-package name version fs cargo-target target-dependencies crate enabled-features pending-features dependencies build-data build-script)
resolved-package?
(name resolved-package-name)
(version resolved-package-version)
@ -90,6 +81,7 @@
(target-dependencies resolved-package-target-dependencies)
(crate resolved-package-crate)
(enabled-features resolved-package-enabled-features set-resolved-package-enabled-features!)
(pending-features resolved-package-pending-features set-resolved-package-pending-features!)
(dependencies resolved-package-dependencies set-resolved-package-dependencies!)
(build-data resolved-package-build-data set-resolved-package-build-data!)
(build-script resolved-package-build-script set-resolved-package-build-script!))
@ -100,16 +92,20 @@
(error "Resolver wanted non-versioned download" name))
(define dir (force (cddr (mapping-ref (mapping-ref (resolver-locked-dependencies resolver) name) (version-str version)))))
(define vfs (vfs-from-store dir))
(define-values (parsed-cargo parsed-workspace) (parse-cargo-toml vfs (call-with-port (store-path-open (vfs-file-ref vfs "" "Cargo.toml")) (lambda (p) (read-string 99999999 p))) #f))
(resolver-process resolver name vfs #f))
(define (resolver-process resolver name vfs workspace)
(define-values (parsed-cargo parsed-workspace) (parse-cargo-toml vfs (call-with-port (store-path-open (vfs-file-ref vfs "" "Cargo.toml")) (lambda (p) (read-string 99999999 p))) workspace))
(define version (parse-version (cargo-crate-version parsed-cargo)))
(unless (cargo-crate-lib-target parsed-cargo)
(error "Crate does not have valid [lib] target" (list name version)))
(error "Crate does not have valid [lib] target" (list name)))
(define build-script #f)
(when (cargo-crate-build-script parsed-cargo)
(set! build-script (make-resolved-package (string-append name "_build") version vfs (cargo-crate-build-script parsed-cargo) (cargo-crate-build-dependencies parsed-cargo) parsed-cargo '() (mapping (make-default-comparator)) #f #f))
(set! build-script (make-resolved-package (string-append name "_build") version vfs (cargo-crate-build-script parsed-cargo) (cargo-crate-build-dependencies parsed-cargo) parsed-cargo '() (mapping (make-default-comparator)) (mapping (make-default-comparator)) #f #f))
(resolver-resolve-nonoptional resolver build-script))
(define pkg (make-resolved-package (string-copy name) version vfs (cargo-crate-lib-target parsed-cargo) (cargo-crate-dependencies parsed-cargo) parsed-cargo '() (mapping (make-default-comparator)) #f build-script))
(define pkg (make-resolved-package (string-copy name) version vfs (cargo-crate-lib-target parsed-cargo) (cargo-crate-dependencies parsed-cargo) parsed-cargo '() (mapping (make-default-comparator)) (mapping (make-default-comparator)) #f build-script))
; Add package to the mapping.
(define existing-mapping (mapping-ref/default (resolver-selected-dependencies resolver) name '()))
@ -138,7 +134,12 @@
(set! resolved-dep (resolver-resolve resolver cargo-dep))
(set-resolved-package-dependencies! pkg (mapping-set! (resolved-package-dependencies pkg) name resolved-dep))
(when (cargo-dependency-default-features cargo-dep) (resolver-activate-features resolver resolved-dep '("default")))
(when (cargo-dependency-features cargo-dep) (resolver-activate-features resolver resolved-dep (cargo-dependency-features cargo-dep))))
(when (cargo-dependency-features cargo-dep) (resolver-activate-features resolver resolved-dep (cargo-dependency-features cargo-dep)))
(let ((pending-features (mapping-ref/default (resolved-package-pending-features pkg) name #f)))
(when pending-features
(set-resolved-package-pending-features! pkg (mapping-delete! (resolved-package-pending-features pkg) name))
(resolver-activate-features resolver resolved-dep pending-features))))
resolved-dep)
;; Activate a series of features on an existing <resolved-package>. This will resolve and activate optional dependencies
@ -155,31 +156,60 @@
; Follow each activation of the feature.
(for-each
(lambda (activation)
; TODO: if dep isn't activated and has optional dep, track it!
(let ((involved-dep (and (car activation) (resolver-resolve-resolved-package resolver resolved (car activation) (cadr activation)))))
(when (and (cddr activation) involved-dep) (resolver-activate-features resolver involved-dep (list (cddr activation))))
(when (and (not (car activation)) (cddr activation)) (resolver-activate-features resolver resolved (list (cddr activation))))))
(define target-package (car activation))
(define must-activate (cadr activation))
(define target-feature (cddr activation))
(define target-dependency (if target-package (resolver-resolve-resolved-package resolver resolved target-package must-activate) resolved))
;; NOTE: this is undocumented behavior by cargo
; if a feature is enabled that enables an optional package, a feature with that same name gets enabled.
; this is not entirely identical in behavior (TODO(puck): this is too wide a net) but it'll do, roughly.
(when (and target-dependency target-package) (resolver-activate-features resolver resolved (list target-package)))
(cond
((and target-feature target-dependency) (resolver-activate-features resolver target-dependency (list target-feature)))
; ((and (not target-feature) target-package target-dependency))) ; noop but activated
((and target-feature (not target-dependency))
(set-resolved-package-pending-features! resolved
(mapping-update!/default (resolved-package-pending-features resolved)
target-package
(lambda (lst) (if (member target-feature lst) lst (cons target-feature lst)))
'()))))
(define build-script (resolved-package-build-script resolved))
(when build-script
(let ((target-dependency-build (and target-package (resolver-resolve-resolved-package resolver build-script target-package must-activate))))
(when (and build-script target-dependency-build target-package) (resolver-activate-features resolver resolved (list target-package)))
(cond
((and target-feature target-dependency-build) (resolver-activate-features resolver target-dependency-build (list target-feature)))
((and target-feature (not target-dependency-build))
(set-resolved-package-pending-features! build-script
(mapping-update!/default (resolved-package-pending-features build-script)
target-package
(lambda (lst) (if (member target-feature lst) lst (cons target-feature lst)))
'())))))))
(cdr (or (assoc feature (cargo-crate-features (resolved-package-crate resolved))) (cons '() '()))))))
to-activate))
;; Register a non-registry crate+vfs with the resolver.
(define (resolver-register resolver vfs crate delayed)
(define target (cargo-crate-lib-target crate))
(unless target
(when (null? (cargo-crate-targets crate))
(error "Crate has _zero_ targets" crate))
(set! target (car (cargo-crate-targets crate))))
(resolver-register-target resolver vfs crate target #f delayed))
(cond
((target)
(list (resolver-register-target resolver vfs crate target #f delayed)))
((null? (cargo-crate-targets crate))
(error "Crate has _zero_ targets" crate))
(else
(map (lambda (target) (resolver-register-target resolver vfs crate target #f delayed)) (cargo-crate-targets crate)))))
;; Register a non-registry crate+vfs with the resolver.
(define (resolver-register-target resolver vfs crate target extra-dependencies delayed)
(define build-script #f)
(unless extra-dependencies
(set! extra-dependencies (mapping (make-default-comparator))))
(define version (parse-version (cargo-crate-version crate)))
(when (cargo-crate-build-script crate)
(set! build-script (make-resolved-package (string-append (cargo-target-name target) "_build") (parse-version (cargo-crate-version crate)) vfs (cargo-crate-build-script crate) (cargo-crate-build-dependencies crate) crate '() (mapping (make-default-comparator)) #f #f))
(resolver-resolve-nonoptional resolver build-script))
(define pkg (make-resolved-package (cargo-target-name target) (parse-version (cargo-crate-version crate)) vfs target (cargo-crate-dependencies crate) crate '() extra-dependencies #f build-script))
(set! build-script (make-resolved-package (string-append (cargo-target-name target) "_build") version vfs (cargo-crate-build-script crate) (cargo-crate-build-dependencies crate) crate '() (mapping (make-default-comparator)) (mapping (make-default-comparator)) #f #f))
(unless delayed (resolver-resolve-nonoptional resolver build-script)))
(define pkg (make-resolved-package (cargo-target-name target) version vfs target (cargo-crate-dependencies crate) crate '() (mapping (make-default-comparator)) extra-dependencies #f build-script))
(define existing-mapping (mapping-ref/default (resolver-selected-dependencies resolver) (cargo-crate-name crate) '()))
(unless (equal? 'bin (cargo-target-crate-type target))
(set-resolver-selected-dependencies! resolver (mapping-set (resolver-selected-dependencies resolver) (cargo-crate-name crate) (cons (cons (parse-version (cargo-crate-version crate)) pkg) existing-mapping))))
@ -201,7 +231,9 @@
(else (find-matching-version (cdr l) best-version))))
(define matching-version (find-matching-version existing-mapping #f))
(unless (or matching-version available-versions)
(error "Resolving ~S: could not find matching dep for reqs ~S in ~S\n" (list package-name requirements existing-mapping)))
(when (cargo-dep-path? (cargo-dependency-origin dep))
(error "unknown path dependency" dep))
(error (sprintf "Resolving ~S: could not find matching dep for reqs ~S in ~S\n" dep requirements existing-mapping)))
(if matching-version
(cdr matching-version)
(let* ((best-version (mapping-fold/reverse (lambda (k v acc) (if (or acc (not (matches-requirements (car v) requirements))) acc (car v))) #f available-versions))
@ -232,6 +264,11 @@
(for-each
(lambda (pair)
(resolver-print-pkg resolver (cdr pair)))
v)
(printf "Package ~S build scripts:\n" k)
(for-each
(lambda (pair)
(when (resolved-package-build-script (cdr pair)) (resolver-print-pkg resolver (resolved-package-build-script (cdr pair)))))
v))
(resolver-selected-dependencies resolver)))
@ -249,10 +286,12 @@
parsed-lockfile)
(define resolver (make-resolver locked-dependencies (mapping (make-default-comparator))))
(define pkg (resolver-register resolver vfs cargo-file #f))
(resolver-activate-features resolver pkg activated-features)
(resolver-print resolver)
pkg)
(define pkgs (resolver-register resolver vfs cargo-file #f))
(for-each
(lambda (pkg)
(resolver-activate-features resolver pkg activated-features))
pkgs)
pkgs)
(define (process-many-with-lockfile vfs-cargo-map parsed-lockfile)
(define locked-dependencies (mapping (make-default-comparator)))
@ -270,16 +309,35 @@
(define resolver (make-resolver locked-dependencies (mapping (make-default-comparator))))
(define pkgs '())
; Resolve each target inside the primary crates.
; Each crate may have any amount of targets; lib targets here are treated
; slightly specially as they are the ones used to link against other crates.
(for-each
(lambda (pkg-and-vfs)
(when (cargo-crate-lib-target (car pkg-and-vfs))
(set! pkgs (cons (resolver-register-target resolver (cdr pkg-and-vfs) (car pkg-and-vfs) (cargo-crate-lib-target (car pkg-and-vfs)) #f #t) pkgs)))
(when (and (pair? (cargo-crate-targets (car pkg-and-vfs))) (equal? 'bin (cargo-target-crate-type (car (cargo-crate-targets (car pkg-and-vfs))))))
(set! pkgs (cons (resolver-register-target resolver (cdr pkg-and-vfs) (car pkg-and-vfs) (car (cargo-crate-targets (car pkg-and-vfs)))
(if (cargo-crate-lib-target (car pkg-and-vfs)) (mapping (make-default-comparator) (cargo-target-name (cargo-crate-lib-target (car pkg-and-vfs))) (car pkgs)) (mapping (make-default-comparator))) #t) pkgs))))
(lambda (crate-and-vfs)
(define crate (car crate-and-vfs))
(define vfs (cdr crate-and-vfs))
(define lib-crate #f)
(when (cargo-crate-lib-target crate)
(set! lib-crate (resolver-register-target resolver vfs crate (cargo-crate-lib-target crate) #f #t))
(set! pkgs (cons lib-crate pkgs)))
(for-each
(lambda (target)
(when (equal? 'bin (cargo-target-crate-type target))
; A binary crate may refer to its lib crate by that name.
; Pass an override for dependencies if that is the case.
(set! pkgs (cons (resolver-register-target resolver vfs crate target
(and (cargo-crate-lib-target crate) (mapping (make-default-comparator) (cargo-target-name (cargo-crate-lib-target crate)) lib-crate))
#t) pkgs))))
(cargo-crate-targets crate)))
vfs-cargo-map)
(for-each (lambda (p) (resolver-resolve-nonoptional resolver p)) pkgs)
(for-each (lambda (p) (resolver-activate-features resolver p '("default"))) pkgs)
; Resolve all non-optional dependencies of the targets we've accepted.
(for-each (lambda (p) (resolver-resolve-nonoptional resolver p) (when (resolved-package-build-script p) (resolver-resolve-nonoptional resolver (resolved-package-build-script p)))) pkgs)
; Enable default features on all binary targets. This hopefully resolves into a set of valid features being set on the full project.
(for-each (lambda (p) (when (eq? (cargo-target-crate-type (resolved-package-cargo-target p)) 'bin) (resolver-activate-features resolver p '("default")))) pkgs)
; Print our resolved package set.
(resolver-print resolver)
pkgs)
@ -303,7 +361,7 @@
(reverse envs))
(mapping-map->list (lambda (k v) (cons k v)) final-env))
(define (build-package resolved)
(define (build-package resolved build-script-env-overrides compiler-env-overrides)
; Info we need to collect:
; - enabled features
; - dependencies
@ -340,7 +398,7 @@
(define rustc-env
#~(
; ("CARGO" . "")
("CARGO" . #$(force cargo-stub))
("CARGO_MANIFEST_DIR" . #$crate-root)
,@(if (cargo-crate-links (resolved-package-crate resolved)) (list (cons "CARGO_MANIFEST_LINKS" (cargo-crate-links (resolved-package-crate resolved)))) '())
("CARGO_PKG_VERSION" . ,(version-str (resolved-package-version resolved)))
@ -357,7 +415,8 @@
("CARGO_PKG_LICENSE_FILE" . "")
("CARGO_PKG_RUST_VERSION" . "")
("CARGO_PKG_README" . "")
("CARGO_CRATE_NAME" . ,crate-name)))
("CARGO_CRATE_NAME" . ,crate-name)
#$@(compiler-env-overrides (cargo-crate-name (resolved-package-crate resolved)))))
; CARGO_BIN_NAME, CARGO_BIN_EXE_*: skipping for now
; CARGO_PRIMARY_PACKAGE: not sensible here
; CARGO_TARGET_TMPDIR: integration/benchmark only
@ -379,24 +438,36 @@
(else out)))
(define build-script-envs #f)
(define seen-crates '())
(define (transitively-check-package pkg)
(define name (cargo-crate-name (resolved-package-crate pkg)))
(unless (member name seen-crates)
(set! seen-crates (cons name seen-crates))
(let ((env-override (build-script-env-overrides name #t)))
(when env-override
(set! build-script-envs (cons env-override build-script-envs))))
(mapping-for-each (lambda (k v) (transitively-check-package v)) (resolved-package-dependencies pkg))))
(when (resolved-package-build-script resolved)
;; "build" here is a misnomer; it's handling the .drv:s.
(unless (resolved-package-build-data (resolved-package-build-script resolved))
(build-package (resolved-package-build-script resolved)))
(build-package (resolved-package-build-script resolved) build-script-env-overrides compiler-env-overrides))
(set! build-script-envs (list (build-script-env-overrides-for-crate (cargo-crate-name (resolved-package-crate resolved)) #f)))
;; Process the transitive build dependencies for the build script, and set env overrides
;; based on them.
(set! build-script-envs (list (build-script-env-overrides (cargo-crate-name (resolved-package-crate resolved)) #f)))
(mapping-for-each
(lambda (key value)
(transitively-check-package value))
(resolved-package-dependencies (resolved-package-build-script resolved)))
; For each package dependency, check if it has "links" metadata, as well as build script env overrides.
; For each package dependency, check if it has "links" metadata.
; Crate that immediately depend on other crates that have this metadata have
; the cargo::metadata pairs passed to the build script.
(mapping-for-each
(lambda (key value)
(unless (resolved-package-build-data value)
(build-package value))
(let ((env-override (build-script-env-overrides-for-crate (cargo-crate-name (resolved-package-crate value)) #t)))
(when env-override
(set! build-script-envs (cons env-override build-script-envs))))
(build-package value build-script-env-overrides compiler-env-overrides))
(when (cargo-crate-links (resolved-package-crate value))
; Track (link-name . build-data) for each crate in immediate dependencies that applies
(set! crate-links (cons (cons (cargo-crate-links (resolved-package-crate value)) (resolved-package-build-data value)) crate-links))))
@ -436,8 +507,11 @@
(mapping-for-each
(lambda (key value)
(unless (resolved-package-build-data value)
(build-package value))
(for-each (lambda (dep) (unless (member dep transitive-dependencies) (set! transitive-dependencies (cons dep transitive-dependencies)))) (resolved-package-build-data-transitive-dependencies (resolved-package-build-data value)))
(build-package value build-script-env-overrides compiler-env-overrides))
(for-each
(lambda (dep)
(unless (member dep transitive-dependencies) (set! transitive-dependencies (cons dep transitive-dependencies))))
(resolved-package-build-data-transitive-dependencies (resolved-package-build-data value)))
(unless (member value transitive-dependencies) (set! transitive-dependencies (cons value transitive-dependencies)))
(define data (resolved-package-build-data value))
(define meta-or-rlib (or (resolved-package-build-data-metadata data) (resolved-package-build-data-rlib data)))
@ -475,7 +549,12 @@
all-crate-features)))
(cargo-crate-features (resolved-package-crate resolved)))
(set! params `(check-cfg: ,(string-append "cfg(feature, values(" (string-join all-crate-features ",") "))") . ,params))
(set! params `(check-cfg: ,(string-append "cfg(feature, values(" (string-join all-crate-features ",") "))")
check-cfg: "cfg(docsrs,test)"
cap-lints: "warn"
. ,params))
(for-each (lambda (check) (set! params `(check-cfg: ,check . ,params))) (cargo-crate-check-cfg-lint (resolved-package-crate resolved)))
(define inherited-build-script-out '())
(define transitive-bin-flags '())
@ -488,7 +567,7 @@
(unless (eq? crate-type 'rlib)
(set! params `(codegen-flags: ("linker" . ,(force linker)) . ,params)))
(define path #~,(string-append #$(vfs-to-store (resolved-package-fs resolved)) "/" (cargo-target-path (resolved-package-cargo-target resolved))))
(define dep-info (cdar (apply call-rustc `(,path ,rustc-env search-path: ("dependency" . ,transitive-dependencies-meta) emits: (dep-info: #t) . ,params-meta))))
@ -503,12 +582,10 @@
; (when (or (eq? crate-type 'proc-macro) (eq? crate-type 'bin))
; (set! params (append transitive-bin-flags params)))
(when (eq? crate-type 'bin)
(set! params `(#:codegen-flags ("rpath" . "no") . ,params))
(when (string=? crate-name "tvix_cli")
(set! params `(search-path: ("native" . ,#~,(string-append #$(force magic) "/lib")) . ,params))))
(set! params `(#:codegen-flags ("rpath" . "no") . ,params)))
; Rust is nitpicky about the filenames, fix them with copious symlinking.
(define rlib-file (cdar (apply call-rustc `(,path ,rustc-env search-path: ("dependency" . ,transitive-dependencies-rlib) emits: (link: #t) ,@bin-flags . ,params))))
(define rlib-file (cdar (apply call-rustc `(,path (("_zilch_inherit" . ,#~,(string-join #$inherited-build-script-out " ")) . ,rustc-env) search-path: ("dependency" . ,transitive-dependencies-rlib) emits: (link: #t) ,@bin-flags . ,params))))
(define rlib (cons rlib-name #~,(string-append #$(zdir rlib-name (zsymlink rlib-file)) "/" rlib-name)))
(store-path-materialize rlib-file)