diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index cbc110c..1e4d158 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -10,6 +10,10 @@ *** xref:go/usage.adoc[] *** xref:go/library.adoc[] *** xref:go/man.adoc[Man page] +** Rust +*** xref:rust/usage.adoc[] +*** xref:rust/library.adoc[] +*** xref:rust/man.adoc[Man page] * Code reference include::generated:partial$nav.adoc[] diff --git a/docs/modules/ROOT/pages/rust/library.adoc b/docs/modules/ROOT/pages/rust/library.adoc new file mode 100644 index 0000000..262c904 --- /dev/null +++ b/docs/modules/ROOT/pages/rust/library.adoc @@ -0,0 +1,179 @@ += Zilch's Rust support +:page-pagination: + +Zilch supports using the Rust compiler directly from Scheme. This can be used +to compile single crates, or to drive the crate support more directly. + +== Compiling Rust code +To compile Rust code, xref:generated:zilch.lang.rust.adoc[`(zilch lang rust)`] +provides a `call-rustc` procedure, which handles most of the code necessary to +compile Rust code. + +[,scheme,line-comment=;] +---- +(define gcc (cdr (assoc "out" (nixpkgs "gcc")))) ; <1> + +(define outputs + (call-rustc + (zfile "fn main() { println!(\"Hello, World!\"); }") ; <2> + '() ; <3> + crate-type: 'bin + crate-name: "hello" + edition: "2021" + emits: '(dep-info: #t link: #t) ; <4> + codegen-flags: (cons "linker" #~,(string-append #$gcc "/bin/cc")))) ; <5> +;; (("dep-info" +;; . #) +;; ("link" +;; . #)) +---- +<1> Get a copy of the `out` output for `gcc` in nixpkgs. +<2> Use a file defined inline to compile. +<3> Don't add any variables to the environment. +<4> Make `rustc` only emit the `dep-info` and `link` outputs. +<5> Tell `rustc` where to find the linker. + +The resulting binary is linked dynamically, using the linker passed either as +`codegen-flag`, or findable as `cc` on `$PATH`. By default, each `emits:` entry +ends up as a separate output; they are all part of the same derivation, though. + +To compile more than one file, a more involved structure has to be used; Rust +expects the file structure to exist on-disk, and `mod` works relative to the +directory the current file is in. + +[,scheme,line-comment=;] +---- + +(define multi-file + (zdir + "lib.rs" (zfile "pub mod foo;") + "foo.rs" (zfile "pub fn hello(who: String) { println!(\"Hello, {who}\"); }"))) + +(define outputs + (call-rustc + #~,(string-append #$multi-file "/lib.rs") ; <1> + '() + crate-type: 'rlib + crate-name: "hello-lib" + edition: "2021" + emits: '(metadata: #t link: #t))) +---- +<1> Import `multi-file` (the directory) to store, append `/lib.rs` to its store + path, then use that as the file input to `rustc`. This construction is + equivalent to `"${./multi-file}/lib.rs"` in Nix. + +== Reading Cargo files +Cargo files contain a lot of automated detection of paths. For example, a +binary can live at `src/bin.rs`, or at `src/bins/foo.rs`. To handle this, Zilch +requires the Cargo file be accompanied with a VFS representing the crate. To +handle workspaces, parsing a `Cargo.toml` takes a workspace, and returns both +the crate _and_ workspace defined in the file, where applicable. This is +handled by xref:generated:zilch.lang.rust.cargo.adoc[`parse-cargo-toml`]. + +Reading a `Cargo.lock` is handled by `parse-lockfile`, which takes in the +contents of a lockfile, and parses it into a list of lockfile entries. Each +entry in the lockfile can be used in xref:generated:zilch.lang.rust.registry.adoc[`fetch-and-unpack-crate`] +to fetch the crate, though this is usually done automatically by the resolver. + +== The resolver +The rust resolver in Zilch handles selecting dependencies from a `Cargo.lock` +for one or more Cargo targets. + +To use it, first parse a lockfile and a `Cargo.toml`: + +[,scheme,line-comment=;] +---- +(define example-vfs (vfs-from-directory "example")) + +(define lockfile + (parse-lockfile + (read-file "example/Cargo.lock"))) <1> + +(define-values (crate workspace) + (parse-cargo-toml + example-vfs + (read-file "example/Cargo.toml") + #f)) +---- +<1> `read-file` is an example procedure, and isn't part of Zilch (yet). + +If desired, add more `Cargo.toml` files to prefer over any crates found in the +`Cargo.lock`: + +[,scheme,line-comment=;] +---- +(define override-vfs (vfs-from-directory "override-example")) +(define-values (override-crate workspace) + (parse-cargo-toml + override-vfs + (read-file "override-example/Cargo.toml") + #f)) +---- + +Then, call the resolver: + +[,scheme,line-comment=;] +---- +(define targets + (process-many-with-lockfile + (list + (cons crate example-vfs) + (cons override-crate override-vfs)) + lockfile)) +---- + +The result from `process-many-with-lockfile` is a list of `` +records, each representing a Cargo target in the crates that have been +processed. + +[CAUTION] +==== +By default, the resolver will set _no_ features on any crate, not even default +ones. As exception, if a binary crate is available, it _will_ have the default +feature set, which is likely to percolate down to most other crates it depends +on. There is currently no way to change this behavior easily. +==== + +When a target is to be built, it can be used with `build-package`: + +[,scheme,line-comment=;] +---- +(define (build-script-env-override-example crate-name is-dependency) + (if (and is-dependency (string=? crate-name "test")) ; <1> + (list + (cons + "PATH" + #~,(string-append + #$(cdr (assoc "out" (nixpkgs "hello"))) + "/bin"))) + #f)) +(define (compiler-env-override-example crate-name is-dependency) + (if (and (not is-dependency) (string=? crate-name "test")) ; <2> + '(("HELLO_STRING" . "Hello!")) + #f)) + +(map + (lambda (pkg) + (build-package + pkg + build-script-env-override-example + compiler-env-overridde-example)) ; <3> + targets) +---- +<1> Add GNU Hello to the PATH of any build script that depends on the crate `test` +<2> If the crate `test` is built, set `HELLO_STRING`. This can be read from + Rust using the `env!` macro. +<3> Use the overrides when building each crate's targets. + +[#limits] +=== Limitations +The `(zilch lang rust resolver)` implements most of the logic of a https://doc.rust-lang.org/cargo/reference/resolver.html#resolver-versions[v1 resolver], +which handles selecting dependencies from a set of candidates. The biggest +difference between Zilch's resolver and the canonical implemention (in Cargo) +is that the resolver in Zilch will never fall back to looking for a candidate +inside any configured Cargo registries. If a crate isn't found in the `Cargo.lock` +it is considered an error. While this makes using Zilch a tad more cumbersome, +it means that Zilch itself is entirely deterministic on the files of the crates +it processes. diff --git a/docs/modules/ROOT/pages/rust/man.adoc b/docs/modules/ROOT/pages/rust/man.adoc new file mode 100644 index 0000000..e7aeaa5 --- /dev/null +++ b/docs/modules/ROOT/pages/rust/man.adoc @@ -0,0 +1,63 @@ += zilch-cli-rust(1) +Puck Meerburg +v0.0.1 +:doctype: manpage +:manmanual: ZILCH-CLI-RUST +:mansource: ZILCH +:page-pagination: prev + +== Name + +zilch-cli-rust - builds a Rust crate using Zilch + +== Synopsis + +*zilch-cli-rust* [_OPTION_]... [_TARGET_]... + +== Description + +This command uses Zilch to build one or more Rust crate targets +entirely inside Nix, using content-addressed derivations. + +This program requires a Rust crate consisting of a `Cargo.toml` and +with a valid and up-to-date `Cargo.lock`. + +== Options + +*-h*:: +*--help*:: + Print a help message + +*-j* _COUNT_:: +*--max-jobs* _COUNT_:: + The maximum amount of builds to run. Defaults to the amount of cores. + +*-v*:: +*--verbose*:: + Increase the verbosity configured in the Nix daemon. Can be specified + multiple times. + +*-L*:: +*--print-build-logs*:: + Print derivation logs as they come in. + +*-m* _DIR_:: +*--crate-dir* _DIR_:: + The directory to use as crate. This is also the crate that will be + used to find viable targets to build. + +*-r* _DIR_:: +*--replace* _DIR_:: + Add the crate in _DIR_ to the possible dependencies, prioritizing it + over any crate in the `Cargo.lock`. Can be specified multiple times. + +*-z* _PATH_:: +*--overrides* _PATH_:: + Read build script overrides from this file. By default, a stock set + of overrides is used, aimed at some crates. If an empty string is + used for _PATH_, this default set of overrides is not used. Can be + specified multiple times. + +*--debug*:: + Crash on the first error, rather than continuing to build the next + target. diff --git a/docs/modules/ROOT/pages/rust/usage.adoc b/docs/modules/ROOT/pages/rust/usage.adoc new file mode 100644 index 0000000..be3c84c --- /dev/null +++ b/docs/modules/ROOT/pages/rust/usage.adoc @@ -0,0 +1,76 @@ += Usage +:page-pagination: next + +Using `zilch-cli-rust`, you can compile Rust code incrementally crate-by-crate: + +[,console] +---- +$ git clone https://github.com/BurntSushi/ripgrep + +$ zilch-cli-rust --crate-dir ripgrep/ <1> +… +ripgrep rg bin /nix/store/pxvqbn65lv2f4r3x1cszv5pr5l5mjwh9-rustc-bin-rg-link +grep grep lib not a binary +grep-cli grep_cli lib not a binary +globset globset lib not a binary +… + +$ zilch-cli-rust --crate-dir ripgrep/ rg <2> +… +ripgrep rg bin /nix/store/pxvqbn65lv2f4r3x1cszv5pr5l5mjwh9-rustc-bin-rg-link +---- +<1> Building all binary targets in a crate +<2> Building a specifically targeted target in a crate + +Right now, the daemon has to be able to build `x86_64-linux` derivations, and +only the `x86_64-unknown-linux-gnu` Rust target is supported. + +== Replacing dependencies +As part of Zilch, it's also possible to quickly build a crate with one of its +(transitive) dependencies replaced: + +[,console] +---- +$ git clone https://github.com/BurntSushi/bstr +$ zilch-cli-rust --crate-dir ripgrep/ --replace bstr/ rg +… +ripgrep rg bin /nix/store/6rf4r69sp8ijg2xrdglbnx8smphfg6fr-rustc-bin-rg-link +---- + +After editing a file any of the replaced dependencies, if this crate is used in +the final build, Zilch will apply early-cutoff where possible. + +[IMPORTANT] +.Limitations in dependency selection +==== +Zilch will never select dependencies that aren't found in the lockfile of the +primary crate. If there is no lockfile, or the dependencies have changed, it is +necessary to manually run `cargo generate-lockfile` to make sure all +dependencies can be found. See xref:./library.adoc#limits[The resolver's limitations] for more info. +==== + +== Build script overrides +Many Rust projects use build scripts that require external dependencies. To add +support for these, you can use a JSON file. + +This file consists of a JSON object, where the key is the name of a crate, and +the value is an object with the following keys: + +- `buildScript`: Overrides to apply when executing the build script for this + crate +- `buildScriptDependency`: Overrides to apply when executing the build script + for any crate that depends on this crate. +- `rustc`: Environment variables to add to the `rustc` invocation for this + crate. + +Each of these keys contains a map of key to a Nix expression evaluated inside +`nixpkgs`. If multiple overrides apply to a single crate, each value is +concatenated, separated with a `:`. Zilch's runner supports `_zilch_pkgconfig` +as an environment variable as special case to add both `lib/pkgconfig` and +`share/pkgconfig`, as well as the paths inside all the `propagated-build-inputs` +of each store path, making `pkg-config` work. See +https://puck.moe/git/zilch/tree/cli/overrides.json[the default overrides] for +more examples on real-life crates. + +If this mechanism is not expressive enough, it's possible to use Zilch's Rust +support directly, as a Scheme library. diff --git a/lang/rust/src/build-script.sld b/lang/rust/src/build-script.sld index afcdb00..8148466 100644 --- a/lang/rust/src/build-script.sld +++ b/lang/rust/src/build-script.sld @@ -1,3 +1,4 @@ +;; Helper procedure to call a build script with the right working directory. (define-library (zilch lang rust build-script) (import (scheme base) (scheme write) (scheme process-context) (scheme lazy) @@ -26,7 +27,9 @@ #:crate-name "runner" #:edition "2021" #:emits '(#:link #t)))) - + + ;; Call a build script with specified current working directory (as string) + ;; and environment (as alist). (define (call-runner input-script cwd env) (define output (store-path-for-ca-drv* "build.rs-run" "x86_64-linux" `(,runner-runner) diff --git a/lang/rust/src/cargo.sld b/lang/rust/src/cargo.sld index a07f840..755d7ae 100644 --- a/lang/rust/src/cargo.sld +++ b/lang/rust/src/cargo.sld @@ -1,3 +1,4 @@ +;; Procedures to parse Cargo files. (define-library (zilch lang rust cargo) (import (scheme base) (scheme write) (scheme process-context) (scheme lazy) @@ -13,21 +14,6 @@ (zilch vfs)) (export - make-cargo-target cargo-target? - cargo-target-name cargo-target-path cargo-target-test cargo-target-doctest - cargo-target-bench cargo-target-doc cargo-target-proc-macro cargo-target-harness - cargo-target-edition cargo-target-crate-type cargo-target-required-features - - make-cargo-dep-git cargo-dep-git? - cargo-dep-git-url cargo-dep-git-rev-type cargo-dep-git-rev - make-cargo-dep-path cargo-dep-path? cargo-dep-path-path - make-cargo-dep-registry cargo-dep-registry? cargo-dep-registry-name - - make-cargo-dependency cargo-dependency? - cargo-dependency-name cargo-dependency-origin cargo-dependency-version - cargo-dependency-default-features cargo-dependency-features cargo-dependency-package - cargo-dependency-optional - make-cargo-crate cargo-crate? cargo-crate-name cargo-crate-version cargo-crate-edition cargo-crate-dependencies cargo-crate-features cargo-crate-lib-target cargo-crate-targets @@ -38,6 +24,21 @@ cargo-workspace-members cargo-workspace-exclude cargo-workspace-dependencies cargo-workspace-version cargo-workspace-edition + make-cargo-dependency cargo-dependency? + cargo-dependency-name cargo-dependency-origin cargo-dependency-version + cargo-dependency-default-features cargo-dependency-features cargo-dependency-package + cargo-dependency-optional + + make-cargo-dep-git cargo-dep-git? + cargo-dep-git-url cargo-dep-git-rev-type cargo-dep-git-rev + make-cargo-dep-path cargo-dep-path? cargo-dep-path-path + make-cargo-dep-registry cargo-dep-registry? cargo-dep-registry-name + + make-cargo-target cargo-target? + cargo-target-name cargo-target-path cargo-target-test cargo-target-doctest + cargo-target-bench cargo-target-doc cargo-target-proc-macro cargo-target-harness + cargo-target-edition cargo-target-crate-type cargo-target-required-features + cfg-values parse-cargo-toml) @@ -56,7 +57,22 @@ ; (define-values (_ _ _) (process-wait pid)) parsed) - ;; dependencies here is a list of (name . version-or-#f). if #f, use any version (should be unambiguous!) + ;; A single target for a crate. + ;; + ;; Each crate consists of multiple targets; e.g. `lib`, ``bin``s, and things like test/doctest. + ;; + ;; The fields of this record closely match the Cargo.toml. For more information on these, see https://doc.rust-lang.org/cargo/reference/cargo-targets.html#configuring-a-target:[the Cargo documentation]. + ;; + ;; - `name`: The name of this target. By default, the name of the binary or + ;; of the crate, if lib. + ;; - `path`: The root source file for this target (e.g. `"src/lib.rs"`, `"src/bin/example.rs"`). + ;; Relative to the crate's root. + ;; - `crate-type`: The type of crate, as a symbol (bin, lib, rlib, etc). + ;; - `edition`: The target's edition + ;; - `required-features`: A specifying which features on the `lib` have to be set. + ;; (non-lib targets implicitly depend on the lib target) + ;; + ;; Based on the `crate-type` the other fields may be valid; these closely match the `Cargo.toml`'s definitions as well. (define-record-type (make-cargo-target name path test doctest bench doc proc-macro harness edition crate-type required-features) cargo-target? @@ -71,7 +87,7 @@ (edition cargo-target-edition) ; defaults to package's edition field (crate-type cargo-target-crate-type) ; [bin, lib, rlib, dylib, cdylib, staticlib, proc-macro] (required-features cargo-target-required-features)) ; list. has no effect on lib - + (define-record-printer ( entry out) (fprintf out "#" (cargo-target-name entry) @@ -85,7 +101,7 @@ (cargo-target-edition entry) (cargo-target-crate-type entry) (cargo-target-required-features entry))) - + ; either: ; - git + optionally tag/rev/branch (this supports looking at workspace.toml, as an exception) ; - path (relative) @@ -98,7 +114,10 @@ ; - package (used for resolving against the registry) ; - optional (only if feature is enabled!) ; or like, workspace (+ optional/features, whee) - + + ;; A Git cargo dependency. + ;; `rev-type` is either `'tag`, `'rev`, `'branch`, or `#f`, and specifies + ;; how `rev` is interpreted. (define-record-type (make-cargo-dep-git url rev-type rev) cargo-dep-git? @@ -111,7 +130,9 @@ (cargo-dep-git-url entry) (cargo-dep-git-rev-type entry) (cargo-dep-git-rev entry))) - + + ;; A path cargo dependency. + ;; The path is relative to the crate root. (define-record-type (make-cargo-dep-path path) cargo-dep-path? @@ -120,7 +141,8 @@ (define-record-printer ( entry out) (fprintf out "#" (cargo-dep-path-path entry))) - + + ;; A registry dependency. (define-record-type (make-cargo-dep-registry name) cargo-dep-registry? @@ -129,6 +151,17 @@ (fprintf out "#" (cargo-dep-registry-name entry))) + ;; A crate's dependency. + ;; + ;; - `name`: the name this dependency has in the Rust code. + ;; - `origin`: a ``, ``, or + ;; ``, specifying the source of the dependency. + ;; - `version`: The version requirements, as a string. Optional for + ;; non-registry depndencies. + ;; - `default-features`: Whether default features are requested on this dependency. + ;; - `features`: A list of features requested. + ;; - `package`: The name of the crate being depended upon. + ;; - `optional`: Whether this dependency is optional. (define-record-type (make-cargo-dependency name origin version default-features features package optional) cargo-dependency? @@ -148,7 +181,39 @@ (cargo-dependency-features entry) (cargo-dependency-package entry) (cargo-dependency-optional entry))) - + + ;; A representation of a Cargo crate. + ;; + ;; - `name`: The name of the crate + ;; - `version`: The version of the crate + ;; - `edition`: The edition of Rust this crate uses. Can be overridden by + ;; individual targets. + ;; - `workspace`: A reference to the workspace this crate is part of, or + ;; `#f` if none. + ;; - `dependencies`: A list of `` records. + ;; - `build-dependencies`: A list of `` records, used for + ;; the `build-script` only. + ;; - `features`: An alist of feature names to any features it depends on. + ;; A feature dependency is represented as a triple `((crate-name . activates-crate) . crate-feature)`. + ;; `crate-name` and `activates-crate` govern the target crate. If + ;; `crate-name` is `#f`, the feature depends on another feature in the + ;; same crate. `activates-crate` specifies whether this is a required + ;; dependency, or whether the feature should be enabled when another + ;; feature requires the crate. `crate-feature` specifies the name of the + ;; feature in the target crate, or `#f` if this feature only depends on + ;; the crate itself. As examples: + ;; + + ;; ** "dep:foo" resolves to (("foo" . #t) . #f) + ;; ** "foo/bar" resolves to (("foo" . #t) . "bar") + ;; ** "foo?/bar" resolves to (("foo" . #f) . "bar") + ;; + ;; - `lib-target`: the `` of the library of the crate, or `#f` if this crate + ;; has no library. + ;; - `build-script`: the `` of the build script, if any is present. + ;; - `links`: the name of a native library being linked to, or `#f` if none. + ;; - `check-cfg-lint`: A list of `--check-cfg` flags to be set on compiling + ;; this crate. These values are sourced from `lints.rust.unexpected_cfgs` + ;; in the `Cargo.toml`. (define-record-type (make-cargo-crate name version edition workspace dependencies build-dependencies features lib-target build-script targets links check-cfg-lint) cargo-crate? @@ -175,6 +240,15 @@ (cargo-crate-lib-target entry) (cargo-crate-targets entry))) + ;; A representation of a Cargo.toml's workspace. + ;; + ;; - `members`: A list of members of this workspace. + ;; - `exclude`: A list of globs that should be excluded, even when mentioned + ;; in `members`. + ;; - `dependencies`: A list of ````s that any crate part + ;; of this workspace can reference. + ;; - `edition`, `version`, `check-cfg-lint`: Default values for the + ;; equivalent fields in this workspace's ````s. (define-record-type (make-cargo-workspace members exclude dependencies edition version check-cfg-lint) cargo-workspace? @@ -190,7 +264,7 @@ (cargo-workspace-members entry) (cargo-workspace-exclude entry) (cargo-workspace-dependencies entry))) - + (foreign-declare "#include \"cfgfetch_source.h\"") (define cfgfetch-bin (cdar @@ -210,6 +284,8 @@ (let* ((line (cfg-parse line))) (read-cfg-value port (cons (cdr line) rest))))) + ;; The list of config values that `rustc` sets by default, processed by xref:zilch.lang.rust.cfg.adoc#cfg-parse[`cfg-parse`]. + ;; These are used to parse `Cargo.toml` conditional dependencies. (define cfg-values (let ((vals (cdar (store-path-for-ca-drv* "cfg-values" "x86_64-linux" #~(#$cfgfetch-bin) #~(("rustc" . ,(string-append #$rustc "/bin/rustc"))) '("out"))))) (call-with-port (store-path-open vals) (lambda (p) (read-cfg-value p '()))))) @@ -224,7 +300,7 @@ (define (and-cdr-default val default) (if val (cdr val) default)) - + (define (find-dependency-in-list l dep-name) (cond ((null? l) #f) @@ -238,12 +314,12 @@ (define pkg-features (and-cdr-default (assoc "features" object-internals) '())) (define package (or (and-cdr (assoc "package" object-internals)) name)) (define optional (and-cdr (assoc "optional" object-internals))) - + (define git-url (and-cdr (assoc "git" object-internals))) (define git-tag (and-cdr (assoc "tag" object-internals))) (define git-rev (and-cdr (assoc "rev" object-internals))) (define git-branch (and-cdr (assoc "branch" object-internals))) - + (define registry-name (and-cdr (assoc "registry" object-internals))) (define path (and-cdr (assoc "path" object-internals))) @@ -310,7 +386,7 @@ (define required-features (or (and-cdr (assoc "required-features" object-internals)) '())) (make-cargo-target name path test doctest bench doc proc-macro harness edition crate-type required-features)) - + ; A feature is two parts: ((crate-name . activates-crate) package-feature) ; "dep:foo" resolves to (("foo" . #t) . #f) ; "foo/bar" resolves to (("foo" . #t) . "bar") @@ -343,7 +419,7 @@ (unless vfs (error "no vfs")) (define package (vector->list (cdr (assoc "package" internals)))) (define package-name (cdr (assoc "name" package))) - (define package-version (cdr (assoc "version" package))) + (define package-version (or (and-cdr (assoc "version" package)) "0.0.0")) (define package-links (and-cdr (assoc "links" package))) (define package-edition (or (and-cdr (assoc "edition" package)) "2015")) @@ -363,7 +439,7 @@ (define lib-target #f) (when (or (assoc "lib" internals) (vfs-file-ref vfs "src" "lib.rs")) (set! lib-target (cargo-target-from-toml vfs (or (and-cdr (assoc "lib" internals)) #()) package-name 'lib package-edition))) - + (define other-targets '()) (cond ((assoc "bin" internals) @@ -445,6 +521,14 @@ (set-cargo-workspace-dependencies! workspace-record dependencies) workspace-record) + ;; Parse the contents of a `Cargo.toml`. Returns two values: A `` + ;; representing the crate (or `#f` if this is purely a workspace), and a + ;; `` the `Cargo.toml` defined (or the workspace passed in, + ;; if any). + ;; + ;; - `vfs`: The VFS to use to automatically detect `bin` and `lib` targets. + ;; - `cargo-file`: A string representing the contents of the `Cargo.toml` to use. + ;; - `workspace`: A workspace that this rcate is part of, or `#f` if none. (define (parse-cargo-toml vfs cargo-file workspace) (define internals (vector->list (parse-toml cargo-file))) (define crate #f) diff --git a/lang/rust/src/cfg.sld b/lang/rust/src/cfg.sld index f11e195..22d0f13 100644 --- a/lang/rust/src/cfg.sld +++ b/lang/rust/src/cfg.sld @@ -1,3 +1,5 @@ +;; Procedures to parse `cfg` attributes, as well as match them against +;; conditionals found in crate definitions. (define-library (zilch lang rust cfg) (import (scheme base) @@ -35,6 +37,12 @@ (let ((end (or (string-skip strval is-ident-rest (+ index 1)) (string-length strval)))) (tokenize-cfg strval end (cons (cons 'ident (string-copy strval index end)) tail))) (error "Unexpected character in cfg() string" strval)))))) + + ;; Parses a configuration string or expression. + ;; + ;; - `key="value"` is represented as `('value . (key . value))`, where value can be `#f`. + ;; - `all` and `any` are represented as `('all/'any . items)`, where `items` is a list of sub-expressions. + ;; - `not(...)` is represented as `('not . value)`, where `value` is a sub-expression. (define (cfg-parse str) (define tokens (tokenize-cfg str 0 '())) (define (expect token) @@ -100,6 +108,8 @@ (cons 'value (cons left right)))))) (parse-expr)) + ;; Checks whether the parsed expression `expr` matches against the list of + ;; config value pairs in `cfgs`. (define (cfg-matches expr cfgs) (define (parse-any tail) (cond diff --git a/lang/rust/src/registry.sld b/lang/rust/src/registry.sld index 6bdde12..b8cdd1a 100644 --- a/lang/rust/src/registry.sld +++ b/lang/rust/src/registry.sld @@ -1,3 +1,4 @@ +;; Procedures to parse lockfiles, and fetch crates from lockfile entries. (define-library (zilch lang rust registry) (import (scheme base) (scheme write) (scheme process-context) (scheme lazy) @@ -11,8 +12,10 @@ (export parse-lockfile fetch-and-unpack-crate - - lockfile-entry? lockfile-entry-name lockfile-entry-version lockfile-entry-source lockfile-entry-checksum lockfile-entry-dependencies) + + lockfile-entry? lockfile-entry-name + lockfile-entry-version lockfile-entry-source + lockfile-entry-checksum lockfile-entry-dependencies) (begin (define yj-path (foreign-value "YJ_PATH" nonnull-c-string)) @@ -26,9 +29,19 @@ (close-input-port read-port) ; (define-values (_ _ _) (process-wait pid)) parsed) - - ;; TODO(puck): source here should probably be a record? - ;; dependencies here is a list of (name . version-or-#f). if #f, use any version (should be unambiguous!) + + + ;; The contents of a single lockfile entry. + ;; + ;; - `name`: The name of the crate + ;; - `version`: The version of the crate. + ;; - `source`: The source of the crate, as raw URL from the `Cargo.lock`. + ;; - `checksum`: A bytevector containing the sha256 of the `.crate` file, if + ;; one is available. + ;; - `dependencies`: A list of dependencies for this crate. Each dependency + ;; is a pair `(crate-name . crate-version)`, where `crate-version` is only + ;; set if there is more than one version of the depended crate in the + ;; lockfile. (define-record-type (make-lockfile-entry name version source checksum dependencies) lockfile-entry? @@ -37,7 +50,7 @@ (source lockfile-entry-source) (checksum lockfile-entry-checksum) (dependencies lockfile-entry-dependencies)) - + (define-record-printer ( entry out) (fprintf out "#" (lockfile-entry-name entry) @@ -152,6 +165,8 @@ (define git-dir (or (get-environment-variable "XDG_CACHE_HOME") (string-append (get-environment-variable "HOME") "/.cache/zilch/git"))) + ;; Fetch a crate from the internet. Supports the `crates.io` registry source, + ;; as well as `git` dependencies. Returns a store path. (define (fetch-and-unpack-crate lockfile-entry) (define src (lockfile-entry-source lockfile-entry)) (cond @@ -162,6 +177,7 @@ ((equal? src "registry+https://github.com/rust-lang/crates.io-index") (fetch-from-registry lockfile-entry)) (else (error "unknown source " lockfile-entry)))) + ;; Parse a `Cargo.lock`, returning a list of ````s. (define (parse-lockfile file-contents) (define inputs (vector->list (parse-toml file-contents))) (define lockfile-version (assoc "version" inputs)) diff --git a/lang/rust/src/resolver.sld b/lang/rust/src/resolver.sld index f28b8d5..f2179be 100644 --- a/lang/rust/src/resolver.sld +++ b/lang/rust/src/resolver.sld @@ -1,3 +1,5 @@ +;; Implements the resolver, which takes in a lockfile and other crates, and +;; allows resolving the dependencies for said crates. (define-library (zilch lang rust resolver) (import (scheme base) (scheme write) (scheme process-context) (scheme lazy) @@ -13,16 +15,17 @@ (export make-resolver resolver? resolver-locked-dependencies resolver-selected-dependencies - make-resolved-package-build-data resolved-package-build-data? - resolved-package-build-data-dep-info resolved-package-build-data-metadata resolved-package-build-data-rlib - resolved-package-build-data-transitive-dependencies make-resolved-package resolved-package? resolved-package-name resolved-package-version resolved-package-fs resolved-package-cargo-target resolved-package-enabled-features resolved-package-pending-features resolved-package-dependencies resolved-package-crate resolved-package-build-data - + + make-resolved-package-build-data resolved-package-build-data? + resolved-package-build-data-dep-info resolved-package-build-data-metadata resolved-package-build-data-rlib + resolved-package-build-data-transitive-dependencies resolved-package-build-data-build-script-out + resolver-download resolver-resolve-nonoptional resolver-resolve-resolved-package @@ -49,28 +52,58 @@ #:crate-name "false" #:edition "2021" #:emits '(#:link #t))))) - - ; Used to select a set of crates plus their versions. + + ;; A helper record used to select a set of crates plus their versions. + ;; + ;; - `locked-dependencies`: An SRFI 146 mapping of package names to a mapping + ;; of version to a pair of `( . unpack-promise)`, where + ;; `unpack-promise` is a promise that resolves to a store path containing + ;; the crate. + ;; - `selected-dependencies`: An SRFI 146 mapping of package names to a list + ;; of `(version . )` pairs, representing the versions of + ;; this crate that have been activated. (define-record-type - ; locked-dependencies is a mapping of package-name to a mapping of version to (version . (lockfile-entry . unpack-promise)) - ; selected-dependencies is a mapping of package-name to a list of (version . resolved-package)(?) - ; pending-features is a mapping of (package-name . version) to a list of features (make-resolver locked-dependencies selected-dependencies) resolver? (locked-dependencies resolver-locked-dependencies set-resolver-locked-dependencies!) (selected-dependencies resolver-selected-dependencies set-resolver-selected-dependencies!)) - + + ;; A list of store paths for a `'`''s build output build output. + ;; + ;; - `dep-info`: Equivalent to the rustc `dep-info` emit. + ;; - `metadata`/`rlib`: A pair `(canonical-name . store-path)` for the + ;; `metadata` and `obj` emits respectively. + ;; - `transitive-dependencies`: A list of `` for all + ;; transitive dependencies of this resolved package. + ;; - `build-script-out`: The store path output from the build script. + ;; The format for this output is defined by the `buildscript-runner`. (define-record-type - (make-resolved-package-build-data dep-info metadata rlib transitive-dependencies build-script-metadata bin-flags build-script-out) + (make-resolved-package-build-data dep-info metadata rlib transitive-dependencies build-script-out) resolved-package-build-data? (dep-info resolved-package-build-data-dep-info set-resolved-package-build-data-dep-info!) (metadata resolved-package-build-data-metadata set-resolved-package-build-data-metadata!) (rlib resolved-package-build-data-rlib set-resolved-package-build-data-rlib!) (transitive-dependencies resolved-package-build-data-transitive-dependencies set-resolved-package-build-data-transitive-dependencies!) - (build-script-metadata resolved-package-build-data-build-script-metadata) - (bin-flags resolved-package-build-data-bin-flags) (build-script-out resolved-package-build-data-build-script-out)) - + + ;; A crate target that has been activated. + ;; + ;; - `name`, `version`: Equivalent to the name and version of the crate this + ;; package is a part of. + ;; - `cargo-target`: The `` this `` is part of. + ;; - `target-dependencies': A list of `` records for this + ;; package. + ;; - `crate`: The `` this `` is part of. + ;; - `enabled-features`: A list of feature names that have been activated on + ;; this package. + ;; - `pending-features`: an SRFI 146 mapping of dependency names to feature + ;; names that are waiting for said optional dependency to be activated. + ;; - `dependencies`: An SRFI 146 mapping of dependency name to `` + ;; record for said dependency. + ;; - `build-data`: ``, set once `build-package` + ;; is called on it. + ;; - `build-script`: The `` for the build script for this + ;; package, if a build script target exists. (define-record-type (make-resolved-package name version fs cargo-target target-dependencies crate enabled-features pending-features dependencies build-data build-script) resolved-package? @@ -85,8 +118,9 @@ (dependencies resolved-package-dependencies set-resolved-package-dependencies!) (build-data resolved-package-build-data set-resolved-package-build-data!) (build-script resolved-package-build-script set-resolved-package-build-script!)) - - ;; Download and activate a dependency from the registry. + + ;; Download and activate a dependency from the registry, in the context of `resolver`. + ;; `name` and `version` are used to find the necessary dependency from the `resolver-locked-dependencies`. (define (resolver-download resolver name version) (unless version (error "Resolver wanted non-versioned download" name)) @@ -99,7 +133,7 @@ (define version (parse-version (cargo-crate-version parsed-cargo))) (unless (cargo-crate-lib-target parsed-cargo) (error "Crate does not have valid [lib] target" (list name))) - + (define build-script #f) (when (cargo-crate-build-script parsed-cargo) (set! build-script (make-resolved-package (string-append name "_build") version vfs (cargo-crate-build-script parsed-cargo) (cargo-crate-build-dependencies parsed-cargo) parsed-cargo '() (mapping (make-default-comparator)) (mapping (make-default-comparator)) #f #f)) @@ -113,16 +147,17 @@ (resolver-resolve-nonoptional resolver pkg) pkg) - ;; Preemptively resolve and activate all dependencies not marked optional. + ;; Resolve and activate all dependencies not marked optional in `` `pkg`. (define (resolver-resolve-nonoptional resolver pkg) (for-each (lambda (dep) (unless (cargo-dependency-optional dep) (resolver-resolve-resolved-package resolver pkg (cargo-dependency-name dep) #t))) (resolved-package-target-dependencies pkg))) - - ;; Resolve a name of a dependency of a , activating it if `activate` is #t. + + ;; Resolve a dependency of `` `pkg` by name, activating it if requested. + ;; Returns the resolved dependency. (define (resolver-resolve-resolved-package resolver pkg name activate) (define resolved-dep (mapping-ref/default (resolved-package-dependencies pkg) name #f)) (define cargo-dep @@ -141,9 +176,9 @@ (resolver-activate-features resolver resolved-dep pending-features)))) resolved-dep) - - ;; Activate a series of features on an existing . This will resolve and activate optional dependencies - ;; where needed. + + ;; Activate a list of features on `` `resolved`, + ;; resolving and activating optional dependencies where needed. (define (resolver-activate-features resolver resolved to-activate) (for-each (lambda (feature) @@ -188,8 +223,11 @@ '()))))))) (cdr (or (assoc feature (cargo-crate-features (resolved-package-crate resolved))) (cons '() '())))))) to-activate)) - - ;; Register a non-registry crate+vfs with the resolver. + + ;; Register a non-registry `` and corresponding vfs with the + ;; resolver. If `delayed` is `#t`, does not resolve any non-optional + ;; dependency. This is used for loading multiple crates into the resolver in + ;; arbitrary order. (define (resolver-register resolver vfs crate delayed) (define target (cargo-crate-lib-target crate)) (cond @@ -201,6 +239,7 @@ (map (lambda (target) (resolver-register-target resolver vfs crate target #f delayed)) (cargo-crate-targets crate))))) ;; Register a non-registry crate+vfs with the resolver. + ;; If `delayed`, do not resolve the crate's (non-optional) dependencies yet. (define (resolver-register-target resolver vfs crate target extra-dependencies delayed) (define build-script #f) (unless extra-dependencies @@ -216,8 +255,8 @@ (unless delayed (resolver-resolve-nonoptional resolver pkg)) pkg) - - ;; Resolves a , returning the . + + ;; Resolves a `` through the `resolver`, returning the ``. (define (resolver-resolve resolver dep) (define package-name (cargo-dependency-package dep)) (define requirements (apply append (map parse-version-requirement (if (cargo-dependency-version dep) (string-split (cargo-dependency-version dep) "," 'strict-infix) '())))) @@ -243,7 +282,8 @@ (when (cargo-dependency-features dep) (resolver-activate-features resolver resolved (cargo-dependency-features dep))) resolved))) - + + ;; Print the information of a `` to `current-output-port`. (define (resolver-print-pkg resolver pkg) (printf " - version: ~S\n" (resolved-package-version pkg)) (printf " features: ~S\n" (resolved-package-enabled-features pkg)) @@ -257,6 +297,7 @@ (printf "\n"))) (resolved-package-target-dependencies pkg))) + ;; Print the information of this resolver to `current-output-port`. (define (resolver-print resolver) (mapping-for-each (lambda (k v) @@ -272,6 +313,14 @@ v)) (resolver-selected-dependencies resolver))) + ;; Resolve a single crate, using a `` internally. + ;; + ;; - `vfs`: The VFS of the crate. + ;; - `cargo-file`: The `` to process. + ;; - `parsed-lockfile`: A list of ````s necessary to resolve the crate. + ;; - `activated-features`: A list of features to activate on the targets in `` + ;; + ;; Returns a list of ````s for the targets in `cargo-file`. (define (process-cargo-with-lockfile vfs cargo-file parsed-lockfile activated-features) (define locked-dependencies (mapping (make-default-comparator))) (for-each @@ -293,6 +342,8 @@ pkgs) pkgs) + ;; Resolve a list of `( . vfs)` pairs plus a list of ````, + ;; returning a list of `` for each target in the crates. (define (process-many-with-lockfile vfs-cargo-map parsed-lockfile) (define locked-dependencies (mapping (make-default-comparator))) (for-each @@ -361,6 +412,14 @@ (reverse envs)) (mapping-map->list (lambda (k v) (cons k v)) final-env)) + ;; Builds a ``, using specified build script and compiler overrides. + ;; + ;; - `build-script-env-overrides`: A procedure that takes two arguments `(proc crate-name is-dependency)`, + ;; and returns `#f`, or a list of pairs to add to the environment of the build script, if any is executed. + ;; - `compiler-env-overrides`: A procedure that takes two arguments `(proc crate-name is-dependency)`, and + ;; returns `#f` or a list of pairs to add to the environment of `rustc` when compiling said crate. + ;; + ;; Returns the `rlib` (or linked output, for binaries) of this ``. (define (build-package resolved build-script-env-overrides compiler-env-overrides) ; Info we need to collect: ; - enabled features @@ -377,11 +436,9 @@ ; These should probably be translated into distinct targets? (when (list? crate-type) (set! crate-type 'rlib)) - (define buildscript-metadata '()) (define buildscript-out #f) (define crate-links '()) (define dependency-metadata '()) - (define bin-flags '()) (define params `(#:crate-type ,crate-type #:remap-path-prefix (,crate-root . ,(string-append crate-name "-" (version-str (resolved-package-version resolved)))) @@ -524,7 +581,7 @@ (set! params `(#:externs (,name . ,(cdr (resolved-package-build-data-rlib data))) . ,params))) (resolved-package-dependencies resolved)) - + (define transitive-dependencies-meta (zdir (map (lambda (dep) (define data (resolved-package-build-data dep)) @@ -557,10 +614,8 @@ (for-each (lambda (check) (set! params `(check-cfg: ,check . ,params))) (cargo-crate-check-cfg-lint (resolved-package-crate resolved))) (define inherited-build-script-out '()) - (define transitive-bin-flags '()) (for-each (lambda (dep) - (set! transitive-bin-flags (append (resolved-package-build-data-bin-flags (resolved-package-build-data dep)) transitive-bin-flags)) (when (resolved-package-build-data-build-script-out (resolved-package-build-data dep)) (set! inherited-build-script-out (cons (resolved-package-build-data-build-script-out (resolved-package-build-data dep)) inherited-build-script-out)))) transitive-dependencies) @@ -570,33 +625,31 @@ (define path #~,(string-append #$(vfs-to-store (resolved-package-fs resolved)) "/" (cargo-target-path (resolved-package-cargo-target resolved)))) (define dep-info (cdar (apply call-rustc `(,path ,rustc-env search-path: ("dependency" . ,transitive-dependencies-meta) emits: (dep-info: #t) . ,params-meta)))) - + (define rlib-name (string-append "lib" crate-name "-v" crate-version ".rlib")) (define rmeta-name (string-append "lib" crate-name "-v" crate-version ".rmeta")) - + (when (eq? crate-type 'proc-macro) (set! rlib-name (string-append "lib" crate-name "-v" crate-version ".so"))) (when (eq? crate-type 'bin) (set! rlib-name crate-name)) - ; (when (or (eq? crate-type 'proc-macro) (eq? crate-type 'bin)) - ; (set! params (append transitive-bin-flags params))) (when (eq? crate-type 'bin) (set! params `(#:codegen-flags ("rpath" . "no") . ,params))) - + ; Rust is nitpicky about the filenames, fix them with copious symlinking. - (define rlib-file (cdar (apply call-rustc `(,path (("_zilch_inherit" . ,#~,(string-join #$inherited-build-script-out " ")) . ,rustc-env) search-path: ("dependency" . ,transitive-dependencies-rlib) emits: (link: #t) ,@bin-flags . ,params)))) + (define rlib-file (cdar (apply call-rustc `(,path (("_zilch_inherit" . ,#~,(string-join #$inherited-build-script-out " ")) . ,rustc-env) search-path: ("dependency" . ,transitive-dependencies-rlib) emits: (link: #t) . ,params)))) (define rlib (cons rlib-name #~,(string-append #$(zdir rlib-name (zsymlink rlib-file)) "/" rlib-name))) (store-path-materialize rlib-file) - + (define metadata #f) (define metadata-file #f) (unless (eq? crate-type 'proc-macro) (set! metadata-file (cdar (apply call-rustc `(,path ,rustc-env search-path: ("dependency" . ,transitive-dependencies-meta) emits: (metadata: #t) . ,params-meta)))) (set! metadata (cons rmeta-name #~,(string-append #$(zdir rmeta-name (zsymlink metadata-file)) "/" rmeta-name))) (store-path-materialize metadata-file)) - - (set-resolved-package-build-data! resolved (make-resolved-package-build-data dep-info metadata rlib transitive-dependencies buildscript-metadata bin-flags buildscript-out)) + + (set-resolved-package-build-data! resolved (make-resolved-package-build-data dep-info metadata rlib transitive-dependencies buildscript-out)) rlib-file) (define (matches-requirements ver req) diff --git a/lang/rust/src/rust.sld b/lang/rust/src/rust.sld index e775f23..60dc657 100644 --- a/lang/rust/src/rust.sld +++ b/lang/rust/src/rust.sld @@ -1,3 +1,4 @@ +;; Procedures to build code with `rustc`. (define-library (zilch lang rust) (import (scheme base) (scheme write) (scheme process-context) (scheme lazy) @@ -6,10 +7,12 @@ json (chicken foreign) (chicken format) (srfi 4)) - + (export rustc call-rustc) (begin + ;; Helper containing a store path for the `rustc` used by the rest of the + ;; Zilch Rust support. (define rustc (cdr (assoc "out" (nixpkgs "rustc")))) (define gcc (delay (cdr (assoc "out" (nixpkgs "gcc"))))) (define linker (delay #~,(string-append #$(force gcc) "/bin/cc"))) @@ -59,7 +62,7 @@ (check-one (rustc-emits-dep-info emits) "dep-info") (check-one (rustc-emits-mir emits) "mir") (values tail types)) - + (define (parse-rustc-emits out items) (if (eq? items '()) out @@ -73,7 +76,7 @@ ((#:dep-info) (set-rustc-emits-dep-info! out (cadr items)) (parse-rustc-emits out (cddr items))) ((#:mir) (set-rustc-emits-mir! out (cadr items)) (parse-rustc-emits out (cddr items))) (else (error "unknown rustc emits param" items))))) - + (define (parse-rustc-params out items) (if (eq? items '()) out @@ -91,11 +94,35 @@ ((#:remap-path-prefix) (set-rustc-params-remap-path-prefix! out (cons (cadr items) (rustc-params-remap-path-prefix out))) (parse-rustc-params out (cddr items))) ((#:cap-lints) (set-rustc-params-cap-lints! out (cadr items)) (parse-rustc-params out (cddr items))) (else (error "unknown rustc param" (car items)))))) - + (foreign-declare "#include \"rustc_wrap_source.h\"") (define rustc_wrap-bin #f) + ;; Call rustc with a series of known parameters. + ;; `input` represents the input file to call. `env` is an alist of environment + ;; variables. The rest of the parameters are passed as two arguments, `foo: bar`, + ;; similar to SRFI 89. + ;; + ;; - `crate-type: 'bin/'lib/etc`: Required. The crate type. + ;; - `crate-name: "foo"`: Required. The crate name. + ;; - `edition: "2021"/etc`: Required. The edition of Rust to use. + ;; - `cfg: "val"`: equivalent to `--cfg val` + ;; - `check-cfg: "val"`: equivalent to `--check-cfg val` + ;; - `search-path: foo`: Adds `foo` to the library search path. If `foo` is a + ;; pair, the `car` is the kind (`"dependency"`, `"crate"`, `"all"`, etc), + ;; and the `cdr` is the value. + ;; - `link: foo`: Links the generated output with `foo`. If `foo` is a pair, + ;; the `car` represents the kind and the `cdr` is the value. + ;; - `emits: (..emits data..)`: Set the outputs that `rustc` will generate. + ;; Emits data is a list structured like this one. For example, `(asm: #t llvm-bc: "/foo")` + ;; will output assembly to a derivation output called "asm", while + ;; outputting LLVM bytecode to the path "/foo" in the derivation. + ;; - `codegen-flags: ("foo" . "bar")`: Append `-C foo=bar` to the rustc + ;; command line. the `cdr` of this pair may be a zexp. + ;; - `remap-path-prefix: ("foo" . "bar")`: Remaps the path `foo` to the path + ;; `bar` for all error traces. Both values can be a zexp. + ;; - `cap-lints: "warn"`: Set the most restrictive lint level. (define (call-rustc input env . params) (call-rustc-internal input env (parse-rustc-params (make-rustc-params '() '() '() '() #f #f #f #f '() '() '() #f) params)))