From 93a1ebba008ee9ba3e42e6723c2e34882695abc9 Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Thu, 3 Jul 2025 18:57:48 +0000 Subject: [PATCH] (zilch lang ninja): document Change-Id: I6a6a6964dcc713504ec57f40981a601696a573af --- cli/default.nix | 1 + cli/zilch-ninja.scm | 30 +----- docs/manpages.nix | 1 + docs/modules/ROOT/nav.adoc | 3 + docs/modules/ROOT/pages/ninja/man.adoc | 63 +++++++++++ docs/modules/ROOT/pages/ninja/usage.adoc | 131 +++++++++++++++++++++++ lang/ninja/src/build.sld | 32 +++++- lang/ninja/src/config.sld | 35 +++++- lang/ninja/src/depfile.sld | 4 + lang/ninja/src/ninja.sld | 41 +++++-- lang/ninja/src/nixpkgs.sld | 31 +++++- 11 files changed, 329 insertions(+), 43 deletions(-) create mode 100644 docs/modules/ROOT/pages/ninja/man.adoc create mode 100644 docs/modules/ROOT/pages/ninja/usage.adoc diff --git a/cli/default.nix b/cli/default.nix index 5e8ee65..c6c6eb8 100644 --- a/cli/default.nix +++ b/cli/default.nix @@ -17,5 +17,6 @@ in eggDerivation { (cat ${./overrides.json}; printf '\0') | ${xxd}/bin/xxd -i -n stock_overrides > stock_overrides.h (cat ${manPages.rust.txt}; printf '\0') | ${xxd}/bin/xxd -i -n man_rust > man_rust.h (cat ${manPages.go.txt}; printf '\0') | ${xxd}/bin/xxd -i -n man_go > man_go.h + (cat ${manPages.ninja.txt}; printf '\0') | ${xxd}/bin/xxd -i -n man_ninja > man_ninja.h ''; } diff --git a/cli/zilch-ninja.scm b/cli/zilch-ninja.scm index 238d201..55bef44 100644 --- a/cli/zilch-ninja.scm +++ b/cli/zilch-ninja.scm @@ -5,35 +5,13 @@ (foreign-lambda* int () "cpu_set_t set; sched_getaffinity(0, sizeof(set), &set); C_return(CPU_COUNT(&set));")) +(foreign-declare "#include \"man_ninja.h\"") +(define man-page (foreign-value "man_ninja" nonnull-c-string)) + (define (print-help msg) (when msg (write-string (string-append msg "\n\n") (current-error-port))) - (write-string "Usage: zilch-cli-ninja [OPTION] SUBCOMMAND ... -Processes a Ninja build based on the configuration in the passed-in -config file (or zilch.scm, if not set), and reproducibly builds either -to final build, or to specific Ninja targets. - -Supported commands: - build [TARGET] Build the full drv, or specific targets from - its Ninja file. - source [DIR] Extract the source of the derivation to DIR - (or src, if unspecified), ready for Zilch. - diff Print the difference between the original and - modified sources as found in either `src' or - the directory selected by --source. - -Arguments: - -h, --help Print this help message. - -f, --config-file PATH Path to the Zilch config file. - -s, --source DIR Override the input source for builds. Doesn't - reconfigure the build, so changes to the - build system will not apply. - -j, --max-jobs COUNT The maximum amount of builds to run. Defaults - to the amount of cores. - -v, --verbose Increase the verbosity configured in the Nix - daemon. - -L, --print-build-logs Print derivation logs as they come in. -" (current-error-port)) + (write-string man-page (current-error-port)) (exit (or (not msg) 1))) (define-values (options args) diff --git a/docs/manpages.nix b/docs/manpages.nix index b1967ed..17a7f20 100644 --- a/docs/manpages.nix +++ b/docs/manpages.nix @@ -10,4 +10,5 @@ in { rust = generateBoth "zilch-cli-rust.1" ./modules/ROOT/pages/rust/man.adoc; go = generateBoth "zilch-cli-go.1" ./modules/ROOT/pages/go/man.adoc; + ninja = generateBoth "zilch-cli-ninja.1" ./modules/ROOT/pages/ninja/man.adoc; } diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 1e4d158..c94182a 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -14,6 +14,9 @@ *** xref:rust/usage.adoc[] *** xref:rust/library.adoc[] *** xref:rust/man.adoc[Man page] +** Ninja +*** xref:ninja/usage.adoc[] +*** xref:ninja/man.adoc[Man page] * Code reference include::generated:partial$nav.adoc[] diff --git a/docs/modules/ROOT/pages/ninja/man.adoc b/docs/modules/ROOT/pages/ninja/man.adoc new file mode 100644 index 0000000..d2da5da --- /dev/null +++ b/docs/modules/ROOT/pages/ninja/man.adoc @@ -0,0 +1,63 @@ += zilch-cli-ninja(1) +Puck Meerburg +v0.0.1 +:doctype: manpage +:manmanual: ZILCH-CLI-NINJA +:mansource: ZILCH +:page-pagination: prev + +== Name + +zilch-cli-ninja - builds a Nixpkgs package using Ninja, using Zilch + +== Synopsis + +*zilch-cli-ninja* [_OPTION_]... _COMMAND_ [_ARG_]... + +== Description + +This command uses Zilch to build a Ninja-based project entirely within +Nix, using content-addressed derivations. + +This program requires the derivation to build has a configuration file +written. See the documentation at https://puck.moe/zilch/docs/. + +== Subcommands +*build* [_TARGET_]...:: + Build the derivation described by the configuration file, or the + target nammes from its Ninja file, if any are passed as arguments. + +*source* [_DIR_]:: + Write the source of the derivation to _DIR_, or to `src` if no + directory is provided. + +*diff*:: + Compare the 'canonical' source of the derivation to that in the + source directory (`src`, or the path selected by `--source`.) + +== Options + +*-h*:: +*--help*:: + Print a help message. + +*-s* _DIR_:: +*--source* _DIR_:: + Override the input source for builds. Doesn't reconfigure the build, + so changes to the build system do not apply. + +*-f* _PATH_:: +*--config-file* _PATH_:: + The location of the configuration file. Defaults to `zilch.scm`. + +*-j* _COUNT_:: +*--max-jobs* _COUNT_:: + The maximum amount of builds to run. Defaults to the amount of cores. + +*-v*:: +*--verbose*:: + Increase the verbosity configured in the Nix daemon. + +*-L*:: +*--print-build-logs*:: + Print derivation logs as they come in. diff --git a/docs/modules/ROOT/pages/ninja/usage.adoc b/docs/modules/ROOT/pages/ninja/usage.adoc new file mode 100644 index 0000000..6def8ab --- /dev/null +++ b/docs/modules/ROOT/pages/ninja/usage.adoc @@ -0,0 +1,131 @@ += Usage +:page-pagination: next + +`zilch-cli-ninja` allows you to use Zilch to compile a Ninja-based nixpkgs +derivation reproducibly, and to quickly modify it and rebuild it reproducibly +as well. To use the tool, you first have to write a configuration file. This +document will use this example, for `gtest`: + +.`gtest.scm` +[,scheme] +---- +(environment: "pkgs.gtest" + depfile-path: "gtest-deps.scm") +---- + +This can then be used with the various subcommands of `zilch-cli-ninja`: + +== Building +To build a derivation with `zilch-cli-ninja`, use the `build` subcommand. +[,console] +---- +$ zilch-cli-ninja -f gtest.scm build <1> +… +dev -> # +out -> # + +$ zilch-cli-ninja -f gtest.scm build googlemock/CMakeFiles/gmock_main.dir/src/gmock_main.cc.o <2> +… +googlemock/CMakeFiles/gmock_main.dir/src/gmock_main.cc.o -> "/nix/store/c96j8sqk08xj15pififz5myjzskg0cqw-Building-CXX-object-googlemock-CMakeFiles-gmock_main.dir-src-gmock_main.cc.o" +---- +<1> Build the full derivation +<2> Build a specific output of the Ninja file + +== Source editing +Zilch makes it very easy to edit the source of any derivation built in this +manner. With one command, you can export the source code used for building. +This source can then be used to rebuild parts of the derivation that have changed. + +[,console] +---- +$ zilch-cli-ninja -f gtest.scm source gtest-src <1> +… + +$ zilch-cli-ninja -f gtest.scm -s gtest-src diff <2> +… + +$ zilch-cli-ninja -f gtest.scm -s gtest-src build <3> +… +dev -> # +out -> # +---- + +WARNING: Zilch will not regenerate the build files when editing source in this +manner, for performance and logistical reasons. To change the build files, +override the `src` attribute of the environment's derivation. + +== Limitations +To make the Nixpkgs integration work, the Zilch code breaks up the derivation's +phases into two steps: unpack + patch + configure, and check + install. This +can cause issues with derivations that expect to share variables between these +phases. If `meson` is present, it is assumed to be used, otherwise it is +assumed the derivation is `cmake`-based. Other build systems are not yet +supported. + +The final store path of the derivation is not known at configuration time, +unlike normal derivations. Meson and CMake are configured to install into a +prefix, and are then rewritten (using `sed`) to the final output store path. +This may cause issues with programs that e.g. compress a file containing these +paths during their build.footnote:[The only one to have known issues is Node.] + +== Patches +In some cases, the code may not handle running in Zilch's environment. In this +case, it's possible to add extra code to run before running the command of a +target. + +These patches are defined as part of the configuration file, and are arbitrary +Scheme code: + +[,scheme] +---- +(environment: "import ./sample.drv" + patch: + (lambda (edge) + (if (string=? (build-edge-rule edge) "foo") + "rm src/foo; echo 'replacement file' > src/foo" + #f))) +---- + +Patches are arbitrary shell scripts, run with GNU Coreutils in `$PATH`. The +environment variable `$_ZILCH_ROOT` points to the root of the source + build +file tree, and can be used to copy non-symlinked files into the working +directory. + +== Automatic file elision +To implement incremental builds efficiently, Zilch automatically slims down the +part of the source directory that is accessible to each target. If a filename +ends in `.cc` or `.c` and is directly depended upon as in input file to a +target, it will be elided, and only the targets that depend on it directly will +be able to read its contents. + +The `build/meson-private` directory is fully elided, except for derivations +that directly depend upon it. This is to limit the amount of impurities that +are part of the configuration process. + +=== Dependency tracking +When a depfile cache is provided, and the output being built supports +dependency files, any files ending in `.h` and `.hh` that aren't a known +dependency are elided. However, if the build fails, it will be retried with all +header files visible to the client. + +In some cases, this dependency tracking is imprecise, and causes some header +files to not be properly visible to targets that need it. In this case, you can +add an extra entry to the configuration file: + +.`lix.scm` +[,scheme] +---- +(environment: "(pkgs.lix.override { enableDocumentation = false; }).overrideAttrs (a: { doCheck = false; doInstallCheck = false; })" + depfile-path: "lix-depfile.scm" + disallow-elide: (lambda (path) (string=? path "config.h"))) +---- + +This example ensures that `config.h` will be available to all targets being +built for the derivation. + +== Library stubs +When using Meson, any libraries that are linked will also simultaneously +generate a stub `.so` file, using `llvm-ifs`. This stub library is valid for +linking, but contains no code, and is generated deterministically. When linking +any other libraries or binaries that depend on this library, the stub is +automatically used, which improves early cutoff between library builds. diff --git a/lang/ninja/src/build.sld b/lang/ninja/src/build.sld index 6f6d249..febfffc 100644 --- a/lang/ninja/src/build.sld +++ b/lang/ninja/src/build.sld @@ -1,3 +1,4 @@ +;; Turns a Ninja file into a set of binaries. (define-library (zilch lang ninja build) (import (scheme base) (scheme lazy) (scheme file) @@ -8,7 +9,10 @@ (srfi 128) (srfi 146) (srfi 152) (zilch lang ninja) (zilch lang ninja config) (zilch lang ninja depfile)) - (export process-ninja-file + (export + process-ninja-file + + built-edge-edge built-edge-out-drv built-edge-lib-placeholder built-edge-phony-inputs) @@ -17,6 +21,16 @@ (define patchelf (cdr (assoc "out" (nixpkgs "patchelf")))) (define llvm-bintools (cdr (assoc "out" (nixpkgs-eval "llvmPackages_latest.bintools-unwrapped")))) + ;; Represents a single built ``. + ;; + ;; - `edge`: The `` that this `` represents. + ;; - `out-drv`: A store-path that contains all output files of this edge. + ;; - `lib-placeholder`: If set, this built edge represents a library. This + ;; field will then contain a store path that represents the same library as + ;; output in `out-drv`, but with all infnormation other than symbol names + ;; removed. This can be used as a stub while linking other binaries. + ;; - `phony-inputs`: a list of strings representing the edges that make up + ;; this edge. Only set if this edge is phony. (define-record-type (make-built-edge edge out-drv lib-placeholder phony-inputs) built-edge? @@ -315,11 +329,19 @@ (string-suffix? ".h" path))) ;; process a ninja file and corresponding vfs, and return two values: - ;; - `edge-ref`, a lambda that lets one fetch any build edge; - ;; - `defaults`, a list containing the default build edges. ;; - ;; If the `environment` is a or a , it is considered - ;; to be a nixpkgs-style derivation, the same way `nix-shell` works. + ;; - `edge-ref`: A lambda that lets one look up a pair of + ;; `( . )` representing the path of an output, and the + ;; edge that produced that output. + ;; - `defaults`: A list containing the names of the default build edges. + ;; + ;; This procedure takes three arguments: + ;; + ;; - `file`: The parsed ``. + ;; - `conf`: A `` record describing the details on how to + ;; handle this Ninja file. + ;; - `relative-to`: The directory in which the Ninja file was found. Used to + ;; resolve relative references in the Ninja file. (define (process-ninja-file file conf relative-to) (unless (or (string=? relative-to "") (string-suffix? "/" relative-to)) (set! relative-to (string-append relative-to "/"))) diff --git a/lang/ninja/src/config.sld b/lang/ninja/src/config.sld index 22edff8..530a621 100644 --- a/lang/ninja/src/config.sld +++ b/lang/ninja/src/config.sld @@ -1,3 +1,4 @@ +;; Parses the Zilch-specific Ninja configuration. (define-library (zilch lang ninja config) (import (scheme base) (scheme eval) @@ -6,7 +7,7 @@ (prefix (only scheme eval) scheme-)) (export - ninja-build-config? + ninja-build-config? ninja-build-config-environment ninja-build-config-environment-drv ninja-build-config-root-dir ninja-build-config-patches ninja-build-config-targets ninja-build-config-override-source ninja-build-config-depfile ninja-build-config-depfile-path @@ -17,6 +18,8 @@ parse-ninja-config) (begin + ;; Represents a parsed Ninja build configuration. + ;; See `parse-ninja-config` for the definition of these fields. (define-record-type (make-ninja-build-config environment environment-drv root-dir patches targets override-source depfile depfile-path disallow-elide) ninja-build-config? @@ -86,7 +89,35 @@ (set-ninja-build-config-targets! conf (append list-val (ninja-build-config-targets conf)))) (parse-config-inner conf (cddr data))) (else (error (string-append "Unknown directive " (keyword->string (car data)) " parsing Zilch Ninja config"))))))) - + + ;; Parses a Zilch Ninja configuration file. + ;; + ;; A Zilch Ninja configuration file is a list containing repeated keys and values. + ;; + ;; - `env:`/`environment: ...`: The environment to be used when processing + ;; the Ninja file. If the environment is a string, it is assumed to be a + ;; Nix expression resolving to a derivation, and will be evaluated in the + ;; context of Nixpkgs. Otherwise, it is assumed to be a (optionally + ;; `zexp`) alist of environment variables. + ;; - `root: dir`: The root directory that contains both the source and + ;; necessary Ninja build files. If string, assumed to be a directory + ;; relative to the working directory; assumed to be a `` otherwise. + ;; - `override-source: dir`: The source override directory. Only used by + ;; `build-nixpkgs-drv-reproducibly`, ignored otherwise. + ;; - `depfile-path: path`: The location of a depfile cache to use, as a + ;; string relative to the current working directory. Only used by + ;; `zilch-cli-ninja`. + ;; - `depfile: #`: An SRFI146 mapping output file names to a list + ;; of input dependencies. Used to elide inputs where possible. + ;; - `patch: proc`: A patch procedure (or a list which evaluates to one). + ;; A patch procedure takes a `, and outputs `#f` or a string + ;; of shell commands to run before evaluating the rule's command. + ;; - `disallow-elide: proc`: A procedure that takes the path of a file and + ;; can return `#t` to reject its elision. Used to special-case files that + ;; would normally be elided, but shouldn't, in cases where the depfile is + ;; incorrect. + ;; - `target: "foo"`/`targets: '("foo" "bar")`: Build these targets instead + ;; of the default specified in the Ninja file. (define (parse-ninja-config config) (unless (list? config) (error "expected Zilch Ninja config to be a list")) diff --git a/lang/ninja/src/depfile.sld b/lang/ninja/src/depfile.sld index 566e17e..7d39136 100644 --- a/lang/ninja/src/depfile.sld +++ b/lang/ninja/src/depfile.sld @@ -1,3 +1,5 @@ +;; Implements the bare minimum Makefile parser to read `.d` files generated by +;; GCC, rustc, and more. (define-library (zilch lang ninja depfile) (import (scheme base) @@ -7,6 +9,8 @@ parse-depfile) (begin + ;; Parse a depfile. Takes a bytevector representing the depfile, and returns + ;; a mapping of output path to input dependencies. (define (parse-depfile bytes) (define buf (make-bytevector 1024 0)) (define buf-i 0) diff --git a/lang/ninja/src/ninja.sld b/lang/ninja/src/ninja.sld index 4ccb6de..c5bbac2 100644 --- a/lang/ninja/src/ninja.sld +++ b/lang/ninja/src/ninja.sld @@ -1,3 +1,8 @@ +;; Processes Ninja files into Scheme records. +;; +;; This library uses a special format to store strings that contain variables. +;; If a string can contain variables, it may be represented as a list, where +;; each value of the list is either a string, or a pair `'(varname . "variable_name")`. (define-library (zilch lang ninja) (import (scheme base) (scheme write) (scheme process-context) (scheme lazy) @@ -12,7 +17,6 @@ (export make-build-file build-file? build-file-global-variables build-file-default-targets build-file-rules build-file-build-edges - build-file-pools make-build-rule build-rule? build-rule-name build-rule-command build-rule-depfile build-rule-deps build-rule-description @@ -121,14 +125,20 @@ (define (pr out name value) (unless (eq? value #f) (fprintf out " ~A = ~A\n" name (render-evalstring value #f)))) ;; Represents a Ninja build file. + ;; + ;; - `global-variables`: An SRFI146 mapping containing all variables defined + ;; globally. + ;; - `default-targets`: A list of target names that should be built if no + ;; specific target is requested. + ;; - `rules`: A mapping of rule names to `` records. + ;; - `build-edges`: A list of `` records. (define-record-type - (make-build-file global-variables default-targets rules build-edges pools) + (make-build-file global-variables default-targets rules build-edges) build-file? (global-variables build-file-global-variables set-build-file-global-variables!) (default-targets build-file-default-targets set-build-file-default-targets!) (rules build-file-rules set-build-file-rules!) - (build-edges build-file-build-edges set-build-file-build-edges!) - (pools build-file-pools set-build-file-pools!)) + (build-edges build-file-build-edges set-build-file-build-edges!)) (define-record-printer ( file out) (fprintf out "# debug build file\n") @@ -141,7 +151,8 @@ (for-each (lambda (edge) (fprintf out "~S" edge)) (build-file-build-edges file))) ;; Represents a Ninja build rule. - ; variables that are ignored for now: dyndep, generator, pool, msvc_deps_prefix + ;; Each of the fields of this record represent a variable that can be set + ;; in a Ninja build rule. (define-record-type (make-build-rule name command depfile deps description restat rspfile rspfile-content) build-rule? @@ -165,7 +176,14 @@ (pr out "rspfile-content" (build-rule-rspfile-content rule)) (fprintf out "\n")) - ;; Represents a Ninja build edge (aka one or more files built by this Ninja file) + ;; Represents a Ninja build edge (aka a Ninja build statement). + ;; + ;; - `rule`: The name of the build rule to use for this edge. + ;; - `outputs`/`inputs`/`implicit-dependencies`/`order-only-dependencies`/`validations`/`implicit-outputs`: + ;; The corresponding parsed part of the build statement, represented as a + ;; possibly-variable-containing string. + ;; - `variables`: A mapping of variable name to a possibly-variable-containing string. + ;; - `resolved`: A `` record where each value has its variables already resolved. (define-record-type (make-build-edge rule outputs inputs implicit-dependencies order-only-dependencies validations implicit-outputs variables resolved) build-edge? @@ -230,7 +248,6 @@ #f (inner-loop 0))) - ;; Reads a full Ninja file, returning a record. (define (read-ninja-file-inner strval into read-other-file) (define i 0) (define (eat-whitespace) @@ -463,7 +480,15 @@ (read-toplevel file))) file) (read-toplevel into)) + + ;; Parses a full Ninja file from a string, returning a record + ;; representing the file. + ;; + ;; `read-other-file` is a procedure used to read a Ninja file relative to + ;; this one, used to implement the `include` keyword. It is expected to take + ;; a string representing the path to read, and to return a string containing + ;; the full contents of said Ninja file. (define (read-ninja-file strval read-other-file) - (define out-file (read-ninja-file-inner strval (make-build-file (mapping (make-default-comparator)) '() (mapping (make-default-comparator)) '() (mapping (make-default-comparator))) read-other-file)) + (define out-file (read-ninja-file-inner strval (make-build-file (mapping (make-default-comparator)) '() (mapping (make-default-comparator)) '()) read-other-file)) (for-each (lambda (f) (unless (string=? (build-edge-rule f) "phony") (set-build-edge-resolved! f (build-rule-resolve (mapping-ref (build-file-rules out-file) (build-edge-rule f)) f out-file)))) (build-file-build-edges out-file)) out-file))) diff --git a/lang/ninja/src/nixpkgs.sld b/lang/ninja/src/nixpkgs.sld index 3f4d674..fb28516 100644 --- a/lang/ninja/src/nixpkgs.sld +++ b/lang/ninja/src/nixpkgs.sld @@ -1,3 +1,4 @@ +;; Helpers to work with Zilch around Nixpkgs derivations. (define-library (zilch lang ninja nixpkgs) (import (scheme base) (chicken format) (scheme lazy) @@ -74,7 +75,23 @@ (set! output (string-copy output 0 32))) (string-append "/nix/store/" output (string-copy base-placeholder (string-length output)) name (if (string=? output "out") "" (string-append "-" output)))) - + + ;; Takes a `` representing a Nixpkgs derivation, and + ;; preprocesses the derivation such that it can be reconstituted once Zilch + ;; has taken over the Ninja build requirements. + ;; + ;; This procedure is used internally, and shouldn't be relied upon; it + ;; encodes many specific parts that are unlikely to be useful by external + ;; parties. + ;; + ;; Returns 6 values: + ;; + ;; - The initial Nixpkgs derivation as `` + ;; - The Nixpkgs derivation after running all phases up to and including + ;; configuration + ;; - An alist of output names to placeholder store paths + ;; - The `edge-ref`, `defaults`, and `export-depfile` values from calling + ;; `process-ninja-file` (define (setup-ninja-environment conf) (define initial-drv (ninja-build-config-environment-drv conf)) (when (store-path? initial-drv) @@ -125,7 +142,17 @@ (define-values (edge-ref defaults export-depfile) (process-ninja-file ninja-file conf "build")) (values initial-drv configured-drv placeholders edge-ref defaults export-depfile)) - + + ;; Takes a `` representing a Nixpkgs derivation, and + ;; build it using Zilch. + ;; + ;; Returns two values: + ;; + ;; - An alist of output name to store paths, representing the built + ;; derivation + ;; - A procedure that, when called, returns a mapping of output path to + ;; necessary inputs generated from the depfile data. This can be stored in + ;; a file for later rebuilds. (define (build-nixpkgs-drv-reproducibly conf) (define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf))