puck/zilch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

(zilch lang ninja): document

Browse source 
Change-Id: I6a6a6964dcc713504ec57f40981a601696a573af
This commit is contained in:
puck 2025-07-03 18:57:48 +00:00
parent 154ba9be1c
commit 93a1ebba00
11 changed files with 329 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

1
cli/default.nix
View file

@ -17,5 +17,6 @@ in eggDerivation {
(cat ${./overrides.json}; printf '\0') | ${xxd}/bin/xxd -i -n stock_overrides > stock_overrides.h
(cat ${manPages.rust.txt}; printf '\0') | ${xxd}/bin/xxd -i -n man_rust > man_rust.h
(cat ${manPages.go.txt}; printf '\0') | ${xxd}/bin/xxd -i -n man_go > man_go.h
(cat ${manPages.ninja.txt}; printf '\0') | ${xxd}/bin/xxd -i -n man_ninja > man_ninja.h
'';
}

30
cli/zilch-ninja.scm
View file

@ -5,35 +5,13 @@
(foreign-lambda* int ()
"cpu_set_t set; sched_getaffinity(0, sizeof(set), &set); C_return(CPU_COUNT(&set));"))
(foreign-declare "#include \"man_ninja.h\"")
(define man-page (foreign-value "man_ninja" nonnull-c-string))
(define (print-help msg)
(when msg
(write-string (string-append msg "\n\n") (current-error-port)))
(write-string "Usage: zilch-cli-ninja [OPTION] SUBCOMMAND ...
Processes a Ninja build based on the configuration in the passed-in
config file (or zilch.scm, if not set), and reproducibly builds either
to final build, or to specific Ninja targets.
Supported commands:
build [TARGET] Build the full drv, or specific targets from
its Ninja file.
source [DIR] Extract the source of the derivation to DIR
(or src, if unspecified), ready for Zilch.
diff Print the difference between the original and
modified sources as found in either `src' or
the directory selected by --source.
Arguments:
-h, --help Print this help message.
-f, --config-file PATH Path to the Zilch config file.
-s, --source DIR Override the input source for builds. Doesn't
reconfigure the build, so changes to the
build system will not apply.
-j, --max-jobs COUNT The maximum amount of builds to run. Defaults
to the amount of cores.
-v, --verbose Increase the verbosity configured in the Nix
daemon.
-L, --print-build-logs Print derivation logs as they come in.
" (current-error-port))
(write-string man-page (current-error-port))
(exit (or (not msg) 1)))
(define-values (options args)

1
docs/manpages.nix
View file

@ -10,4 +10,5 @@
in {
rust = generateBoth "zilch-cli-rust.1" ./modules/ROOT/pages/rust/man.adoc;
go = generateBoth "zilch-cli-go.1" ./modules/ROOT/pages/go/man.adoc;
ninja = generateBoth "zilch-cli-ninja.1" ./modules/ROOT/pages/ninja/man.adoc;
}

3
docs/modules/ROOT/nav.adoc
View file

@ -14,6 +14,9 @@
*** xref:rust/usage.adoc[]
*** xref:rust/library.adoc[]
*** xref:rust/man.adoc[Man page]
** Ninja
*** xref:ninja/usage.adoc[]
*** xref:ninja/man.adoc[Man page]
* Code reference
include::generated:partial$nav.adoc[]

63
docs/modules/ROOT/pages/ninja/man.adoc Normal file
View file

@ -0,0 +1,63 @@
= zilch-cli-ninja(1)
Puck Meerburg
v0.0.1
:doctype: manpage
:manmanual: ZILCH-CLI-NINJA
:mansource: ZILCH
:page-pagination: prev
== Name
zilch-cli-ninja - builds a Nixpkgs package using Ninja, using Zilch
== Synopsis
*zilch-cli-ninja* [_OPTION_]... _COMMAND_ [_ARG_]...
== Description
This command uses Zilch to build a Ninja-based project entirely within
Nix, using content-addressed derivations.
This program requires the derivation to build has a configuration file
written. See the documentation at https://puck.moe/zilch/docs/.
== Subcommands
*build* [_TARGET_]...::
Build the derivation described by the configuration file, or the
target nammes from its Ninja file, if any are passed as arguments.
*source* [_DIR_]::
Write the source of the derivation to _DIR_, or to `src` if no
directory is provided.
*diff*::
Compare the 'canonical' source of the derivation to that in the
source directory (`src`, or the path selected by `--source`.)
== Options
*-h*::
*--help*::
Print a help message.
*-s* _DIR_::
*--source* _DIR_::
Override the input source for builds. Doesn't reconfigure the build,
so changes to the build system do not apply.
*-f* _PATH_::
*--config-file* _PATH_::
The location of the configuration file. Defaults to `zilch.scm`.
*-j* _COUNT_::
*--max-jobs* _COUNT_::
The maximum amount of builds to run. Defaults to the amount of cores.
*-v*::
*--verbose*::
Increase the verbosity configured in the Nix daemon.
*-L*::
*--print-build-logs*::
Print derivation logs as they come in.

131
docs/modules/ROOT/pages/ninja/usage.adoc Normal file
View file

@ -0,0 +1,131 @@
= Usage
:page-pagination: next
`zilch-cli-ninja` allows you to use Zilch to compile a Ninja-based nixpkgs
derivation reproducibly, and to quickly modify it and rebuild it reproducibly
as well. To use the tool, you first have to write a configuration file. This
document will use this example, for `gtest`:
.`gtest.scm`
[,scheme]
----
(environment: "pkgs.gtest"
depfile-path: "gtest-deps.scm")
----
This can then be used with the various subcommands of `zilch-cli-ninja`:
== Building
To build a derivation with `zilch-cli-ninja`, use the `build` subcommand.
[,console]
----
$ zilch-cli-ninja -f gtest.scm build <1>
…
dev -> #<store path /nix/store/lmdgblqm0prkx3hcs2q76rzcvg1xm6cm-gtest-1.16.0-dev (/nix/store/ksrbqg2nkgm8zq3g59gzydf5976griis-gtest-1.16.0.drv!dev)>
out -> #<store path /nix/store/b2ylcnfsaf2wmzk51whgq5r86xb955xq-gtest-1.16.0 (/nix/store/ksrbqg2nkgm8zq3g59gzydf5976griis-gtest-1.16.0.drv!out)>
$ zilch-cli-ninja -f gtest.scm build googlemock/CMakeFiles/gmock_main.dir/src/gmock_main.cc.o <2>
…
googlemock/CMakeFiles/gmock_main.dir/src/gmock_main.cc.o -> "/nix/store/c96j8sqk08xj15pififz5myjzskg0cqw-Building-CXX-object-googlemock-CMakeFiles-gmock_main.dir-src-gmock_main.cc.o"
----
<1> Build the full derivation
<2> Build a specific output of the Ninja file
== Source editing
Zilch makes it very easy to edit the source of any derivation built in this
manner. With one command, you can export the source code used for building.
This source can then be used to rebuild parts of the derivation that have changed.
[,console]
----
$ zilch-cli-ninja -f gtest.scm source gtest-src <1>
…
$ zilch-cli-ninja -f gtest.scm -s gtest-src diff <2>
…
$ zilch-cli-ninja -f gtest.scm -s gtest-src build <3>
…
dev -> #<store path /nix/store/lmdgblqm0prkx3hcs2q76rzcvg1xm6cm-gtest-1.16.0-dev (/nix/store/ksrbqg2nkgm8zq3g59gzydf5976griis-gtest-1.16.0.drv!dev)>
out -> #<store path /nix/store/b2ylcnfsaf2wmzk51whgq5r86xb955xq-gtest-1.16.0 (/nix/store/ksrbqg2nkgm8zq3g59gzydf5976griis-gtest-1.16.0.drv!out)>
----
WARNING: Zilch will not regenerate the build files when editing source in this
manner, for performance and logistical reasons. To change the build files,
override the `src` attribute of the environment's derivation.
== Limitations
To make the Nixpkgs integration work, the Zilch code breaks up the derivation's
phases into two steps: unpack + patch + configure, and check + install. This
can cause issues with derivations that expect to share variables between these
phases. If `meson` is present, it is assumed to be used, otherwise it is
assumed the derivation is `cmake`-based. Other build systems are not yet
supported.
The final store path of the derivation is not known at configuration time,
unlike normal derivations. Meson and CMake are configured to install into a
prefix, and are then rewritten (using `sed`) to the final output store path.
This may cause issues with programs that e.g. compress a file containing these
paths during their build.footnote:[The only one to have known issues is Node.]
== Patches
In some cases, the code may not handle running in Zilch's environment. In this
case, it's possible to add extra code to run before running the command of a
target.
These patches are defined as part of the configuration file, and are arbitrary
Scheme code:
[,scheme]
----
(environment: "import ./sample.drv"
patch:
(lambda (edge)
(if (string=? (build-edge-rule edge) "foo")
"rm src/foo; echo 'replacement file' > src/foo"
#f)))
----
Patches are arbitrary shell scripts, run with GNU Coreutils in `$PATH`. The
environment variable `$_ZILCH_ROOT` points to the root of the source + build
file tree, and can be used to copy non-symlinked files into the working
directory.
== Automatic file elision
To implement incremental builds efficiently, Zilch automatically slims down the
part of the source directory that is accessible to each target. If a filename
ends in `.cc` or `.c` and is directly depended upon as in input file to a
target, it will be elided, and only the targets that depend on it directly will
be able to read its contents.
The `build/meson-private` directory is fully elided, except for derivations
that directly depend upon it. This is to limit the amount of impurities that
are part of the configuration process.
=== Dependency tracking
When a depfile cache is provided, and the output being built supports
dependency files, any files ending in `.h` and `.hh` that aren't a known
dependency are elided. However, if the build fails, it will be retried with all
header files visible to the client.
In some cases, this dependency tracking is imprecise, and causes some header
files to not be properly visible to targets that need it. In this case, you can
add an extra entry to the configuration file:
.`lix.scm`
[,scheme]
----
(environment: "(pkgs.lix.override { enableDocumentation = false; }).overrideAttrs (a: { doCheck = false; doInstallCheck = false; })"
depfile-path: "lix-depfile.scm"
disallow-elide: (lambda (path) (string=? path "config.h")))
----
This example ensures that `config.h` will be available to all targets being
built for the derivation.
== Library stubs
When using Meson, any libraries that are linked will also simultaneously
generate a stub `.so` file, using `llvm-ifs`. This stub library is valid for
linking, but contains no code, and is generated deterministically. When linking
any other libraries or binaries that depend on this library, the stub is
automatically used, which improves early cutoff between library builds.

32
lang/ninja/src/build.sld
View file

@ -1,3 +1,4 @@
;; Turns a Ninja file into a set of binaries.
(define-library (zilch lang ninja build)
(import
(scheme base) (scheme lazy) (scheme file)
@ -8,7 +9,10 @@
(srfi 128) (srfi 146) (srfi 152)
(zilch lang ninja) (zilch lang ninja config) (zilch lang ninja depfile))
(export process-ninja-file
(export
process-ninja-file
<built-edge>
built-edge-edge built-edge-out-drv
built-edge-lib-placeholder built-edge-phony-inputs)
@ -17,6 +21,16 @@
(define patchelf (cdr (assoc "out" (nixpkgs "patchelf"))))
(define llvm-bintools (cdr (assoc "out" (nixpkgs-eval "llvmPackages_latest.bintools-unwrapped"))))
;; Represents a single built `<build-edge>`.
;;
;; - `edge`: The `<build-edge>` that this `<built-edge>` represents.
;; - `out-drv`: A store-path that contains all output files of this edge.
;; - `lib-placeholder`: If set, this built edge represents a library. This
;; field will then contain a store path that represents the same library as
;; output in `out-drv`, but with all infnormation other than symbol names
;; removed. This can be used as a stub while linking other binaries.
;; - `phony-inputs`: a list of strings representing the edges that make up
;; this edge. Only set if this edge is phony.
(define-record-type <built-edge>
(make-built-edge edge out-drv lib-placeholder phony-inputs)
built-edge?
@ -315,11 +329,19 @@
(string-suffix? ".h" path)))
;; process a ninja file and corresponding vfs, and return two values:
;; - `edge-ref`, a lambda that lets one fetch any build edge;
;; - `defaults`, a list containing the default build edges.
;;
;; If the `environment` is a <derivation> or a <store-path>, it is considered
;; to be a nixpkgs-style derivation, the same way `nix-shell` works.
;; - `edge-ref`: A lambda that lets one look up a pair of
;; `(<zexp> . <built-edge>)` representing the path of an output, and the
;; edge that produced that output.
;; - `defaults`: A list containing the names of the default build edges.
;;
;; This procedure takes three arguments:
;;
;; - `file`: The parsed `<build-file>`.
;; - `conf`: A `<ninja-build-config>` record describing the details on how to
;; handle this Ninja file.
;; - `relative-to`: The directory in which the Ninja file was found. Used to
;; resolve relative references in the Ninja file.
(define (process-ninja-file file conf relative-to)
(unless (or (string=? relative-to "") (string-suffix? "/" relative-to)) (set! relative-to (string-append relative-to "/")))

35
lang/ninja/src/config.sld
View file

@ -1,3 +1,4 @@
;; Parses the Zilch-specific Ninja configuration.
(define-library (zilch lang ninja config)
(import
(scheme base) (scheme eval)
@ -6,7 +7,7 @@
(prefix (only scheme eval) scheme-))
(export
ninja-build-config?
<ninja-build-config> ninja-build-config?
ninja-build-config-environment ninja-build-config-environment-drv
ninja-build-config-root-dir ninja-build-config-patches ninja-build-config-targets
ninja-build-config-override-source ninja-build-config-depfile ninja-build-config-depfile-path
@ -17,6 +18,8 @@
parse-ninja-config)
(begin
;; Represents a parsed Ninja build configuration.
;; See `parse-ninja-config` for the definition of these fields.
(define-record-type <ninja-build-config>
(make-ninja-build-config environment environment-drv root-dir patches targets override-source depfile depfile-path disallow-elide)
ninja-build-config?
@ -86,7 +89,35 @@
(set-ninja-build-config-targets! conf (append list-val (ninja-build-config-targets conf))))
(parse-config-inner conf (cddr data)))
(else (error (string-append "Unknown directive " (keyword->string (car data)) " parsing Zilch Ninja config")))))))
;; Parses a Zilch Ninja configuration file.
;;
;; A Zilch Ninja configuration file is a list containing repeated keys and values.
;;
;; - `env:`/`environment: ...`: The environment to be used when processing
;; the Ninja file. If the environment is a string, it is assumed to be a
;; Nix expression resolving to a derivation, and will be evaluated in the
;; context of Nixpkgs. Otherwise, it is assumed to be a (optionally
;; `zexp`) alist of environment variables.
;; - `root: dir`: The root directory that contains both the source and
;; necessary Ninja build files. If string, assumed to be a directory
;; relative to the working directory; assumed to be a `<vfs>` otherwise.
;; - `override-source: dir`: The source override directory. Only used by
;; `build-nixpkgs-drv-reproducibly`, ignored otherwise.
;; - `depfile-path: path`: The location of a depfile cache to use, as a
;; string relative to the current working directory. Only used by
;; `zilch-cli-ninja`.
;; - `depfile: #<mapping>`: An SRFI146 mapping output file names to a list
;; of input dependencies. Used to elide inputs where possible.
;; - `patch: proc`: A patch procedure (or a list which evaluates to one).
;; A patch procedure takes a `<build-edge>, and outputs `#f` or a string
;; of shell commands to run before evaluating the rule's command.
;; - `disallow-elide: proc`: A procedure that takes the path of a file and
;; can return `#t` to reject its elision. Used to special-case files that
;; would normally be elided, but shouldn't, in cases where the depfile is
;; incorrect.
;; - `target: "foo"`/`targets: '("foo" "bar")`: Build these targets instead
;; of the default specified in the Ninja file.
(define (parse-ninja-config config)
(unless (list? config)
(error "expected Zilch Ninja config to be a list"))

4
lang/ninja/src/depfile.sld
View file

@ -1,3 +1,5 @@
;; Implements the bare minimum Makefile parser to read `.d` files generated by
;; GCC, rustc, and more.
(define-library (zilch lang ninja depfile)
(import
(scheme base)
@ -7,6 +9,8 @@
parse-depfile)
(begin
;; Parse a depfile. Takes a bytevector representing the depfile, and returns
;; a mapping of output path to input dependencies.
(define (parse-depfile bytes)
(define buf (make-bytevector 1024 0))
(define buf-i 0)

41
lang/ninja/src/ninja.sld
View file

@ -1,3 +1,8 @@
;; Processes Ninja files into Scheme records.
;;
;; This library uses a special format to store strings that contain variables.
;; If a string can contain variables, it may be represented as a list, where
;; each value of the list is either a string, or a pair `'(varname . "variable_name")`.
(define-library (zilch lang ninja)
(import
(scheme base) (scheme write) (scheme process-context) (scheme lazy)
@ -12,7 +17,6 @@
(export
<build-file> make-build-file build-file?
build-file-global-variables build-file-default-targets build-file-rules build-file-build-edges
build-file-pools
<build-rule> make-build-rule build-rule?
build-rule-name build-rule-command build-rule-depfile build-rule-deps build-rule-description
@ -121,14 +125,20 @@
(define (pr out name value) (unless (eq? value #f) (fprintf out " ~A = ~A\n" name (render-evalstring value #f))))
;; Represents a Ninja build file.
;;
;; - `global-variables`: An SRFI146 mapping containing all variables defined
;; globally.
;; - `default-targets`: A list of target names that should be built if no
;; specific target is requested.
;; - `rules`: A mapping of rule names to `<build-rule>` records.
;; - `build-edges`: A list of `<build-edge>` records.
(define-record-type <build-file>
(make-build-file global-variables default-targets rules build-edges pools)
(make-build-file global-variables default-targets rules build-edges)
build-file?
(global-variables build-file-global-variables set-build-file-global-variables!)
(default-targets build-file-default-targets set-build-file-default-targets!)
(rules build-file-rules set-build-file-rules!)
(build-edges build-file-build-edges set-build-file-build-edges!)
(pools build-file-pools set-build-file-pools!))
(build-edges build-file-build-edges set-build-file-build-edges!))
(define-record-printer (<build-file> file out)
(fprintf out "# debug build file\n")
@ -141,7 +151,8 @@
(for-each (lambda (edge) (fprintf out "~S" edge)) (build-file-build-edges file)))
;; Represents a Ninja build rule.
; variables that are ignored for now: dyndep, generator, pool, msvc_deps_prefix
;; Each of the fields of this record represent a variable that can be set
;; in a Ninja build rule.
(define-record-type <build-rule>
(make-build-rule name command depfile deps description restat rspfile rspfile-content)
build-rule?
@ -165,7 +176,14 @@
(pr out "rspfile-content" (build-rule-rspfile-content rule))
(fprintf out "\n"))
;; Represents a Ninja build edge (aka one or more files built by this Ninja file)
;; Represents a Ninja build edge (aka a Ninja build statement).
;;
;; - `rule`: The name of the build rule to use for this edge.
;; - `outputs`/`inputs`/`implicit-dependencies`/`order-only-dependencies`/`validations`/`implicit-outputs`:
;; The corresponding parsed part of the build statement, represented as a
;; possibly-variable-containing string.
;; - `variables`: A mapping of variable name to a possibly-variable-containing string.
;; - `resolved`: A `<build-rule>` record where each value has its variables already resolved.
(define-record-type <build-edge>
(make-build-edge rule outputs inputs implicit-dependencies order-only-dependencies validations implicit-outputs variables resolved)
build-edge?
@ -230,7 +248,6 @@
#f
(inner-loop 0)))
;; Reads a full Ninja file, returning a <build-file> record.
(define (read-ninja-file-inner strval into read-other-file)
(define i 0)
(define (eat-whitespace)
@ -463,7 +480,15 @@
(read-toplevel file)))
file)
(read-toplevel into))
;; Parses a full Ninja file from a string, returning a <build-file> record
;; representing the file.
;;
;; `read-other-file` is a procedure used to read a Ninja file relative to
;; this one, used to implement the `include` keyword. It is expected to take
;; a string representing the path to read, and to return a string containing
;; the full contents of said Ninja file.
(define (read-ninja-file strval read-other-file)
(define out-file (read-ninja-file-inner strval (make-build-file (mapping (make-default-comparator)) '() (mapping (make-default-comparator)) '() (mapping (make-default-comparator))) read-other-file))
(define out-file (read-ninja-file-inner strval (make-build-file (mapping (make-default-comparator)) '() (mapping (make-default-comparator)) '()) read-other-file))
(for-each (lambda (f) (unless (string=? (build-edge-rule f) "phony") (set-build-edge-resolved! f (build-rule-resolve (mapping-ref (build-file-rules out-file) (build-edge-rule f)) f out-file)))) (build-file-build-edges out-file))
out-file)))

31
lang/ninja/src/nixpkgs.sld
View file

@ -1,3 +1,4 @@
;; Helpers to work with Zilch around Nixpkgs derivations.
(define-library (zilch lang ninja nixpkgs)
(import
(scheme base) (chicken format) (scheme lazy)
@ -74,7 +75,23 @@
(set! output (string-copy output 0 32)))
(string-append "/nix/store/" output (string-copy base-placeholder (string-length output)) name (if (string=? output "out") "" (string-append "-" output))))
;; Takes a `<ninja-build-config>` representing a Nixpkgs derivation, and
;; preprocesses the derivation such that it can be reconstituted once Zilch
;; has taken over the Ninja build requirements.
;;
;; This procedure is used internally, and shouldn't be relied upon; it
;; encodes many specific parts that are unlikely to be useful by external
;; parties.
;;
;; Returns 6 values:
;;
;; - The initial Nixpkgs derivation as `<derivation>`
;; - The Nixpkgs derivation after running all phases up to and including
;; configuration
;; - An alist of output names to placeholder store paths
;; - The `edge-ref`, `defaults`, and `export-depfile` values from calling
;; `process-ninja-file`
(define (setup-ninja-environment conf)
(define initial-drv (ninja-build-config-environment-drv conf))
(when (store-path? initial-drv)
@ -125,7 +142,17 @@
(define-values (edge-ref defaults export-depfile) (process-ninja-file ninja-file conf "build"))
(values initial-drv configured-drv placeholders edge-ref defaults export-depfile))
;; Takes a `<ninja-build-config>` representing a Nixpkgs derivation, and
;; build it using Zilch.
;;
;; Returns two values:
;;
;; - An alist of output name to store paths, representing the built
;; derivation
;; - A procedure that, when called, returns a mapping of output path to
;; necessary inputs generated from the depfile data. This can be stored in
;; a file for later rebuilds.
(define (build-nixpkgs-drv-reproducibly conf)
(define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf))