(zilch lang ninja): document

Change-Id: I6a6a6964dcc713504ec57f40981a601696a573af

cli/default.nix

@@ -17,5 +17,6 @@ in eggDerivation {
     (cat ${./overrides.json}; printf '\0') | ${xxd}/bin/xxd -i -n stock_overrides > stock_overrides.h
     (cat ${manPages.rust.txt}; printf '\0') | ${xxd}/bin/xxd -i -n man_rust > man_rust.h
     (cat ${manPages.go.txt}; printf '\0') | ${xxd}/bin/xxd -i -n man_go > man_go.h
+    (cat ${manPages.ninja.txt}; printf '\0') | ${xxd}/bin/xxd -i -n man_ninja > man_ninja.h
   '';
 }

cli/zilch-ninja.scm

@@ -5,35 +5,13 @@
   (foreign-lambda* int ()
     "cpu_set_t set; sched_getaffinity(0, sizeof(set), &set); C_return(CPU_COUNT(&set));"))
 
+(foreign-declare "#include \"man_ninja.h\"")
+(define man-page (foreign-value "man_ninja" nonnull-c-string))
+
 (define (print-help msg)
   (when msg
     (write-string (string-append msg "\n\n") (current-error-port)))
-  (write-string "Usage: zilch-cli-ninja [OPTION] SUBCOMMAND ...
+  (write-string man-page (current-error-port))
-Processes a Ninja build based on the configuration in the passed-in
-config file (or zilch.scm, if not set), and reproducibly builds either
-to final build, or to specific Ninja targets.
-
-Supported commands:
-  build [TARGET]          Build the full drv, or specific targets from
-                          its Ninja file.
-  source [DIR]            Extract the source of the derivation to DIR
-                          (or src, if unspecified), ready for Zilch.
-  diff                    Print the difference between the original and
-                          modified sources as found in either `src' or
-                          the directory selected by --source.
-
-Arguments:
-  -h, --help              Print this help message.
-  -f, --config-file PATH  Path to the Zilch config file.
-  -s, --source DIR        Override the input source for builds. Doesn't
-                          reconfigure the build, so changes to the
-                          build system will not apply.
-  -j, --max-jobs COUNT    The maximum amount of builds to run. Defaults
-                          to the amount of cores.
-  -v, --verbose           Increase the verbosity configured in the Nix
-                          daemon.
-  -L, --print-build-logs  Print derivation logs as they come in.
-" (current-error-port))
   (exit (or (not msg) 1)))
 
 (define-values (options args)

docs/manpages.nix

@@ -10,4 +10,5 @@
 in {
   rust = generateBoth "zilch-cli-rust.1" ./modules/ROOT/pages/rust/man.adoc;
   go = generateBoth "zilch-cli-go.1" ./modules/ROOT/pages/go/man.adoc;
+  ninja = generateBoth "zilch-cli-ninja.1" ./modules/ROOT/pages/ninja/man.adoc;
 }

docs/modules/ROOT/nav.adoc

@@ -14,6 +14,9 @@
 *** xref:rust/usage.adoc[]
 *** xref:rust/library.adoc[]
 *** xref:rust/man.adoc[Man page]
+** Ninja
+*** xref:ninja/usage.adoc[]
+*** xref:ninja/man.adoc[Man page]
 
 * Code reference
 include::generated:partial$nav.adoc[]

docs/modules/ROOT/pages/ninja/man.adoc

@@ -0,0 +1,63 @@
+= zilch-cli-ninja(1)
+Puck Meerburg
+v0.0.1
+:doctype: manpage
+:manmanual: ZILCH-CLI-NINJA
+:mansource: ZILCH
+:page-pagination: prev
+
+== Name
+
+zilch-cli-ninja - builds a Nixpkgs package using Ninja, using Zilch
+
+== Synopsis
+
+*zilch-cli-ninja* [_OPTION_]... _COMMAND_ [_ARG_]...
+
+== Description
+
+This command uses Zilch to build a Ninja-based project entirely within
+Nix, using content-addressed derivations.
+
+This program requires the derivation to build has a configuration file
+written. See the documentation at https://puck.moe/zilch/docs/.
+
+== Subcommands
+*build* [_TARGET_]...::
+  Build the derivation described by the configuration file, or the
+  target nammes from its Ninja file, if any are passed as arguments.
+
+*source* [_DIR_]::
+  Write the source of the derivation to _DIR_, or to `src` if no
+  directory is provided.
+
+*diff*::
+  Compare the 'canonical' source of the derivation to that in the
+  source directory (`src`, or the path selected by `--source`.)
+
+== Options
+
+*-h*::
+*--help*::
+  Print a help message.
+
+*-s* _DIR_::
+*--source* _DIR_::
+  Override the input source for builds. Doesn't reconfigure the build,
+  so changes to the build system do not apply.
+
+*-f* _PATH_::
+*--config-file* _PATH_::
+  The location of the configuration file. Defaults to `zilch.scm`.
+
+*-j* _COUNT_::
+*--max-jobs* _COUNT_::
+  The maximum amount of builds to run. Defaults to the amount of cores.
+
+*-v*::
+*--verbose*::
+  Increase the verbosity configured in the Nix daemon.
+
+*-L*::
+*--print-build-logs*::
+  Print derivation logs as they come in.

docs/modules/ROOT/pages/ninja/usage.adoc

@@ -0,0 +1,131 @@
+= Usage
+:page-pagination: next
+
+`zilch-cli-ninja` allows you to use Zilch to compile a Ninja-based nixpkgs
+derivation reproducibly, and to quickly modify it and rebuild it reproducibly
+as well. To use the tool, you first have to write a configuration file. This
+document will use this example, for `gtest`:
+
+.`gtest.scm`
+[,scheme]
+----
+(environment: "pkgs.gtest"
+ depfile-path: "gtest-deps.scm")
+----
+
+This can then be used with the various subcommands of `zilch-cli-ninja`:
+
+== Building
+To build a derivation with `zilch-cli-ninja`, use the `build` subcommand.
+[,console]
+----
+$ zilch-cli-ninja -f gtest.scm build <1>
+…
+dev	-> #<store path /nix/store/lmdgblqm0prkx3hcs2q76rzcvg1xm6cm-gtest-1.16.0-dev (/nix/store/ksrbqg2nkgm8zq3g59gzydf5976griis-gtest-1.16.0.drv!dev)>
+out	-> #<store path /nix/store/b2ylcnfsaf2wmzk51whgq5r86xb955xq-gtest-1.16.0 (/nix/store/ksrbqg2nkgm8zq3g59gzydf5976griis-gtest-1.16.0.drv!out)>
+
+$ zilch-cli-ninja -f gtest.scm build googlemock/CMakeFiles/gmock_main.dir/src/gmock_main.cc.o <2>
+…
+googlemock/CMakeFiles/gmock_main.dir/src/gmock_main.cc.o	-> "/nix/store/c96j8sqk08xj15pififz5myjzskg0cqw-Building-CXX-object-googlemock-CMakeFiles-gmock_main.dir-src-gmock_main.cc.o"
+----
+<1> Build the full derivation
+<2> Build a specific output of the Ninja file
+
+== Source editing
+Zilch makes it very easy to edit the source of any derivation built in this
+manner. With one command, you can export the source code used for building.
+This source can then be used to rebuild parts of the derivation that have changed.
+
+[,console]
+----
+$ zilch-cli-ninja -f gtest.scm source gtest-src <1>
+…
+
+$ zilch-cli-ninja -f gtest.scm -s gtest-src diff <2>
+…
+
+$ zilch-cli-ninja -f gtest.scm -s gtest-src build <3>
+…
+dev	-> #<store path /nix/store/lmdgblqm0prkx3hcs2q76rzcvg1xm6cm-gtest-1.16.0-dev (/nix/store/ksrbqg2nkgm8zq3g59gzydf5976griis-gtest-1.16.0.drv!dev)>
+out	-> #<store path /nix/store/b2ylcnfsaf2wmzk51whgq5r86xb955xq-gtest-1.16.0 (/nix/store/ksrbqg2nkgm8zq3g59gzydf5976griis-gtest-1.16.0.drv!out)>
+----
+
+WARNING: Zilch will not regenerate the build files when editing source in this
+manner, for performance and logistical reasons. To change the build files,
+override the `src` attribute of the environment's derivation.
+
+== Limitations
+To make the Nixpkgs integration work, the Zilch code breaks up the derivation's
+phases into two steps: unpack + patch + configure, and check + install. This
+can cause issues with derivations that expect to share variables between these
+phases. If `meson` is present, it is assumed to be used, otherwise it is
+assumed the derivation is `cmake`-based. Other build systems are not yet
+supported.
+
+The final store path of the derivation is not known at configuration time,
+unlike normal derivations. Meson and CMake are configured to install into a
+prefix, and are then rewritten (using `sed`) to the final output store path.
+This may cause issues with programs that e.g. compress a file containing these
+paths during their build.footnote:[The only one to have known issues is Node.]
+
+== Patches
+In some cases, the code may not handle running in Zilch's environment. In this
+case, it's possible to add extra code to run before running the command of a
+target.
+
+These patches are defined as part of the configuration file, and are arbitrary
+Scheme code:
+
+[,scheme]
+----
+(environment: "import ./sample.drv"
+ patch:
+  (lambda (edge)
+   (if (string=? (build-edge-rule edge) "foo")
+     "rm src/foo; echo 'replacement file' > src/foo"
+     #f))) 
+----
+
+Patches are arbitrary shell scripts, run with GNU Coreutils in `$PATH`. The
+environment variable `$_ZILCH_ROOT` points to the root of the source + build
+file tree, and can be used to copy non-symlinked files into the working
+directory.
+
+== Automatic file elision
+To implement incremental builds efficiently, Zilch automatically slims down the
+part of the source directory that is accessible to each target. If a filename
+ends in `.cc` or `.c` and is directly depended upon as in input file to a
+target, it will be elided, and only the targets that depend on it directly will
+be able to read its contents.
+
+The `build/meson-private` directory is fully elided, except for derivations
+that directly depend upon it. This is to limit the amount of impurities that
+are part of the configuration process. 
+
+=== Dependency tracking
+When a depfile cache is provided, and the output being built supports
+dependency files, any files ending in `.h` and `.hh` that aren't a known
+dependency are elided. However, if the build fails, it will be retried with all
+header files visible to the client.
+
+In some cases, this dependency tracking is imprecise, and causes some header
+files to not be properly visible to targets that need it. In this case, you can
+add an extra entry to the configuration file:
+
+.`lix.scm`
+[,scheme]
+----
+(environment: "(pkgs.lix.override { enableDocumentation = false; }).overrideAttrs (a: { doCheck = false; doInstallCheck = false; })"
+ depfile-path: "lix-depfile.scm"
+ disallow-elide: (lambda (path) (string=? path "config.h")))
+----
+
+This example ensures that `config.h` will be available to all targets being
+built for the derivation.
+
+== Library stubs
+When using Meson, any libraries that are linked will also simultaneously
+generate a stub `.so` file, using `llvm-ifs`. This stub library is valid for
+linking, but contains no code, and is generated deterministically. When linking
+any other libraries or binaries that depend on this library, the stub is
+automatically used, which improves early cutoff between library builds.

lang/ninja/src/build.sld

@@ -1,3 +1,4 @@
+;; Turns a Ninja file into a set of binaries.
 (define-library (zilch lang ninja build)
  (import
   (scheme base) (scheme lazy) (scheme file)
@@ -8,7 +9,10 @@
   (srfi 128) (srfi 146) (srfi 152)
   (zilch lang ninja) (zilch lang ninja config) (zilch lang ninja depfile))
 
- (export process-ninja-file
+ (export
+   process-ninja-file
+
+   <built-edge>
    built-edge-edge built-edge-out-drv
    built-edge-lib-placeholder built-edge-phony-inputs)
 
@@ -17,6 +21,16 @@
   (define patchelf (cdr (assoc "out" (nixpkgs "patchelf"))))
   (define llvm-bintools (cdr (assoc "out" (nixpkgs-eval "llvmPackages_latest.bintools-unwrapped"))))
 
+  ;; Represents a single built `<build-edge>`.
+  ;;
+  ;; - `edge`: The `<build-edge>` that this `<built-edge>` represents.
+  ;; - `out-drv`: A store-path that contains all output files of this edge.
+  ;; - `lib-placeholder`: If set, this built edge represents a library. This
+  ;;   field will then contain a store path that represents the same library as
+  ;;   output in `out-drv`, but with all infnormation other than symbol names
+  ;;   removed. This can be used as a stub while linking other binaries.
+  ;; - `phony-inputs`: a list of strings representing the edges that make up
+  ;;   this edge. Only set if this edge is phony.
   (define-record-type <built-edge>
     (make-built-edge edge out-drv lib-placeholder phony-inputs)
     built-edge?
@@ -315,11 +329,19 @@
     (string-suffix? ".h" path)))
 
   ;; process a ninja file and corresponding vfs, and return two values:
-  ;; - `edge-ref`, a lambda that lets one fetch any build edge;
-  ;; - `defaults`, a list containing the default build edges.
   ;;
-  ;; If the `environment` is a <derivation> or a <store-path>, it is considered
+  ;; - `edge-ref`: A lambda that lets one look up a pair of
-  ;; to be a nixpkgs-style derivation, the same way `nix-shell` works.
+  ;;   `(<zexp> . <built-edge>)` representing the path of an output, and the
+  ;;   edge that produced that output.
+  ;; - `defaults`: A list containing the names of the default build edges.
+  ;;
+  ;; This procedure takes three arguments:
+  ;;
+  ;; - `file`: The parsed `<build-file>`.
+  ;; - `conf`: A `<ninja-build-config>` record describing the details on how to
+  ;;   handle this Ninja file.
+  ;; - `relative-to`: The directory in which the Ninja file was found. Used to
+  ;;   resolve relative references in the Ninja file.
   (define (process-ninja-file file conf relative-to)
     (unless (or (string=? relative-to "") (string-suffix? "/" relative-to)) (set! relative-to (string-append relative-to "/")))
 

lang/ninja/src/config.sld

@@ -1,3 +1,4 @@
+;; Parses the Zilch-specific Ninja configuration.
 (define-library (zilch lang ninja config)
  (import
   (scheme base) (scheme eval)
@@ -6,7 +7,7 @@
   (prefix (only scheme eval) scheme-))
 
  (export
-  ninja-build-config?
+  <ninja-build-config> ninja-build-config?
   ninja-build-config-environment ninja-build-config-environment-drv
   ninja-build-config-root-dir ninja-build-config-patches ninja-build-config-targets
   ninja-build-config-override-source ninja-build-config-depfile ninja-build-config-depfile-path
@@ -17,6 +18,8 @@
   parse-ninja-config)
 
  (begin
+   ;; Represents a parsed Ninja build configuration.
+   ;; See `parse-ninja-config` for the definition of these fields.
    (define-record-type <ninja-build-config>
     (make-ninja-build-config environment environment-drv root-dir patches targets override-source depfile depfile-path disallow-elide)
     ninja-build-config?
@@ -86,7 +89,35 @@
          (set-ninja-build-config-targets! conf (append list-val (ninja-build-config-targets conf))))
         (parse-config-inner conf (cddr data)))
        (else (error (string-append "Unknown directive " (keyword->string (car data)) " parsing Zilch Ninja config")))))))
-
+  
+   ;; Parses a Zilch Ninja configuration file.
+   ;;
+   ;; A Zilch Ninja configuration file is a list containing repeated keys and values.
+   ;;
+   ;; - `env:`/`environment: ...`: The environment to be used when processing
+   ;;   the Ninja file. If the environment is a string, it is assumed to be a
+   ;;   Nix expression resolving to a derivation, and will be evaluated in the
+   ;;   context of Nixpkgs. Otherwise, it is assumed to be a (optionally
+   ;;   `zexp`) alist of environment variables.
+   ;; - `root: dir`: The root directory that contains both the source and
+   ;;   necessary Ninja build files. If string, assumed to be a directory
+   ;;   relative to the working directory; assumed to be a `<vfs>` otherwise.
+   ;; - `override-source: dir`: The source override directory. Only used by
+   ;;   `build-nixpkgs-drv-reproducibly`, ignored otherwise.
+   ;; - `depfile-path: path`: The location of a depfile cache to use, as a
+   ;;   string relative to the current working directory. Only used by
+   ;;   `zilch-cli-ninja`.
+   ;; - `depfile: #<mapping>`: An SRFI146 mapping output file names to a list
+   ;;   of input dependencies. Used to elide inputs where possible.
+   ;; - `patch: proc`: A patch procedure (or a list which evaluates to one).
+   ;;   A patch procedure takes a `<build-edge>, and outputs `#f` or a string
+   ;;   of shell commands to run before evaluating the rule's command.
+   ;; - `disallow-elide: proc`: A procedure that takes the path of a file and
+   ;;   can return `#t` to reject its elision. Used to special-case files that
+   ;;   would normally be elided, but shouldn't, in cases where the depfile is
+   ;;   incorrect.
+   ;; - `target: "foo"`/`targets: '("foo" "bar")`: Build these targets instead
+   ;;   of the default specified in the Ninja file.
    (define (parse-ninja-config config)
     (unless (list? config)
      (error "expected Zilch Ninja config to be a list"))

lang/ninja/src/depfile.sld

@@ -1,3 +1,5 @@
+;; Implements the bare minimum Makefile parser to read `.d` files generated by
+;; GCC, rustc, and more.
 (define-library (zilch lang ninja depfile)
  (import
   (scheme base)
@@ -7,6 +9,8 @@
   parse-depfile)
 
  (begin
+  ;; Parse a depfile. Takes a bytevector representing the depfile, and returns
+  ;; a mapping of output path to input dependencies.
   (define (parse-depfile bytes)
    (define buf (make-bytevector 1024 0))
    (define buf-i 0)

lang/ninja/src/ninja.sld

@@ -1,3 +1,8 @@
+;; Processes Ninja files into Scheme records.
+;;
+;; This library uses a special format to store strings that contain variables.
+;; If a string can contain variables, it may be represented as a list, where
+;; each value of the list is either a string, or a pair `'(varname . "variable_name")`.
 (define-library (zilch lang ninja)
  (import
   (scheme base) (scheme write) (scheme process-context) (scheme lazy)
@@ -12,7 +17,6 @@
  (export
   <build-file> make-build-file build-file?
   build-file-global-variables build-file-default-targets build-file-rules build-file-build-edges
-  build-file-pools
 
   <build-rule> make-build-rule build-rule?
   build-rule-name build-rule-command build-rule-depfile build-rule-deps build-rule-description
@@ -121,14 +125,20 @@
    (define (pr out name value) (unless (eq? value #f) (fprintf out "  ~A = ~A\n" name (render-evalstring value #f))))
 
    ;; Represents a Ninja build file.
+   ;;
+   ;; - `global-variables`: An SRFI146 mapping containing all variables defined
+   ;;   globally.
+   ;; - `default-targets`: A list of target names that should be built if no
+   ;;   specific target is requested.
+   ;; - `rules`: A mapping of rule names to `<build-rule>` records.
+   ;; - `build-edges`: A list of `<build-edge>` records.
    (define-record-type <build-file>
-     (make-build-file global-variables default-targets rules build-edges pools)
+     (make-build-file global-variables default-targets rules build-edges)
      build-file?
      (global-variables build-file-global-variables set-build-file-global-variables!)
      (default-targets build-file-default-targets set-build-file-default-targets!)
      (rules build-file-rules set-build-file-rules!)
-     (build-edges build-file-build-edges set-build-file-build-edges!)
+     (build-edges build-file-build-edges set-build-file-build-edges!))
-     (pools build-file-pools set-build-file-pools!))
 
    (define-record-printer (<build-file> file out)
      (fprintf out "# debug build file\n")
@@ -141,7 +151,8 @@
      (for-each (lambda (edge) (fprintf out "~S" edge)) (build-file-build-edges file)))
 
    ;; Represents a Ninja build rule.
-   ; variables that are ignored for now: dyndep, generator, pool, msvc_deps_prefix
+   ;; Each of the fields of this record represent a variable that can be set
+   ;; in a Ninja build rule.
    (define-record-type <build-rule>
      (make-build-rule name command depfile deps description restat rspfile rspfile-content)
      build-rule?
@@ -165,7 +176,14 @@
      (pr out "rspfile-content" (build-rule-rspfile-content rule))
      (fprintf out "\n"))
 
-   ;; Represents a Ninja build edge (aka one or more files built by this Ninja file)
+   ;; Represents a Ninja build edge (aka a Ninja build statement).
+   ;;
+   ;; - `rule`: The name of the build rule to use for this edge.
+   ;; - `outputs`/`inputs`/`implicit-dependencies`/`order-only-dependencies`/`validations`/`implicit-outputs`:
+   ;;   The corresponding parsed part of the build statement, represented as a
+   ;;   possibly-variable-containing string.
+   ;; - `variables`: A mapping of variable name to a possibly-variable-containing string.
+   ;; - `resolved`: A `<build-rule>` record where each value has its variables already resolved.
    (define-record-type <build-edge>
      (make-build-edge rule outputs inputs implicit-dependencies order-only-dependencies validations implicit-outputs variables resolved)
      build-edge?
@@ -230,7 +248,6 @@
        #f
        (inner-loop 0)))
 
-   ;; Reads a full Ninja file, returning a <build-file> record.
    (define (read-ninja-file-inner strval into read-other-file)
     (define i 0)
     (define (eat-whitespace)
@@ -463,7 +480,15 @@
          (read-toplevel file)))
       file)
     (read-toplevel into))
+
+   ;; Parses a full Ninja file from a string, returning a <build-file> record
+   ;; representing the file.
+   ;;
+   ;; `read-other-file` is a procedure used to read a Ninja file relative to
+   ;; this one, used to implement the `include` keyword. It is expected to take
+   ;; a string representing the path to read, and to return a string containing
+   ;; the full contents of said Ninja file.
    (define (read-ninja-file strval read-other-file)
-     (define out-file (read-ninja-file-inner strval (make-build-file (mapping (make-default-comparator)) '() (mapping (make-default-comparator)) '() (mapping (make-default-comparator))) read-other-file))
+     (define out-file (read-ninja-file-inner strval (make-build-file (mapping (make-default-comparator)) '() (mapping (make-default-comparator)) '()) read-other-file))
     (for-each (lambda (f) (unless (string=? (build-edge-rule f) "phony") (set-build-edge-resolved! f (build-rule-resolve (mapping-ref (build-file-rules out-file) (build-edge-rule f)) f out-file)))) (build-file-build-edges out-file))
     out-file)))

lang/ninja/src/nixpkgs.sld

@@ -1,3 +1,4 @@
+;; Helpers to work with Zilch around Nixpkgs derivations.
 (define-library (zilch lang ninja nixpkgs)
  (import
    (scheme base) (chicken format) (scheme lazy)
@@ -74,7 +75,23 @@
       (set! output (string-copy output 0 32)))
 
     (string-append "/nix/store/" output (string-copy base-placeholder (string-length output)) name (if (string=? output "out") "" (string-append "-" output))))
-
+  
+  ;; Takes a `<ninja-build-config>` representing a Nixpkgs derivation, and
+  ;; preprocesses the derivation such that it can be reconstituted once Zilch
+  ;; has taken over the Ninja build requirements.
+  ;;
+  ;; This procedure is used internally, and shouldn't be relied upon; it 
+  ;; encodes many specific parts that are unlikely to be useful by external
+  ;; parties.
+  ;;
+  ;; Returns 6 values:
+  ;;
+  ;; - The initial Nixpkgs derivation as `<derivation>`
+  ;; - The Nixpkgs derivation after running all phases up to and including
+  ;;   configuration
+  ;; - An alist of output names to placeholder store paths
+  ;; - The `edge-ref`, `defaults`, and `export-depfile` values from calling
+  ;;   `process-ninja-file`
   (define (setup-ninja-environment conf)
     (define initial-drv (ninja-build-config-environment-drv conf))
     (when (store-path? initial-drv)
@@ -125,7 +142,17 @@
     (define-values (edge-ref defaults export-depfile) (process-ninja-file ninja-file conf "build"))
 
     (values initial-drv configured-drv placeholders edge-ref defaults export-depfile))
-
+  
+  ;; Takes a `<ninja-build-config>` representing a Nixpkgs derivation, and
+  ;; build it using Zilch.
+  ;;
+  ;; Returns two values:
+  ;;
+  ;; - An alist of output name to store paths, representing the built
+  ;;   derivation
+  ;; - A procedure that, when called, returns a mapping of output path to
+  ;;   necessary inputs generated from the depfile data. This can be stored in
+  ;;   a file for later rebuilds.
   (define (build-nixpkgs-drv-reproducibly conf)
     (define-values (initial-drv configured-drv placeholders edge-ref defaults export-depfile) (setup-ninja-environment conf))