Clean up documentation

docs/README

@@ -0,0 +1,2 @@
+To generate the documentation, run:
+`docread; npx antora generate antora-playbook.yml`.

docs/docread/docread.scm

@@ -60,7 +60,7 @@
 
     (define out-file (open-output-file out-path))
     ; Print out the comments
-    (fprintf out-file "= `~S`\n" (car contents))
+    (fprintf out-file "= `~S`\n\n" (car contents))
     (for-each (lambda (l) (fprintf out-file "~A\n" l)) lib-comments)
     (fprintf out-file "\n:toc:\n\n")
     (for-each (lambda (i)

docs/modules/ROOT/nav.adoc

@@ -1,5 +1,5 @@
 * xref:index.adoc[]
-* xref:nixexpr.adoc[]
+* xref:go.adoc[]
 * xref:zexp.adoc[]
 
 * ++(zilch)++
@@ -27,6 +27,3 @@
 *** xref:generated:zilch.nix.drv.adoc[++(zilch nix drv)++]
 *** xref:generated:zilch.nix.hash.adoc[++(zilch nix hash)++]
 *** xref:generated:zilch.nix.path.adoc[++(zilch nix path)++]
-
-
-* xref:architecture.adoc[]

docs/modules/ROOT/pages/architecture.adoc

@@ -1,11 +0,0 @@
-= Architecture
-
-`(zilch nix binproto)` contains an implementation of the binary protocol used both
-to talk to the daemon and to build NAR files.
-
-On top of that is `(zilch nix daemon)`, which implements a version (which?) of
-the Nix worker protocol.
-
-`(zilch nix drv)` allows reading and writing .drv objects.
-
-`(zilch nix path)` contains the helpers for building store paths of various types.

docs/modules/ROOT/pages/go.adoc

@@ -0,0 +1,35 @@
+= Go
+
+Currently, the primary binary produced by the `zilch` repository is a tool to
+generate content-addressed derivations from a Go project, called `zilch-cli-go`.
+
+It requires the `ca-derivations` and `impure-derivations` experimental features
+to be enabled on the Nix daemon, and a Nix daemon to be available at the
+default store path. It also requires the daemon to be able to run
+`x86_64-linux` derivations, right now.
+
+Once run, it will use Zilch to build a series of derivations, and output a
+`.drv` for each executable package in the module.
+
+The help page:
+
+[source]
+----
+Usage: zilch-cli-go [OPTION] [PACKAGE...]
+Process the given module (or the current directory, if unspecified) and
+output derivations for each package given on the command line (or all
+executables in the module, if unspecified)
+
+  -h, --help              Print this help message.
+  -b, --build             Build the store paths, rather than show their
+                          derivations.
+  -L, --print-build-logs  Print derivation logs as they come in.
+  -m, --module-dir DIR    The directory to use as root module.
+  -r, --replace DIR       Replace the module specified by the go.mod
+                          with this source directory, rather than using
+                          the upstream module. Can be specified more
+                          than once.
+
+   --debug                Crash on the first error, rather than
+                          continuing with the next package.
+----

docs/modules/ROOT/pages/index.adoc

@@ -15,7 +15,7 @@ Features:
 
 == Current work
 Current effort in Zilch is working on making "incremental", bitesize,
-derivations to work. This is currently being implemented for Go.
+derivations to work. This is xref:go.adoc[currently being implemented for Go].
 
 == Contributing
 

docs/modules/ROOT/pages/nixexpr.adoc

@@ -1,41 +0,0 @@
-= Nix expression support
-
-When `(nix reader)` is imported, it is possible to execute Nix code inline with
-Scheme code. Simply wrap your Nix code in curly brackets:
-
-[,scheme]
-----
-(write
-  (string-append
-    "Hello, Nix version"
-    { builtins.nixVersion }))
-----
-
-The following values can be translated:
-
-|===
-| Nix            | Scheme  | Notes
-
-| string         | string | (Loses string context.)
-| integer        | number |
-| float          | number |
-| boolean        | boolean | 
-| lambda         | procedure | (with single argument)
-| list           | vector or list | Depends on the status of `\*translate-list-as-vector*`
-| attrset        | alist |
-| builtin        | procedure |
-| external value | symbol, other unknown objects |
-|===
-
-If a value in Nix is preceded with a comma, it is unquoted, similar to
-`(unquote)` in a Scheme quasiquotation. If prefixed with a single quote, it is
-`(quote)`-d.
-
-[,scheme]
-----
-(define
-  (test-append foo)
-  (string-append "Hello, " foo))
-(write
-  { ,test-append "world!") })
-----

docs/modules/ROOT/pages/zexp.adoc

@@ -1,11 +1,15 @@
 = zexps
 
 zexps, similar to g-expressions in Guix, are a way to generate
-S-expressions that are taggged with store paths. But that's where
+S-expressions that are tagged with store paths. The syntax in Zilch is inspired
-the similarity ends.
+by it, but has been developed separately.
 
+A zexp is used similarly to a ``quasiquote``d value in Scheme, but has an extra type
+of unquoting, called `zexp-unquote`, which unquotes a value such as a `++<store-path>++`
+or another `zexp`.
 
-To create a zexp, you can use either the full syntax, or the reader macro:
+Each `zexp` keeps track of the derivation outputs and store files it depends on,
+similarly to how string context works in Nix:
 
 [,scheme]
 ----
@@ -14,15 +18,14 @@ To create a zexp, you can use either the full syntax, or the reader macro:
 (zexp (foo bar (zexp-unquote baz)))
 ----
 
-`(zexp-unquote VAL)` returns the value that the zexp (or any compatible record)
+When a `zexp-unquote` (or its reader syntax, `#$`) is encountered, the value
-contains, while gathering the `zexp-unquote`d values used.
+contained in the `zexp` (or compatible object) is used as-is, with no
-
+evaluation. It is possible to mix and match `zexp-unquote` with `unquote`,
-Like quasiquotation, zexps can use `unquote`, including ``zexp-unquote``d values
+allowing building e.g. dynamic strings from zexps. `zexp-unquote` is always
-inside the ``unquote``d code. ``unquote``d code is evaluated when the `zexp` is evaluated.
+evaluated before `unquote` is.
 
 [,scheme]
 ----
-(define world #~,(begin (write "hello") "world"))
+(define world ...)
-(define hello #~("hello" ,(string-append "very " "cute") #$world))
+(define hello #~("hello" ,(string-append "very " "cute" #$world)))
-; When hello is used as zexp, it will also write "hello" to the output port.
 ----

lang/go/src/core.sld

@@ -1,3 +1,18 @@
+;; Defines the baseline definitions needed to compile Go code using Zilch.
+;;
+;; To make incremental builds work, this library uses up to four distinct
+;; outputs:
+;;
+;; - `api` is any `.a` file needed for compiling any other Go code that depends
+;;   on said package, and is usually an archive containing a file named
+;;   `__.PKGDEF`.
+;; - `code` is a `.a` file containing the actual binary code for the target
+;;   arch, and is used only during linking.
+;; - `asmhdr` is a header file generated by the Go code, and used during
+;;   assembly compilation only.
+;; - `symabi` is a text file contaning the functions defined by assembly, and
+;;   their ABI, which is used while compiling the Go code that contains the
+;;   stubs for any assembly code.
 (define-library (zilch lang go core)
  (import
   (scheme base) (scheme write) (scheme process-context) (scheme lazy)
@@ -22,15 +37,19 @@
   
 
  (begin
+   ;; The architecture to target the Go code at.
+   ;; Equivalent to `GOARCH`.
    (define %goarch (make-parameter "amd64"))
 
-   ; Import the existing Go from nixpkgs.
+   ;; The Go toolchain to use to compile this all.
    (define go-toolchain (cdr (assoc "out" (nixpkgs "go_1_23"))))
   
-   ;; Builds an importcfg file, containing an alist of packages to .a files,
+   ;; Builds an importcfg file. This file describes the mapping of both
-   ;; and an alist of package names to actual package names.
+   ;; packages to their api, and the mapping of package name as used in `import`
-   ;; `++packagefiles++` is a alist of package name to .a file (api type),
+   ;; to the actual package names (e.g in case of `replace`.)
-   ;; `++importmap++` is an alist of package name to actual package name (used in cases of e.g. replace)
+   ;;
+   ;; - `++packagefiles++` is a alist of package name to .a file.
+   ;; - `++importmap++` is an alist of package name to actual package name.
    (define (build-importcfg packagefiles importmap)
      (call-with-port (open-output-string)
        (lambda (outstr)
@@ -53,8 +72,11 @@
            importmap)
          (get-output-string outstr))))
 
-   ;; `++patterns++` is an alist of the pattern used to match files (e.g. `++foo/++`, or `++a.*++`) to a list of filenames.
+   ;; Builds an embedcfg file, which maps from the pattern used in `go:embed`
-   ;; `++files++` is an alist of file name to actual path.
+   ;; to a list of files, as well as a filename to on-disk file mapping.
+   ;;
+   ;; - `++patterns++` is an alist of the pattern used to match files (e.g. `++foo/++`, or `++a.*++`) to a list of filenames.
+   ;; - `++files++` is an alist of file name to actual path.
    (define (build-embedcfg patterns files)
      (call-with-port (open-output-string)
        (lambda (outstr)
@@ -65,7 +87,7 @@
            outstr)
          (get-output-string outstr))))
 
-   ; Clean up the package name to use in drv names.
+   ;; Clean up the package name to use in drv names.
    (define (rewrite-package-name name)
      (set! name (string-copy name))
      (do ((x 0 (+ x 1)))
@@ -75,10 +97,10 @@
          (when (char=? (string-ref name x) #\space) (string-set! name x #\_))
          (when (char=? (string-ref name x) #\]) (string-set! name x #\_))))
 
-   ;; An empty go_asm.h file used to generate symabis.
+   ;; An empty go_asm.h file used when generating symabis.
    (define empty-asmhdr (zdir `(("go_asm.h" . ,(zfile "")))))
   
-   ;; Environment to append to the build environment for Go.
+   ;; The environment to append to the build environment for Go.
    (define (env-for-goarch)
      `(("GOARCH" . ,(%goarch))))
   
@@ -89,16 +111,19 @@
        "-D" ,(string-append "GOARCH_" (%goarch))
        ,@(if (string=? (%goarch) "amd64") '("-D" "GOAMD64_v1") '())))
 
-   ;; Returns an alist of three store paths; `++api++` containing the compiler's output,
+   ;; Returns an alist of three store paths.
-   ;; `++code++` containing the linkobj, and `++asmhdr++` containing the headers needed for assembly
+   ;;
-   ;; code to properly use Go functions and variables.
+   ;; - `++api++` containing the compiler's output, used when compiling
+   ;; - `++code++` contains the compiled code, used during linking only.
+   ;; - `++asmhdr++` contains the headers needed for any assembly code inside this package.
    (define (go-compile std package-name importcfg symabis embeds files)
      (define args
        #~(
           ,@(if std '("-std") '())
           #$@(if symabis `("-symabis" ,#$symabis) '())
           #$@(if embeds `("-embedcfg" ,#$embeds) '())
-          "-buildid" "zilch go-compile" ; this goes into both code and __.PKGDEF, so can't be a reference to the code output, sadly
+          ; this goes into both code and __.PKGDEF, so can't be a reference to the code output, sadly
+          "-buildid" "zilch go-compile"
           "-p" #$package-name
           "-o" ,(make-placeholder "api")
           "-linkobj" ,(make-placeholder "code")
@@ -114,7 +139,7 @@
        #~(,(string-append #$go-toolchain "/bin/go") "tool" "compile" . #$args)
        (env-for-goarch) '("api" "code" "asmhdr")))
   
-   ;; Returns a store path containing the symabi for the files provided.
+   ;; Returns a store path containing the symabi for the assembly files provided.
    (define (go-generate-symabi package-name include-path files)
      (define args
        #~(
@@ -133,7 +158,9 @@
              #~(,(string-append #$go-toolchain "/bin/go") "tool" "asm" . #$args)
              (env-for-goarch) '("symabi"))))
 
-   ;; Returns a store path containing the `++code++` of the provided assembly files.
+   ;; Returns a store path containing the `++code++` of the provided assembly
+   ;; files. Assembly files have no `api`, and cannot be directly interacted
+   ;; with from other packages.
    (define (go-compile-assembly package-name include-path include-path2 files)
      (define args
        #~(

lang/go/src/fetch.sld

@@ -1,26 +1,30 @@
+;; Helpers for fetching files from the Go module proxy, slightly impurely.
 (define-library (zilch lang go fetch)
   (import
     (scheme base) (scheme write) (scheme read) (scheme file) (scheme char)
     (zilch magic) (zilch zexpr)
     (zilch nixpkgs)
-    (chicken format))
+    (chicken process-context) (chicken format) (chicken file))
  
   (export
     fetch-with-known-url rewrite-module-name-for-url)
 
   (begin
-    (define fetch-cache (call-with-input-file "/home/.zilchcache" read))
+    (define fetch-cache-file (string-append (get-environment-variable "HOME") "/.cache/zilch-fetch.scm"))
+    (define fetch-cache (if (file-exists? fetch-cache-file) (call-with-input-file fetch-cache-file read) '()))
     (define (fetch-with-known-url name url)
       (define cache-entry (assoc url fetch-cache))
       (define hash (if cache-entry
                      (cdr cache-entry)
-                     (begin (printf "..fetching ~S ~S\n" name url)
+                     (begin (fprintf (current-error-port) "(pre)fetching ~S ~S\n" name url)
                       (nix-prefetch-url name url #f))))
       (unless cache-entry
         (set! fetch-cache (cons (cons url hash) fetch-cache))
-        (call-with-output-file "/home/.zilchcache" (lambda (out) (write fetch-cache out))))
+        (call-with-output-file fetch-cache-file (lambda (out) (write fetch-cache out))))
-      (store-path-for-fod name "builtin" '("builtin:fetchurl") `(("url" . ,url) ("outputHashMode" . "flat")) "sha256" hash #f))
+      (store-path-for-fod name "builtin" '("builtin:fetchurl") `(("url" . ,url)) "sha256" hash #f))
 
+    ;; Rewrites the module name to prefix all uppercase letters with an
+    ;; exclamation mark instead, as required by the various Go webservices.
     (define (rewrite-module-name-for-url name)
       (define out "")
       (string-for-each

lang/go/src/mod.sld

@@ -1,4 +1,4 @@
-;; Processes go modules.
+;; Processes go module files.
 (define-library (zilch lang go mod)
  (import
   (scheme base) (scheme write) (scheme read) (scheme file) (scheme process-context) (scheme lazy) (scheme case-lambda)
@@ -27,7 +27,8 @@
        (cons (car lis) (filter proc (cdr lis))))
       (else (filter proc (cdr lis)))))
 
-  ;; Read a go.mod file. This returns a processed json object, like `go mod edit -json` outputs by default.
+  ;; Read a go.mod file. This returns a processed json object, like
+  ;; `go mod edit -json` outputs by default.
   (define (read-go-mod mod-file)
     (call-with-port
       ;; TODO(puck): don't use /bin/sh here.
@@ -38,7 +39,8 @@
     (vector-any (lambda (v) (and (string=? (car v) key) (cdr v))) vec))
   
   ;; Reads in the module rooted by the vfs, and resolves its requirements list.
-  ;; This returns two values: the name of the root module, and a mapping of module name to a pair of its version and the vfs.
+  ;; This returns two values: the name of the root module, and a mapping of
+  ;; module name to a pair of its version and the vfs.
   (define (collect-requirements-for-module vfs replaces)
     (define sum-lines '())
     (define (parse-sumfile go-sum)
@@ -135,7 +137,8 @@
     (tick)
     (values root-path-name collected-requires))
   
-  ;; Processes a mapping of module name to a pair of version and vfs, and returns a procedure that takes a package name and returns its go-package.
+  ;; Processes a mapping of module name to a pair of version and vfs, and
+  ;; returns a procedure that takes a package name and returns its go-package.
   (define (collect-packages-from-requires collected-requires)
     (define (process-package vfs last-part full-path pairs headers)
       (define name (cdr (assoc "name" pairs)))

lang/go/src/package.sld

@@ -17,6 +17,9 @@
           (map go-stdlib-ref '("encoding/json" "fmt" "go/build" "io" "io/fs" "os" "path" "path/filepath" "sort" "strings" "time"))
           (list (cons "main.go" (zfile (foreign-value "parser_source" nonnull-c-string)))))))
 
+    ;; Uses IFD to find each Go package defined inside this virtual filesystem,
+    ;; returning a vector containing pairs, mapping each directory to the
+    ;; package defined within.
     (define (find-packages-inside-vfs vfs)
       (define input
         #~,(call-with-port

lang/go/src/stdlib.sld

@@ -152,7 +152,7 @@
     ; Use `++delay++` to resolve the DAG lazily on use.
     (define stdlib-data (map (lambda (v) (list (cdr (assoc "ImportPath" v)) v (delay (make-stdlib-inner v)))) stdlib-objects))
   
-    ;; Wrapper that forces evaluation of the promise fetching from a Go stdlib entry.
+    ;; Finds any package contained within Go's standard library.
     (define (go-stdlib-ref name)
       (define entry (assoc name stdlib-data))
       (unless entry (error (string-append "Could not find package " name " in stdlib")))

lang/go/src/sum.sld

@@ -1,3 +1,4 @@
+;; Parses `go.sum` files.
 (define-library (zilch lang go sum)
   (import
     (scheme base) (scheme write) (scheme read) (scheme file) (scheme process-context) (scheme lazy) (scheme case-lambda)
@@ -14,6 +15,7 @@
     parse-go-sum-line parse-go-sum-file go-sum-line? go-sum-module go-sum-version go-sum-path go-sum-hash)
 
   (begin
+    ;; Contains the values from a single line from a `go.sum` file.
     (define-record-type <go-sum-line>
       (make-go-sum-line module version path hash)
       go-sum-line?
@@ -35,6 +37,7 @@
         ((char=? (string-ref str index) char) index)
         (else (string-find str (+ index 1) char))))
 
+    ;; Parses a `go.sum` line, and returns a `<go-sum-line>`.
     (define (parse-go-sum-line line)
       (define version-space-index (string-find line 0 #\space))
       (unless version-space-index (error "go.sum line contains no space characters"))
@@ -52,6 +55,7 @@
         (set! version (string-copy version 0 path-index)))
       (make-go-sum-line module-path version path (base64->bytevector (string-copy hash 3))))
 
+    ;; Parses all the `go.sum` lines from `port`.
     (define (parse-go-sum-file port)
       (do ((parsed '())
            (line "" (read-line port)))

lang/go/src/version.sld

@@ -1,9 +1,13 @@
+;; Procedures to deal with Go's semantic versions.
 (define-library (zilch lang go version)
   (import
     (scheme base) (srfi 152))
 
   (export parse-version version<?)
   (begin
+
+    ;; Returns a list containing five values parsed from the version string:
+    ;; `(major minor patch prerelease build)`
     (define (parse-version vstr)
       (unless (char=? (string-ref vstr 0) #\v) (error "not a valid version" vstr))
       (define first-period (string-index vstr (lambda (ch) (char=? ch #\.)) 1))
@@ -17,6 +21,7 @@
       (define build (and build-dash (string-copy vstr (+ build-dash 1))))
       (list major minor patch prerelease build))
 
+    ;; Returns `#t` if `left` is a smaller version than `right`.
     (define (version<? left right)
       (set! left (parse-version left))
       (set! right (parse-version right))
@@ -43,4 +48,3 @@
              (and (list-ref left 3) (not (list-ref right 3)))
              ; or both have a prerelease and it's comparable
              (and (list-ref left 3) (string<? (list-ref left 3) (list-ref right 3)))))))))))))
-

lang/go/src/vfs.sld

@@ -1,3 +1,10 @@
+;; Contains procedures to work with a very simple virtual filesystem,
+;; abstracting between local and in-store store paths.
+;;
+;; A VFS is defined as a vector containing pairs consisting of the directory's
+;; name, with a forward slash prefixed and postfixed (e.g. `/` or `/foo/bar/`).
+;; Each pair then contains another vector, mapping filename to any value that
+;; can be used as a z-expression (e.g. `store-path-for-fod` or `zfile`).
 (define-library (zilch lang go vfs)
   (import
     (scheme base) (scheme write) (scheme read) (scheme file) (scheme process-context) (scheme lazy) (scheme case-lambda)
@@ -42,6 +49,8 @@
                            (if (char-upper-case? ch) (set! out (string-append out (string #\! (char-downcase ch)))) (set! out (string-append out (string ch))))) name)
       out)
    
+    ;; Takes a VFS and writes its directory structure into the Nix store,
+    ;; returning a zdir describing the root directory.
     (define (vfs-to-store vfs)
       (define dirmap (mapping (make-default-comparator)))
       (vector-for-each
@@ -62,13 +71,17 @@
                 (map (lambda (k) (cons (car k) (translate-dir (cdr k)))) dirs))))
       (translate-dir "/"))
 
-
+    ;; Reads a dirhash from a `go.sum` line. This prefetches the module from
+    ;; the go module proxy, and then generates the dirhash without unpacking
+    ;; said module file.
     (define (fetch-dirhash-for-sum sum-line)
       (when (go-sum-path sum-line) (error "go.sum line is invalid for fetch-dirhash-for-sum" sum-line))
       (define url (string-append "https://proxy.golang.org/" (rewrite-name (go-sum-module sum-line)) "/@v/" (go-sum-version sum-line) ".zip"))
       (define known (fetch-with-known-url "module.zip" url))
       (store-path-for-fod "module" "x86_64-linux" #~(#$dirhash-generator) #~(("src" . #$known)) "sha256" (go-sum-hash sum-line) #f))
 
+    ;; Generates a full VFS structure from a module as described by a `go.sum`
+    ;; line.
     (define (vfs-from-dirhash sum-line)
       (define dirhash-file (fetch-dirhash-for-sum sum-line))
       (define url (string-append "https://proxy.golang.org/" (rewrite-name (go-sum-module sum-line)) "/@v/" (go-sum-version sum-line) ".zip"))
@@ -106,6 +119,7 @@
         lines)
       (list->vector (map (lambda (pair) (cons (car pair) (list->vector (cdr pair)))) dirs)))
 
+    ;; Generates a full VFS structure from an on-disk directory.
     (define (vfs-from-directory osdir)
       (define iter-dir #f)
       (define output '())
@@ -125,6 +139,8 @@
       (iter-dir "")
       (list->vector output))
 
+    ;; Calls `filter` for each file in the virtual filesystem, replacing its
+    ;; contents with an empty file if `filter` returns false.
     (define (filter-vfs vfs filter)
       (vector-map
         (lambda (dir)
@@ -137,13 +153,15 @@
               (cdr dir))))
         vfs))
 
-    ; List extracted from go src/go/build/build.go.
+    ;; List extracted from go src/go/build/build.go.
     (define good-extensions '("go" "c" "cc" "cpp" "cxx" "m" "h" "hh" "hpp" "hxx" "f" "F" "for" "f90" "s" "S" "sx" "swig" "swigcxx" "syso"))
     (define (extract-extension name i)
       (cond ((char=? (string-ref name i) #\.) (string-copy name (+ i 1)))
             ((= i 0) #f)
             (else (extract-extension name (- i 1)))))
 
+    ;; Returns a VFS, filtered down to only contain the contents of files that
+    ;; will be read during the processing of Go packages.
     (define (filter-vfs-for-package-reading vfs)
       (filter-vfs vfs
         (lambda (dir fname)