(zilch lang go): document

Change-Id: I6a6a6964558b4fe2f96d78120b2e899f91d48c22

docs/modules/ROOT/nav.adoc

@@ -5,6 +5,11 @@
 ** xref:core/zexp.adoc[]
 ** xref:core/vfs.adoc[]
 ** xref:core/zilch-as-lib.adoc[]
+* Language support
+** Go
+*** xref:go/usage.adoc[]
+*** xref:go/library.adoc[]
+*** xref:go/man.adoc[Man page]
 
 * Code reference
 include::generated:partial$nav.adoc[]

docs/modules/ROOT/pages/go/library.adoc

@@ -0,0 +1,210 @@
+= Writing Go packages directly in Zilch
+:page-pagination:
+
+It's possible to use Zilch as generic build system for Go. This page
+serves to document that, as well as the internals of the Go build
+library in Zilch.
+
+== Go packages in Zilch
+
+Each Go package is represented in Zilch by a xref:generated:zilch.lang.go.adoc#++_go-package_++[`<go-package>`],
+which represents the compiled API and ABI as separate store paths, plus the
+name of the package, and its dependencies.
+
+The simplest possible Go package consists of a single file:
+
+[,scheme,line-comment=;]
+----
+(define example
+ (go-package-compile
+  "example.com/package" ; <1>
+  '() ; <2>
+  `(("example.go" . ,(zfile "package example"))))) ; <3>
+;; #<go-package example.com/package
+;;   api: #<store path /1ahdindygxf34gjhdj9l6h43i6cwsky0v02ai75g35bp5n1i80gp
+;;          (ca~ /nix/store/f4bj92n5yfhp5f29jmacydghsvkaj7fq-example.com_package-src.drv!api)>
+;;   code: #<store path /0ln85mblval4j4cpjgw65f4phrbiagzh1hd5pryjlnnknbq75agj
+;;          (ca~ /nix/store/8dqr20s72i4w4llp73imm9bkcwmx9via-go-example.com_package-code.drv!code)>
+;;   deps: ()>
+----
+<1> Name of the package
+<2> List of dependency packages (in this case, none)
+<3> Mapping of filename to in-store files
+
+To create an executable, though, a bit more work is necessary, which is
+abstracted by the Go build system by default. The "main" package, which
+contains the `func main()`, always has the package name "main". However, this
+is undesirable for the name in panics. For this case, the extended
+`go-package-compile` procedure allows setting both:
+
+[,scheme,line-comment=;]
+----
+(define binary
+ (go-package-compile
+  "main" ; <1>
+  "example.com/binary" ; <2>
+  (list example) ; <3>
+  `(("main.go" . ,(zfile "package main\nfunc main() { }"))) ; <4>
+  '() '() ; <5>
+  '() '())) ; <6>
+;; #<go-package
+;;   main (example.com/binary)
+;;   api: #<store path /0p33flbfjc0s9ykgmcyzqbr3dhsq0344zswhl6xjvqdbin9ysqch
+;;          (ca~ /nix/store/hg5j28f63b4czfxm4cg76rjbh4bx6ivy-main-src.drv!api)>
+;;   code: #<store path /15is1garmlclnb06qj9ymxdi8xpsgakrx97qdb2jmx0562wn7kgx
+;;          (ca~ /nix/store/z9w0rccki2rhjrgxr0y063qijz64m6p8-go-example.com_binary-code.drv!code)>
+;;   deps: ("example.com/package")>
+----
+<1> The package name, as provided to the compiler (must be `main` for binaries)
+<2> The package name as shown in stacktraces
+<3> Direct dependencies for this package
+<4> The source file for this package
+<5> Assembly-related files
+<6> ``go:embed``-related files
+
+To use this binary, it still needs to be linked. This can be done with
+`go-package-link`.
+
+[IMPORTANT]
+====
+If you're linking a Go binary, make sure to add the `runtime` package to your
+dependencies, either directly or indirectly. Otherwise, you will end up with an
+error like so:
+
+[,scheme,line-comment=;]
+----
+(store-path-realised (go-package-link binary))
+;> [..building "/nix/store/jr17baky22c0zzpjfi97cw16k5wmqqc3-example.com_binary.drv"]
+;> [0/1 builds, 1 running]
+;> loadinternal: cannot find runtime
+;> panic: could not look up runtime.mapinitnoop
+----
+
+This is caused by Zilch not adding any standard library code by default, while
+the Go compiler depends on structures in the standard library to provide runtime
+support for the garbage collector and goroutines.
+====
+
+== Using the Go standard library
+The Go standard library, unlike most other languages, is shipped as source code.
+When you compile Go code, the compiler will also compile the parts of the
+standard library you use. Zilch does the same.
+
+`(zilch lang go stdlib)` contains a single procedure, `go-stdlib-ref`, which
+handles this for you:
+
+[,scheme,line-comment=;]
+----
+(go-stdlib-ref "fmt")
+;; #<go-package fmt
+;;   api: #<store path /1al016fvsnknhpzdzbcf7kjxyzbf3fj8sbm40p8h32x3kr0aswbv
+;;          (ca~ /nix/store/vpvpmx2d43ix6m142jxxg0z1kqddypzl-fmt-src.drv!api)>
+;;   code: #<store path /0sssb21sd2d398fv4vwlykf0shnlyr31kf372k65djqrgxrz5674
+;;           (ca~ /nix/store/pi9nraymjdn12hg7r18h13js56w0s1b6-go-fmt-code.drv!code)>
+;;   deps: ("errors" "internal/fmtsort" "io" "math" "os" "reflect"
+;;          "slices" "strconv" "sync" "unicode/utf8")>
+
+(define hello-world
+ (go-package-compile
+  "main"
+  "example.com/binary"
+  (list example (go-stdlib-ref "fmt"))
+  `(("main.go"
+      . ,(zfile "package main
+                 import \"fmt\"
+                 func main() {
+                   fmt.Printf(\"Hello, world!\\n\")
+                 }")))
+  '() '()
+  '() '()))
+
+(define linked (go-package-link hello-world))
+;; #<store path /1csyxx5m27a4819g5m8bnm88gfgcspxisg19rv2kg6b7cv7wn066
+;;   (ca~ /nix/store/jjqhwl7cq2crhdjdsrkby4ng584n3pc1-example.com_binary.drv!out)>
+
+(store-path-realised linked)
+;> [..building "/nix/store/5bgrfjgnvzr8wqksl5j23xr5wm7rahnw-zilchfile.drv"]
+;> …
+;> [0/2 builds, 0 running]
+;> [..building "/nix/store/jdk0wbnrz16sbn41r0y6f82hxl93q87r-example.com_binary.drv"]
+;; "/nix/store/p71p43d3cv48rgi4yh9dvlzssgfkb5vf-example.com_binary"
+----
+
+The binary that is output by this code is incrementally built, one package at a
+time, including the standard library.
+
+[#vfs]
+== Creating virtual filesystems for Go modules
+Go's `go.sum` files contain enough information to recover the full file
+structure of the module, through a hash format called `dirhash`. Zilch
+implements this, and uses it to guide its VFS generating. Once you have a
+`go.sum` line, e.g. through `parse-go-sum-line`, passing it to
+`vfs-from-dirhash` will have it generate a VFS from a dirhash, plus the Go
+module proxy to fetch the zip:
+
+[,scheme,line-comment=;]
+----
+(define sum-line (parse-go-sum-line "golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw="))
+;; #<go-sum golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=>
+
+(define x-net-vfs (vfs-from-dirhash sum-line))
+;> (pre)fetching "module.zip" "https://proxy.golang.org/golang.org/x/net/@v/v0.41.0.zip"
+;> …
+;; #<zilch.vfs#<vfs>>
+
+(store-path-realised
+ (vfs-to-store x-net-vfs))
+;> [..building "/nix/store/vvs2xsgjpab4gwm530dvzcfzfcs5ak4m-zilchfile.drv"]
+;> …
+;; "/nix/store/r1p1y6bnmddmcrl2avsygp659d8iiih1-zilchfile/-"
+----
+
+== Dynamically generating ``go-package``s from packages
+
+Zilch supports generating `go-package` records dynamically, from `go.mod` files
+and `go.sum` files recursively. To do this, get a VFS for the primary module
+you want to work with, and any module you want to resolve automatically (Zilch
+will automatically crawl `go.sum` files to resolve dependencies, where needed.)
+
+[,scheme,line-comment=;]
+----
+(define-values (root-module-path requirements)
+ (collect-requirements-for-module x-net-vfs '()))
+
+;> Collecting required modules
+;> [..building "/nix/store/s9fmg08x56px39hjkwlba2pra5g1abgr-go.mod.json.drv"]
+;> - found "golang.org/x/net" (requires 4 modules)
+;> - found "golang.org/x/crypto" (requires 4 modules)
+;> …
+;; root-module-path -> "golang.org/x/net"
+;; requirements
+;;  -> #<mapping
+;;       "golang.org/x/net" -> (#f . #<zilch.vfs#<vfs>>)
+;;       "github.com/google/go-cmp" -> ("v0.6.0" . #<zilch.vfs#<vfs>>)
+;;       "golang.org/x/crypto" -> ("v0.39.0" . #<zilch.vfs#<vfs>>)
+;;        …
+;;       "golang.org/x/mod" -> ("v0.25.0" . #<zilch.vfs#<vfs>>)>
+----
+
+The resulting values are a comprehensive set of modules mapped to their final
+versions and the VFS of that module. This information can then be used to
+create a procedure that returns the `go-package` for any package in this set of
+modules, similar to how `go-stdlib-ref` works:
+
+[,scheme,line-comment=;]
+----
+(define x-net-ref
+ (collect-packages-from-requires requirements))
+
+(x-net-ref "golang.org/x/net/idna")
+;; #<go-package idna (golang.org/x/net/idna)
+;;   api: #<store path /1w32lfajzbs83ibvgnn2cx4zk33nmg3xb20dhra1b89zyg3xin4i
+;;          (ca~ /nix/store/az0g6zjc33mg3kfacy7gnfhkbkps8m2y-golang.org_x_net_idna-src.drv!api)>
+;;   code: #<store path /1r40gj99ycm404ak3f2rkhl4qhy19hx1dkhjpmp0rbbsgy2f4wj7
+;;           (ca~ /nix/store/b93f07hkv2yxf80pdwbra2r760yxkh3r-go-golang.org_x_net_idna-code.drv!code)>
+;;   deps: ("fmt" "bidirule" "bidi" "norm" "math" "strings" "unicode/utf8")>
+
+(x-net-ref "golang.org/x/text/runes")
+;; #<go-package runes (golang.org/x/text/runes) …> <1>
+----
+<1> Even packages defined in a dependency can be looked up using this method.

docs/modules/ROOT/pages/go/man.adoc

@@ -0,0 +1,58 @@
+= zilch-cli-go(1)
+Puck Meerburg
+v0.0.1
+:doctype: manpage
+:manmanual: ZILCH-CLI-GO
+:mansource: ZILCH
+:page-pagination: prev
+
+== Name
+
+zilch-cli-go - builds a Go module using Zilch
+
+== Synopsis
+
+*zilch-cli-go* [_OPTION_]... _PACKAGE_...
+
+== Description
+
+This command uses Zilch to build one or more Go packages entirely
+inside Nix, using content-addressed derivations.
+
+A Go module is recognized by its `go.mod`, which contains information
+about the dependencies of any Go package. This versioning info is used
+identically to a normal `go build` call.
+
+== Options
+
+*-h*::
+*--help*::
+  Print a help message
+
+*-j* _COUNT_::
+*--max-jobs* _COUNT_::
+  The maximum amount of builds to run. Defaults to the amount of cores.
+
+*-v*::
+*--verbose*::
+  Increase the verbosity configured in the Nix daemon. Can be specified
+  multiple times.
+
+*-L*::
+*--print-build-logs*::
+  Print derivation logs as they come in.
+
+*-m* _DIR_::
+*--module-dir* _DIR_::
+  The directory to use as root module. All packages to be built must
+  come from this module.
+
+*-r* _DIR_::
+*--replace* _DIR_::
+  Replace a module from the `go.mod` of the root module with
+  this directory, based on the name of the replaced module's `go.mod`.
+  Can be specified multiple times.
+
+*--debug*::
+  Crash on the first error, rather than continuing to build the next
+  package.

docs/modules/ROOT/pages/go/usage.adoc

@@ -0,0 +1,49 @@
+= Usage
+:page-pagination: next
+
+Zilch supports compiling any Go code incrementally using `zilch-cli-go`, with
+no special configuration needed on the part of the Go module being built:
+
+[,console]
+----
+$ git clone https://github.com/tailscale/tailscale
+
+$ zilch-cli-go --module-dir tailscale/ <1>
+tailscale.com/client/tailscale/example/servetls	/nix/store/727ci6sssga0pbi4aqb08vnhfvmh9nl0-tailscale.com_client_tailscale_example_servetls
+tailscale.com/cmd/addlicense	/nix/store/ckc5bx4s0c29sancd05asvhz0dy43xj7-tailscale.com_cmd_addlicense
+…
+tailscale.com/cmd/xdpderper	/nix/store/8wn9w91mjipw1dzks4p3kcmkwcn18jp7-tailscale.com_cmd_xdpderper
+
+$ zilch-cli-go --module-dir tailscale/ \
+>  tailscale.com/cmd/tailscaled <2>
+tailscale.com/cmd/tailscaled	/nix/store/hqrm0f8sd8sx54am921na25w8za67p3m-tailscale.com_cmd_tailscaled
+----
+<1> Building all binary packages in a module
+<2> Building a specifically targeted package
+
+Right now, the daemon has to be able to build `x86_64-linux` derivations,
+and `zilch-cli-go` will only output statically linked amd64 binaries as well.
+
+
+While running, all processing of the Go module, its dependencies, etc, are done
+inside of Nix. Once all dependencies have been resolved, a series of Nix
+derivations will be used to then build the requested packages. If any source
+file's changes do not affect the way dependent packages use it, those packages
+will not need rebuilding; only the final result will have to be re-linked.
+
+== Replacing dependencies
+As part of Zilch, it's also possible to quickly build a module with one of its
+(transitive) dependencies replaced. This keeps the same guarantees as before:
+Any changes made that do not involve the output changing will only need
+relinking of the resulting binary. This is also very simple:
+
+[,console]
+----
+$ git clone https://go.googlesource.com/net x-net   # golang.org/x/net
+$ zilch-cli-go --module-dir tailscale --replace x-net/ tailscale.com/cmd/tailscaled
+…
+tailscale.com/cmd/tailscaled	/nix/store/hqrm0f8sd8sx54am921na25w8za67p3m-tailscale.com_cmd_tailscaled
+----
+
+After editing a file any of the replaced dependencies, if this package is used
+in the final build, Zilch will apply early-cutoff where possible.

lang/go/src/core.sld

@@ -1,6 +1,6 @@
-;; Defines the baseline definitions needed to compile Go code using Zilch.
+;; Defines the functions needed to compile Go code using Zilch.
 ;;
-;; To make incremental builds work, this library uses up to four distinct
+;; To make incremental builds possible, this library uses up to four distinct
 ;; outputs:
 ;;
 ;; - `api` is any `.a` file needed for compiling any other Go code that depends
@@ -21,7 +21,7 @@
   json
   (chicken foreign)
   (srfi 4))
-    
+
  (export
    build-importcfg
    build-embedcfg
@@ -34,7 +34,7 @@
    go-compile-assembly
 
    go-toolchain)
-  
+
 
  (begin
    ;; The architecture to target the Go code at.
@@ -42,14 +42,15 @@
    (define %goarch (make-parameter "amd64"))
 
    ;; The Go toolchain to use to compile this all.
-   (define go-toolchain (cdr (assoc "out" (nixpkgs "go_1_23"))))
+   (define go-toolchain (cdr (assoc "out" (nixpkgs "go_latest"))))
-  
+
    ;; Builds an importcfg file. This file describes the mapping of both
    ;; packages to their api, and the mapping of package name as used in `import`
    ;; to the actual package names (e.g in case of `replace`.)
    ;;
-   ;; - `++packagefiles++` is a alist of package name to .a file.
+   ;; - `++packagefiles++` is an alist mapping a package name to .a file.
-   ;; - `++importmap++` is an alist of package name to actual package name.
+   ;; - `++importmap++` is an alist mapping a package name to actual package name.
+   ;;   This is primarily used by Go's standard library to vendor external packages.
    (define (build-importcfg packagefiles importmap)
      (call-with-port (open-output-string)
        (lambda (outstr)
@@ -72,11 +73,12 @@
            importmap)
          (get-output-string outstr))))
 
-   ;; Builds an embedcfg file, which maps from the pattern used in `go:embed`
+   ;; Builds an `embedcfg` file, which maps the pattern used in `//go:embed` directives
-   ;; to a list of files, as well as a filename to on-disk file mapping.
+   ;; to a list of files that need to be embedded, and a mapping of filename to on-disk
+   ;; file path.
    ;;
-   ;; - `++patterns++` is an alist of the pattern used to match files (e.g. `++foo/++`, or `++a.*++`) to a list of filenames.
+   ;; - `patterns` is an alist of the pattern used to match files (e.g. `foo/`, or `a.*`) to a list of filenames.
-   ;; - `++files++` is an alist of file name to actual path.
+   ;; - `files` is an alist mapping the file name in `patterns` to an on-disk path.
    (define (build-embedcfg patterns files)
      (call-with-port (open-output-string)
        (lambda (outstr)
@@ -88,6 +90,7 @@
          (get-output-string outstr))))
 
    ;; Clean up the package name to use in drv names.
+   ;; Replaces special characters seen in package names to underscores.
    (define (rewrite-package-name name)
      (set! name (string-copy name))
      (do ((x 0 (+ x 1)))
@@ -99,23 +102,29 @@
 
    ;; An empty go_asm.h file used when generating symabis.
    (define empty-asmhdr (zdir `(("go_asm.h" . ,(zfile "")))))
-  
+
-   ;; The environment to append to the build environment for Go.
+   ;; The environment to append to the build environment for any Go derivation.
    (define (env-for-goarch)
      `(("GOARCH" . ,(%goarch))))
-  
+
-   ;; Extra defines to add to `++go tool asm++` uses.
+   ;; Extra arguments to add to uses of `++go tool asm++`, to set the expected preprocessor
+   ;; variables.
    (define (defines-for-goarch)
      `(
        "-D" "GOOS_linux"
        "-D" ,(string-append "GOARCH_" (%goarch))
        ,@(if (string=? (%goarch) "amd64") '("-D" "GOAMD64_v1") '())))
 
-   ;; Returns an alist of three store paths.
+   ;; Calls `go tool compile` with the provided arguments.
+   ;; 
+   ;; - `std` must be `#t` when compiling the standard library, and `#f` otherwise.
+   ;; - `package-name` is the (zexp) string with the name of the package.
+   ;; - `importcfg` must be set, and is a zexp path to a file containing the import map.
+   ;; - `symabis` and `embeds` (optionally) point to a file containing their respective configuration.
+   ;; - `files` is an alist of all files to compile, mapping filename to on-disk path.
+   ;;   The filenames are used in `-trimpath`, providing for better runtime traces and error messages.
    ;;
-   ;; - `++api++` containing the compiler's output, used when compiling
+   ;; Returns an alist containing store paths for `api`, `code`, and `asmhdr` outputs.
-   ;; - `++code++` contains the compiled code, used during linking only.
-   ;; - `++asmhdr++` contains the headers needed for any assembly code inside this package.
    (define (go-compile std package-name importcfg symabis embeds files)
      (define args
        #~(
@@ -133,13 +142,14 @@
           "-trimpath" ,(apply string-append (map (lambda (f) (string-append (cdr f) "=>" package-name "/" (car f) ";")) #$files))
           . ,(map cdr #$files)))
 
-     (store-path-for-ca-drv*
+     (store-path-for-ca-drv
        (string-append (rewrite-package-name package-name) "-src")
        "x86_64-linux"
        #~(,(string-append #$go-toolchain "/bin/go") "tool" "compile" . #$args)
        (env-for-goarch) '("api" "code" "asmhdr")))
-  
+
-   ;; Returns a store path containing the symabi for the assembly files provided.
+   ;; Calls `go tool asm -gensymabis`, returning a store path containing the symabi for the assembly files provided.
+   ;; `include-path` can be set to add a single path to the include path.
    (define (go-generate-symabi package-name include-path files)
      (define args
        #~(
@@ -152,15 +162,17 @@
           ,@(if include-path (list "-I" #$include-path) '())
           . #$files))
 
-     (cdar (store-path-for-ca-drv*
+     (cdar (store-path-for-ca-drv
              (string-append (rewrite-package-name package-name) "-asm-symabis")
              "x86_64-linux"
              #~(,(string-append #$go-toolchain "/bin/go") "tool" "asm" . #$args)
              (env-for-goarch) '("symabi"))))
 
-   ;; Returns a store path containing the `++code++` of the provided assembly
+   ;; Returns a store path containing the `code` of the provided assembly
-   ;; files. Assembly files have no `api`, and cannot be directly interacted
+   ;; files. Assembly files have no `api`, and are used inside the package they're defined in.
-   ;; with from other packages.
+   ;;
+   ;; - `include-path` and `include-path2` are both added to `-I` arguments if set.
+   ;; - `files`, as with `go-compile`, is an alist of file name to on-disk path.
    (define (go-compile-assembly package-name include-path include-path2 files)
      (define args
        #~(
@@ -173,7 +185,7 @@
           "-trimpath" ,(apply string-append (map (lambda (f) (string-append (cdr f) "=>" package-name "/" (car f) ";")) #$files))
           . ,(map cdr #$files)))
 
-     (cdar (store-path-for-ca-drv*
+     (cdar (store-path-for-ca-drv
              (string-append (rewrite-package-name package-name) "-asm")
              "x86_64-linux"
              #~(,(string-append #$go-toolchain "/bin/go") "tool" "asm" . #$args)

lang/go/src/fetch.sld

@@ -1,17 +1,21 @@
-;; Helpers for fetching files from the Go module proxy, slightly impurely.
+;; Helpers for fetching files from the Go module and checksum proxy.
 (define-library (zilch lang go fetch)
   (import
     (scheme base) (scheme write) (scheme read) (scheme file) (scheme char)
     (zilch magic) (zilch zexpr)
     (zilch nixpkgs)
     (chicken process-context) (chicken format) (chicken file))
- 
+
   (export
     fetch-with-known-url rewrite-module-name-for-url)
 
   (begin
     (define fetch-cache-file (string-append (get-environment-variable "HOME") "/.cache/zilch-fetch.scm"))
     (define fetch-cache (if (file-exists? fetch-cache-file) (call-with-input-file fetch-cache-file read) '()))
+
+    ;; Creates a store path that runs `builtin:fetchurl` with specified `name` and `url`.
+    ;; If the hash isn't known in the user-wide cache at `~/.cache/zilch-fetch.scm`, shells
+    ;; out to `nix-prefetch-url` to fetch the path and calculate its hash.
     (define (fetch-with-known-url name url)
       (define cache-entry (assoc url fetch-cache))
       (define hash (if cache-entry

lang/go/src/go.sld

@@ -1,3 +1,6 @@
+;; Higher-level utilities to write Go compilation instructions inside of Zilch.
+;; These act on a `<go-package>` record, which is generated by `go-package-compile`,
+;; and the final program linked together by `go-package-link`.
 (define-library (zilch lang go)
  (import
   (scheme base) (scheme write) (scheme process-context) (scheme lazy)
@@ -8,23 +11,24 @@
   (chicken base) (chicken format) (chicken foreign)
   (srfi-4)
   (zilch lang go core))
-    
+
  (export
+   <go-package>
    make-go-package go-package?
    go-package-name go-package-import-path
    go-package-api go-package-code go-package-dependencies
-  
+
    go-dependency-closure
    go-package-compile go-package-link)
-  
+
 
  (begin
-   ;; A go package consists of a few separate `++(zilch magic)++` store paths.
+   ;; A go package consists of a few separate `(zilch magic)` store paths.
-   ;; The `++name++` is the package name as compiled, and `++import-path++` is a nicer
+   ;; The `name` is the package name as compiled, and `import-path` is a nicer
-   ;; package name for "main" packages. The `++go-package-api++` is a store path consisting
+   ;; package name for "main" packages. The `go-package-api` is a store path consisting
-   ;; of a `++.a++` containing the output of the compiler's `++__.PKGDEF++` only, which
+   ;; of a `.a` containing the output of the compiler's `__.PKGDEF` only, which
    ;; contains the exported types and functions, along with a slight amount of LTO and
-   ;; inlining metadata. the `++go-package-code++` store path contains the actual assembly
+   ;; inlining metadata. the `go-package-code` store path contains the actual assembly
    ;; of the package.
    (define-record-type <go-package>
      (make-go-package name import-path api code dependencies)
@@ -34,7 +38,7 @@
      (api go-package-api)
      (code go-package-code)
      (dependencies go-package-dependencies))
-  
+
    (define-record-printer (<go-package> pkg out)
      (fprintf out "#<go-package ~A  api: ~S code: ~S  deps: ~S>"
        (if (string=? (go-package-import-path pkg) (go-package-name pkg))
@@ -43,8 +47,9 @@
        (go-package-api pkg)
        (go-package-code pkg)
        (map go-package-name (go-package-dependencies pkg))))
- 
+
-   ;; Recursively walk over the dependencies of a `++go-package++`, prepending to the `++vals++` list.
+   ;; Recursively walk over the dependencies of a `<go-package>`, prepending to the `vals` list,
+   ;; and returning the resulting list.
    (define (go-dependency-closure package vals)
     (unless (member package vals)
      (set! vals (cons package vals))
@@ -54,9 +59,16 @@
       (go-package-dependencies package)))
     vals)
 
-   ;; `(go-package-compile name deps source-files)`
-   ;; or `(go-package-compile name path deps source-files assembly-files assembly-includes embed-filenames embed-patterns)`
    ;; Build a Zilch-defined Go package of one store path as source code, and a list of dependencies.
+   ;;
+   ;; - `name` is the full name of the package, or `main` if the package is the main package.
+   ;; - `path` is the full name of the package (e.g. `example.com/foo/bar`).
+   ;; - `deps` is a list of `<go-package>` dependencies.
+   ;; - `source-files` is a (zexp) alist of file name to their location on disk (or store path).
+   ;; - `assembly-files` is identical to `source-files`, but for `.s` files.
+   ;; - `assembly-includes` is either a single on-disk path pointing to a directory, or an alist of file name to on-disk location for header files that should be in scope for `#include` in assembly files.
+   ;; - `embed-patterns` is an alist of Go embed patterns to the filenames they contain;
+   ;; - `embed-filenames` is an alist of filenames used in embed patterns to their on-disk location.
    (define go-package-compile
      (case-lambda
        ((name deps source-files) (go-package-compile name name deps source-files '() '() '() '()))
@@ -69,9 +81,9 @@
 
         (define symabis #f)
         (unless assembly-files (set! assembly-files '()))
-       
+
         (define path-or-name (if (string=? name "main") name path))
-      
+
         (define assembly-includes-dir
           (if (list? assembly-includes)
             (zdir (map (lambda (pair) (cons (car pair) (zsymlink (cdr pair)))) assembly-includes))
@@ -85,7 +97,7 @@
 
         (define merged-asmhdr
           (zdir "go_asm.h" (zsymlink (cdr (assoc "asmhdr" compiled-go)))))
-       
+
         ;; ISSUE: this needs the source dir for assembly imports reasons (filter out .h files?)
         (define compiled-assembly
           (map
@@ -99,26 +111,24 @@
         ; NOTE: .go has to be compiled in one go; but .s is compiled one file at a time.
         (define all-code (cons (cdr (assoc "code" compiled-go)) compiled-assembly))
 
-        ; (printf " -> (store-path-for-ca-drv* meow meow ~S ~S meow)\n" all-code (env-for-goarch))
         ; Use `go tool pack` to merge the code together.
         (define merged-code
           (if (length assembly-files)
-              (cdar (store-path-for-ca-drv*
+              (cdar (store-path-for-ca-drv
                      (string-append "go-" (rewrite-package-name path) "-code") "x86_64-linux"
                      #~(,(string-append #$go-toolchain "/bin/go") "tool" "pack" "c" ,(make-placeholder "code") . #$all-code)
                      (env-for-goarch)
                      '("code")))
               (cdr (assoc "code" compiled-go))))
-      
+
-        ; (printf " -> (make-go-package ~S ~S ~S ~S ~S)\n" name path (cdr (assoc "api" compiled-go)) merged-code deps)
         (make-go-package name path (cdr (assoc "api" compiled-go)) merged-code deps))))
 
-   ;; Link a `++go-package++` into a binary that can be (statically) executed.
+   ;; Link a `<go-package>` into a binary that can be executed.
    (define (go-package-link pkg)
      (define code-importcfg
        (zfile #~,(build-importcfg #$(map (lambda (pkg) (cons (go-package-import-path pkg) (go-package-code pkg))) (go-dependency-closure pkg '())) '())))
 
-     (cdar (store-path-for-ca-drv* (rewrite-package-name (go-package-import-path pkg)) "x86_64-linux"
+     (cdar (store-path-for-ca-drv (rewrite-package-name (go-package-import-path pkg)) "x86_64-linux"
              #~(,(string-append #$go-toolchain "/bin/go") "tool" "link" "-buildid" ,(string-append "zilch out=" (make-placeholder "out")) "-importcfg" #$code-importcfg "-o" ,(make-placeholder "out") #$(go-package-code pkg)) (env-for-goarch) '("out"))))))
 
 

lang/go/src/mod.sld

@@ -1,4 +1,5 @@
-;; Processes go module files.
+;; Resolves dependencies for Go modules and processes them into `<go-package>` files
+;; that can then be used to compile existing Go modules with Zilch.
 (define-library (zilch lang go mod)
  (import
   (scheme base) (scheme write) (scheme read) (scheme file) (scheme process-context) (scheme lazy) (scheme case-lambda)
@@ -13,7 +14,7 @@
   (zilch lang go) (zilch lang go core) (zilch lang go stdlib) (zilch lang go vfs) (zilch lang go sum) (zilch lang go fetch) (zilch lang go package)
   (zilch lang go version)
   (chicken foreign))
- 
+
  (export
    collect-requirements-for-module collect-packages-from-requires)
 
@@ -33,15 +34,16 @@
   (define (read-go-mod mod-file)
     (call-with-port
       ;; TODO(puck): don't use /bin/sh here.
-      (store-path-open (cdar (store-path-for-ca-drv* "go.mod.json" "x86_64-linux" #~("/bin/sh" "-c" ,(string-append #$go-toolchain "/bin/go mod edit -json " #$mod-file " > $out")) '() '("out"))))
+      (store-path-open (cdar (store-path-for-ca-drv "go.mod.json" "x86_64-linux" #~("/bin/sh" "-c" ,(string-append #$go-toolchain "/bin/go mod edit -json " #$mod-file " > $out")) '() '("out"))))
       (lambda (p) (json-read p))))
-  
+
   (define (vector-get-kv-value key vec)
     (vector-any (lambda (v) (and (string=? (car v) key) (cdr v))) vec))
-  
+
-  ;; Reads in the module rooted by the vfs, and resolves its requirements list.
+  ;; Reads in the module rooted by the vfs in `vfs`, and finds all its requirements.
   ;; This returns two values: the name of the root module, and a mapping of
   ;; module name to a pair of its version and the vfs.
+  ;; `replaces` is a list of ``<vfs>``es containing modules that should be prioritized as dependencies.
   (define (collect-requirements-for-module vfs replaces)
     (define sum-lines '())
     (define (parse-sumfile go-sum)
@@ -97,7 +99,7 @@
         (define path-name (handle-vfs vfs))
         (set! collected-requires (mapping-set! collected-requires path-name (cons #f vfs))))
       replaces)
-    
+
     ; we have the right module versions and their files now. Iterate over the packages we have,
     ; until we have none left that need iterating. Once that's done, iterate all the packages and fetch the go.sum for them.
     (define (tick)
@@ -132,14 +134,14 @@
            (call-with-port (store-path-open file) (lambda (port) (read-line port) (parse-sumfile (vector (parse-go-sum-line (read-line port)) (parse-go-sum-line (read-line port))))))))
          found-missing)
         (tick))))
-    
+
     (set! root-path-name (handle-vfs vfs))
     (set! collected-requires (mapping-set! collected-requires root-path-name (cons #f vfs)))
     (tick)
     (values root-path-name collected-requires))
-  
+
   ;; Processes a mapping of module name to a pair of version and vfs, and
-  ;; returns a procedure that takes a package name and returns its go-package.
+  ;; returns a procedure that takes a package name and returns a `<go-package>` representing it.
   (define (collect-packages-from-requires collected-requires)
     (define (process-package vfs last-part full-path pairs headers)
       (define name (cdr (assoc "name" pairs)))
@@ -172,7 +174,7 @@
          (collected-imports (map (lambda (name) (if (is-builtin name) (go-stdlib-ref name) (find-package name)))
                                  (filter (lambda (name) (not (member name '("builtin" "unsafe")))) imports))))
         (go-package-compile name full-path collected-imports collected-files collected-assembly-files collected-assembly-includes embed-filenames embed-patterns)))
-    
+
     (define packages (mapping (make-default-comparator)))
 
     (define (process-packages-for-module root-path vfs)
@@ -200,7 +202,7 @@
           (define pairs (vector->list (cdr pair)))
           (set! packages (mapping-set! packages full-path (delay (process-package vfs (car pair) full-path pairs headers)))))
         module-packages))
-    
+
     (define (find-longest-prefix name)
       (define prefixes (mapping-entries (mapping-filter (lambda (key value) (string-prefix? key name)) collected-requires)))
       (unless (eq? prefixes '())

lang/go/src/package.sld

@@ -1,3 +1,4 @@
+;; Routines to locate Go packages in a `<vfs>`.
 (define-library (zilch lang go package)
   (import
     (scheme base)
@@ -18,9 +19,9 @@
           (map go-stdlib-ref '("encoding/json" "fmt" "go/build" "io" "io/fs" "os" "path" "path/filepath" "sort" "strings" "time"))
           (list (cons "main.go" (zfile (foreign-value "parser_source" nonnull-c-string)))))))
 
-    ;; Uses IFD to find each Go package defined inside this virtual filesystem,
+    ;; Finds each Go package defined inside this virtual filesystem,
-    ;; returning a vector containing pairs, mapping each directory to the
+    ;; and returns a vector containing pairs, mapping the name of each directory to the
-    ;; package defined within.
+    ;; a vector-based structure describing the package defined in said directory.
     (define (find-packages-inside-vfs vfs)
       (define input
         #~,(call-with-port
@@ -29,6 +30,6 @@
               (json-write (vector (cons "GOARCH" (%goarch)) (cons "GOOS" "linux") (cons "files" #$(vfs-to-json (vfs-filter-for-go-package vfs)))) bv)
               (get-output-bytevector bv))))
       (define input-file (zfile input))
-      (define store-path (cdar (store-path-for-ca-drv* "find-packages" "x86_64-linux" #~(#$go-import-parser #$input-file) '() '("out"))))
+      (define store-path (cdar (store-path-for-ca-drv "find-packages" "x86_64-linux" #~(#$go-import-parser #$input-file) '() '("out"))))
       (call-with-port (store-path-open store-path)
                       (lambda (p) (json-read p))))))

lang/go/src/stdlib.sld

@@ -1,3 +1,4 @@
+;; Handles the Go standard library.
 (define-library (zilch lang go stdlib)
   (import
     (scheme base) (scheme file) (scheme write) (scheme process-context) (scheme lazy)
@@ -9,7 +10,7 @@
     (srfi-4)
     (zilch lang go core)
     (zilch lang go))
-    
+
   (export
     go-stdlib-ref)
 
@@ -20,12 +21,12 @@
         out
         (read-all-objects port (cons (json-read port) out))))
 
-    ;; Runs `++go list++` (thru `++/bin/sh++`) and reads (IFD) the output to fetch the metadata of the Go standard library and commands.
+    ;; Runs `go list` (thru `/bin/sh`) and reads the output to fetch the metadata of the Go standard library and commands.
     (define stdlib-objects
       (map vector->list
         (call-with-port
           (store-path-open
-            (cdar (store-path-for-ca-drv*
+            (cdar (store-path-for-ca-drv
                     "stdenv"
                     "x86_64-linux"
                     #~("/bin/sh" "-c" ,(string-append "GOCACHE=$TMPDIR/go-cache " #$go-toolchain "/bin/go list -json -deps std cmd > $out"))
@@ -37,7 +38,7 @@
       (if res (cdr res) '()))
 
     ;; Extract everything until the first space.
-    ;; Space characters are illegal in Go package names, and `++go list -json std++`
+    ;; Space characters are illegal in Go package names, and `go list -json std`
     ;; uses it to disambiguate multiple versions of some internal packages.
     (define (strip-space-bits name)
       (do
@@ -48,9 +49,9 @@
          (if (>= x (string-length name))
            name
            (substring name 0 x)))))
-  
+
     ;; Tail-recursively remove any packages that, if ignoring the postfixed origin
-    ;; (e.g. `++unsafe [cmd/compile]++`), match either `++unsafe++` or `++builtin++`;
+    ;; (e.g. `unsafe [cmd/compile]`), match either `unsafe` or `builtin`;
     ;; these have no source code and are compiler-internal.
     (define (remove-builtin-packages pkgs)
       (if (eq? pkgs '())
@@ -59,21 +60,23 @@
           (if (or (string=? stripped "unsafe") (string=? stripped "builtin"))
             (remove-builtin-packages (cdr pkgs))
             (cons (car pkgs) (remove-builtin-packages (cdr pkgs)))))))
-  
+
     (define (starts-with left right)
       (and
         (>= (string-length right) (string-length left))
         (string=? left (string-copy right 0 (string-length left)))))
-   
+
     (define (filter condition lst)
       (if (eq? lst '())
         '()
         (if (condition (car lst))
           (cons (car lst) (filter condition (cdr lst)))
           (filter condition (cdr lst)))))
-  
+
-    ;; Helper that parses the JSON returned by `++go list -json -deps++` and builds a `++go-package++` record.
+    ;; Helper that parses the JSON returned by `go list -json -deps` and builds a `go-package` record.
-    ;; This is distinct from `go-package-compile` because of format differences,
+    ;; This is distinct from `go-package-compile` because of format differences.
+    ;;
+    ;; TODO(puck): Is this still the case?
     (define (make-stdlib-inner meta)
       (define files (assoc-or-empty "GoFiles" meta)) ; .go files
       (define sfiles (assoc-or-empty "SFiles" meta)) ; .s files
@@ -89,7 +92,7 @@
       (when (string=? package-name "main")
         (set! name package-name))
       (define dir (cdr (assoc "Dir" meta)))
-       
+
       ; Fetch dependencies from the rest of the stdlib data.
       ; We only need the `++api++` at this point.
       (define resolved-imports (map (lambda (v) (cons (strip-space-bits v) (go-package-api (go-stdlib-ref v)))) (remove-builtin-packages imports)))
@@ -140,7 +143,7 @@
       ; Use `go tool pack` to merge the code together.
       (define merged-code
         (cdar
-          (store-path-for-ca-drv*
+          (store-path-for-ca-drv
             (string-append "go-" (rewrite-package-name name) "-code") "x86_64-linux"
             #~(,(string-append #$go-toolchain "/bin/go") "tool" "pack" "c" ,(make-placeholder "code") . #$all-code)
             (env-for-goarch)
@@ -148,11 +151,11 @@
 
       (make-go-package name import-path go-api merged-code (map go-stdlib-ref (remove-builtin-packages imports))))
 
-    ; Each entry is a list (name metadata (api code)).
+    ; Each entry is a list of (name metadata (api code)).
-    ; Use `++delay++` to resolve the DAG lazily on use.
+    ; Use `delay` to resolve the DAG lazily on use.
     (define stdlib-data (map (lambda (v) (list (cdr (assoc "ImportPath" v)) v (delay (make-stdlib-inner v)))) stdlib-objects))
-  
+
-    ;; Finds any package contained within Go's standard library.
+    ;; Finds any package contained within Go's standard library, and returns a `<go-package>` for the package.
     (define (go-stdlib-ref name)
       (define entry (assoc name stdlib-data))
       (unless entry (error (string-append "Could not find package " name " in stdlib")))

lang/go/src/sum.sld

@@ -10,12 +10,16 @@
     (scheme char)
     (srfi 4) (srfi 128) (srfi 146) (srfi 207)
     (chicken foreign))
- 
+
   (export
-    parse-go-sum-line parse-go-sum-file go-sum-line? go-sum-module go-sum-version go-sum-path go-sum-hash)
+    <go-sum-line> go-sum-line?
+    go-sum-module go-sum-version go-sum-path go-sum-hash
+
+    parse-go-sum-line parse-go-sum-file)
 
   (begin
     ;; Contains the values from a single line from a `go.sum` file.
+    ;; `go-sum-hash` is a bytevector containing the base64-decoded hash.
     (define-record-type <go-sum-line>
       (make-go-sum-line module version path hash)
       go-sum-line?
@@ -30,7 +34,7 @@
         (go-sum-version sum)
         (if (go-sum-path sum) (go-sum-path sum) "")
         (bytevector->base64 (go-sum-hash sum))))
-  
+
     (define (string-find str index char)
       (cond
         ((= index (string-length str)) #f)
@@ -55,7 +59,7 @@
         (set! version (string-copy version 0 path-index)))
       (make-go-sum-line module-path version path (base64->bytevector (string-copy hash 3))))
 
-    ;; Parses all the `go.sum` lines from `port`.
+    ;; Returns a list of all ``<go-sum-line>``s read from `port`.
     (define (parse-go-sum-file port)
       (do ((parsed '())
            (line "" (read-line port)))

lang/go/src/version.sld

@@ -1,4 +1,4 @@
-;; Procedures to deal with Go's semantic versions.
+;; Procedures to deal with Go's format of semantic versions.
 (define-library (zilch lang go version)
   (import
     (scheme base) (srfi 152))

lang/go/src/vfs.sld

@@ -44,9 +44,8 @@
                            (if (char-upper-case? ch) (set! out (string-append out (string #\! (char-downcase ch)))) (set! out (string-append out (string ch))))) name)
       out)
 
-    ;; Reads a dirhash from a `go.sum` line. This prefetches the module from
+    ;; Uses the hash in a `<go-sum-line>` to return a store path containing a list of the hashes of each file,
-    ;; the go module proxy, and then generates the dirhash without unpacking
+    ;; following the https://pkg.go.dev/golang.org/x/mod/sumdb/dirhash[`dirhash`] format used by the `go.sum` files.
-    ;; said module file.
     (define (fetch-dirhash-for-sum sum-line)
       (when (go-sum-path sum-line) (error "go.sum line is invalid for fetch-dirhash-for-sum" sum-line))
       (define url (string-append "https://proxy.golang.org/" (rewrite-go-package-name-for-url (go-sum-module sum-line)) "/@v/" (go-sum-version sum-line) ".zip"))
@@ -54,7 +53,8 @@
       (store-path-for-fod "module" "x86_64-linux" #~(#$(force dirhash-generator)) #~(("src" . #$known)) "sha256" (go-sum-hash sum-line) #f))
 
     ;; Generates a full VFS structure from a module as described by a `go.sum`
-    ;; line.
+    ;; line. Uses `fetch-dirhash-for-sum` to generate the dirhash, then creates FODs
+    ;; for each file based on said hash; this is then returned as a `<vfs>` structure.
     (define (vfs-from-dirhash sum-line)
       (define dirhash-file (fetch-dirhash-for-sum sum-line))
       (define url (string-append "https://proxy.golang.org/" (rewrite-go-package-name-for-url (go-sum-module sum-line)) "/@v/" (go-sum-version sum-line) ".zip"))
@@ -106,14 +106,17 @@
             ((= i 0) #f)
             (else (extract-extension name (- i 1)))))
 
-    ;; Returns a VFS, filtered down to only contain the contents of files that
+    ;; Returns a subset of `vfs`, filtered down to only contain the contents of files that
-    ;; will be read during the processing of Go packages.
+    ;; Go cares about.
     (define (vfs-filter-for-go-package vfs)
       (vfs-dir-filter vfs
         (lambda (dir fname contents)
           (define extension (extract-extension fname (- (string-length fname) 1)))
           (member extension good-extensions))))
 
+    ;; Generates a representation of the `vfs`, processed to be turned into JSON.
+    ;; Primarily used by `(zilch lang go package)` to allow extracting the necessary
+    ;; information of each package from a module's vfs.
     (define (vfs-to-json vfs)
       (mapping-map->list
         (lambda (k v) (list (car k) (cdr k) (if (eq? v 'directory) "" v)))