diff --git a/docs/README b/docs/README new file mode 100644 index 0000000..dbab496 --- /dev/null +++ b/docs/README @@ -0,0 +1,2 @@ +To generate the documentation, run: +`docread; npx antora generate antora-playbook.yml`. diff --git a/docs/docread/docread.scm b/docs/docread/docread.scm index 01db1ca..56270df 100644 --- a/docs/docread/docread.scm +++ b/docs/docread/docread.scm @@ -60,7 +60,7 @@ (define out-file (open-output-file out-path)) ; Print out the comments - (fprintf out-file "= `~S`\n" (car contents)) + (fprintf out-file "= `~S`\n\n" (car contents)) (for-each (lambda (l) (fprintf out-file "~A\n" l)) lib-comments) (fprintf out-file "\n:toc:\n\n") (for-each (lambda (i) diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index f52b0a7..11491c7 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -1,5 +1,5 @@ * xref:index.adoc[] -* xref:nixexpr.adoc[] +* xref:go.adoc[] * xref:zexp.adoc[] * ++(zilch)++ @@ -27,6 +27,3 @@ *** xref:generated:zilch.nix.drv.adoc[++(zilch nix drv)++] *** xref:generated:zilch.nix.hash.adoc[++(zilch nix hash)++] *** xref:generated:zilch.nix.path.adoc[++(zilch nix path)++] - - -* xref:architecture.adoc[] diff --git a/docs/modules/ROOT/pages/architecture.adoc b/docs/modules/ROOT/pages/architecture.adoc deleted file mode 100644 index a85e30c..0000000 --- a/docs/modules/ROOT/pages/architecture.adoc +++ /dev/null @@ -1,11 +0,0 @@ -= Architecture - -`(zilch nix binproto)` contains an implementation of the binary protocol used both -to talk to the daemon and to build NAR files. - -On top of that is `(zilch nix daemon)`, which implements a version (which?) of -the Nix worker protocol. - -`(zilch nix drv)` allows reading and writing .drv objects. - -`(zilch nix path)` contains the helpers for building store paths of various types. diff --git a/docs/modules/ROOT/pages/go.adoc b/docs/modules/ROOT/pages/go.adoc new file mode 100644 index 0000000..aed0576 --- /dev/null +++ b/docs/modules/ROOT/pages/go.adoc @@ -0,0 +1,35 @@ += Go + +Currently, the primary binary produced by the `zilch` repository is a tool to +generate content-addressed derivations from a Go project, called `zilch-cli-go`. + +It requires the `ca-derivations` and `impure-derivations` experimental features +to be enabled on the Nix daemon, and a Nix daemon to be available at the +default store path. It also requires the daemon to be able to run +`x86_64-linux` derivations, right now. + +Once run, it will use Zilch to build a series of derivations, and output a +`.drv` for each executable package in the module. + +The help page: + +[source] +---- +Usage: zilch-cli-go [OPTION] [PACKAGE...] +Process the given module (or the current directory, if unspecified) and +output derivations for each package given on the command line (or all +executables in the module, if unspecified) + + -h, --help Print this help message. + -b, --build Build the store paths, rather than show their + derivations. + -L, --print-build-logs Print derivation logs as they come in. + -m, --module-dir DIR The directory to use as root module. + -r, --replace DIR Replace the module specified by the go.mod + with this source directory, rather than using + the upstream module. Can be specified more + than once. + + --debug Crash on the first error, rather than + continuing with the next package. +---- diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index 059469f..aa3d958 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -15,7 +15,7 @@ Features: == Current work Current effort in Zilch is working on making "incremental", bitesize, -derivations to work. This is currently being implemented for Go. +derivations to work. This is xref:go.adoc[currently being implemented for Go]. == Contributing diff --git a/docs/modules/ROOT/pages/nixexpr.adoc b/docs/modules/ROOT/pages/nixexpr.adoc deleted file mode 100644 index 2421a40..0000000 --- a/docs/modules/ROOT/pages/nixexpr.adoc +++ /dev/null @@ -1,41 +0,0 @@ -= Nix expression support - -When `(nix reader)` is imported, it is possible to execute Nix code inline with -Scheme code. Simply wrap your Nix code in curly brackets: - -[,scheme] ----- -(write - (string-append - "Hello, Nix version" - { builtins.nixVersion })) ----- - -The following values can be translated: - -|=== -| Nix | Scheme | Notes - -| string | string | (Loses string context.) -| integer | number | -| float | number | -| boolean | boolean | -| lambda | procedure | (with single argument) -| list | vector or list | Depends on the status of `\*translate-list-as-vector*` -| attrset | alist | -| builtin | procedure | -| external value | symbol, other unknown objects | -|=== - -If a value in Nix is preceded with a comma, it is unquoted, similar to -`(unquote)` in a Scheme quasiquotation. If prefixed with a single quote, it is -`(quote)`-d. - -[,scheme] ----- -(define - (test-append foo) - (string-append "Hello, " foo)) -(write - { ,test-append "world!") }) ----- diff --git a/docs/modules/ROOT/pages/zexp.adoc b/docs/modules/ROOT/pages/zexp.adoc index 07656b5..b073696 100644 --- a/docs/modules/ROOT/pages/zexp.adoc +++ b/docs/modules/ROOT/pages/zexp.adoc @@ -1,11 +1,15 @@ = zexps zexps, similar to g-expressions in Guix, are a way to generate -S-expressions that are taggged with store paths. But that's where -the similarity ends. +S-expressions that are tagged with store paths. The syntax in Zilch is inspired +by it, but has been developed separately. +A zexp is used similarly to a ``quasiquote``d value in Scheme, but has an extra type +of unquoting, called `zexp-unquote`, which unquotes a value such as a `++++` +or another `zexp`. -To create a zexp, you can use either the full syntax, or the reader macro: +Each `zexp` keeps track of the derivation outputs and store files it depends on, +similarly to how string context works in Nix: [,scheme] ---- @@ -14,15 +18,14 @@ To create a zexp, you can use either the full syntax, or the reader macro: (zexp (foo bar (zexp-unquote baz))) ---- -`(zexp-unquote VAL)` returns the value that the zexp (or any compatible record) -contains, while gathering the `zexp-unquote`d values used. - -Like quasiquotation, zexps can use `unquote`, including ``zexp-unquote``d values -inside the ``unquote``d code. ``unquote``d code is evaluated when the `zexp` is evaluated. +When a `zexp-unquote` (or its reader syntax, `#$`) is encountered, the value +contained in the `zexp` (or compatible object) is used as-is, with no +evaluation. It is possible to mix and match `zexp-unquote` with `unquote`, +allowing building e.g. dynamic strings from zexps. `zexp-unquote` is always +evaluated before `unquote` is. [,scheme] ---- -(define world #~,(begin (write "hello") "world")) -(define hello #~("hello" ,(string-append "very " "cute") #$world)) -; When hello is used as zexp, it will also write "hello" to the output port. +(define world ...) +(define hello #~("hello" ,(string-append "very " "cute" #$world))) ---- diff --git a/lang/go/src/core.sld b/lang/go/src/core.sld index aea4827..46e1d9f 100644 --- a/lang/go/src/core.sld +++ b/lang/go/src/core.sld @@ -1,3 +1,18 @@ +;; Defines the baseline definitions needed to compile Go code using Zilch. +;; +;; To make incremental builds work, this library uses up to four distinct +;; outputs: +;; +;; - `api` is any `.a` file needed for compiling any other Go code that depends +;; on said package, and is usually an archive containing a file named +;; `__.PKGDEF`. +;; - `code` is a `.a` file containing the actual binary code for the target +;; arch, and is used only during linking. +;; - `asmhdr` is a header file generated by the Go code, and used during +;; assembly compilation only. +;; - `symabi` is a text file contaning the functions defined by assembly, and +;; their ABI, which is used while compiling the Go code that contains the +;; stubs for any assembly code. (define-library (zilch lang go core) (import (scheme base) (scheme write) (scheme process-context) (scheme lazy) @@ -22,15 +37,19 @@ (begin + ;; The architecture to target the Go code at. + ;; Equivalent to `GOARCH`. (define %goarch (make-parameter "amd64")) - ; Import the existing Go from nixpkgs. + ;; The Go toolchain to use to compile this all. (define go-toolchain (cdr (assoc "out" (nixpkgs "go_1_23")))) - ;; Builds an importcfg file, containing an alist of packages to .a files, - ;; and an alist of package names to actual package names. - ;; `++packagefiles++` is a alist of package name to .a file (api type), - ;; `++importmap++` is an alist of package name to actual package name (used in cases of e.g. replace) + ;; Builds an importcfg file. This file describes the mapping of both + ;; packages to their api, and the mapping of package name as used in `import` + ;; to the actual package names (e.g in case of `replace`.) + ;; + ;; - `++packagefiles++` is a alist of package name to .a file. + ;; - `++importmap++` is an alist of package name to actual package name. (define (build-importcfg packagefiles importmap) (call-with-port (open-output-string) (lambda (outstr) @@ -53,8 +72,11 @@ importmap) (get-output-string outstr)))) - ;; `++patterns++` is an alist of the pattern used to match files (e.g. `++foo/++`, or `++a.*++`) to a list of filenames. - ;; `++files++` is an alist of file name to actual path. + ;; Builds an embedcfg file, which maps from the pattern used in `go:embed` + ;; to a list of files, as well as a filename to on-disk file mapping. + ;; + ;; - `++patterns++` is an alist of the pattern used to match files (e.g. `++foo/++`, or `++a.*++`) to a list of filenames. + ;; - `++files++` is an alist of file name to actual path. (define (build-embedcfg patterns files) (call-with-port (open-output-string) (lambda (outstr) @@ -65,7 +87,7 @@ outstr) (get-output-string outstr)))) - ; Clean up the package name to use in drv names. + ;; Clean up the package name to use in drv names. (define (rewrite-package-name name) (set! name (string-copy name)) (do ((x 0 (+ x 1))) @@ -75,10 +97,10 @@ (when (char=? (string-ref name x) #\space) (string-set! name x #\_)) (when (char=? (string-ref name x) #\]) (string-set! name x #\_)))) - ;; An empty go_asm.h file used to generate symabis. + ;; An empty go_asm.h file used when generating symabis. (define empty-asmhdr (zdir `(("go_asm.h" . ,(zfile ""))))) - ;; Environment to append to the build environment for Go. + ;; The environment to append to the build environment for Go. (define (env-for-goarch) `(("GOARCH" . ,(%goarch)))) @@ -89,16 +111,19 @@ "-D" ,(string-append "GOARCH_" (%goarch)) ,@(if (string=? (%goarch) "amd64") '("-D" "GOAMD64_v1") '()))) - ;; Returns an alist of three store paths; `++api++` containing the compiler's output, - ;; `++code++` containing the linkobj, and `++asmhdr++` containing the headers needed for assembly - ;; code to properly use Go functions and variables. + ;; Returns an alist of three store paths. + ;; + ;; - `++api++` containing the compiler's output, used when compiling + ;; - `++code++` contains the compiled code, used during linking only. + ;; - `++asmhdr++` contains the headers needed for any assembly code inside this package. (define (go-compile std package-name importcfg symabis embeds files) (define args #~( ,@(if std '("-std") '()) #$@(if symabis `("-symabis" ,#$symabis) '()) #$@(if embeds `("-embedcfg" ,#$embeds) '()) - "-buildid" "zilch go-compile" ; this goes into both code and __.PKGDEF, so can't be a reference to the code output, sadly + ; this goes into both code and __.PKGDEF, so can't be a reference to the code output, sadly + "-buildid" "zilch go-compile" "-p" #$package-name "-o" ,(make-placeholder "api") "-linkobj" ,(make-placeholder "code") @@ -114,7 +139,7 @@ #~(,(string-append #$go-toolchain "/bin/go") "tool" "compile" . #$args) (env-for-goarch) '("api" "code" "asmhdr"))) - ;; Returns a store path containing the symabi for the files provided. + ;; Returns a store path containing the symabi for the assembly files provided. (define (go-generate-symabi package-name include-path files) (define args #~( @@ -133,7 +158,9 @@ #~(,(string-append #$go-toolchain "/bin/go") "tool" "asm" . #$args) (env-for-goarch) '("symabi")))) - ;; Returns a store path containing the `++code++` of the provided assembly files. + ;; Returns a store path containing the `++code++` of the provided assembly + ;; files. Assembly files have no `api`, and cannot be directly interacted + ;; with from other packages. (define (go-compile-assembly package-name include-path include-path2 files) (define args #~( diff --git a/lang/go/src/fetch.sld b/lang/go/src/fetch.sld index f59b24a..607befc 100644 --- a/lang/go/src/fetch.sld +++ b/lang/go/src/fetch.sld @@ -1,26 +1,30 @@ +;; Helpers for fetching files from the Go module proxy, slightly impurely. (define-library (zilch lang go fetch) (import (scheme base) (scheme write) (scheme read) (scheme file) (scheme char) (zilch magic) (zilch zexpr) (zilch nixpkgs) - (chicken format)) + (chicken process-context) (chicken format) (chicken file)) (export fetch-with-known-url rewrite-module-name-for-url) (begin - (define fetch-cache (call-with-input-file "/home/.zilchcache" read)) + (define fetch-cache-file (string-append (get-environment-variable "HOME") "/.cache/zilch-fetch.scm")) + (define fetch-cache (if (file-exists? fetch-cache-file) (call-with-input-file fetch-cache-file read) '())) (define (fetch-with-known-url name url) (define cache-entry (assoc url fetch-cache)) (define hash (if cache-entry (cdr cache-entry) - (begin (printf "..fetching ~S ~S\n" name url) + (begin (fprintf (current-error-port) "(pre)fetching ~S ~S\n" name url) (nix-prefetch-url name url #f)))) (unless cache-entry (set! fetch-cache (cons (cons url hash) fetch-cache)) - (call-with-output-file "/home/.zilchcache" (lambda (out) (write fetch-cache out)))) - (store-path-for-fod name "builtin" '("builtin:fetchurl") `(("url" . ,url) ("outputHashMode" . "flat")) "sha256" hash #f)) + (call-with-output-file fetch-cache-file (lambda (out) (write fetch-cache out)))) + (store-path-for-fod name "builtin" '("builtin:fetchurl") `(("url" . ,url)) "sha256" hash #f)) + ;; Rewrites the module name to prefix all uppercase letters with an + ;; exclamation mark instead, as required by the various Go webservices. (define (rewrite-module-name-for-url name) (define out "") (string-for-each diff --git a/lang/go/src/mod.sld b/lang/go/src/mod.sld index 0a8d051..517be4d 100644 --- a/lang/go/src/mod.sld +++ b/lang/go/src/mod.sld @@ -1,4 +1,4 @@ -;; Processes go modules. +;; Processes go module files. (define-library (zilch lang go mod) (import (scheme base) (scheme write) (scheme read) (scheme file) (scheme process-context) (scheme lazy) (scheme case-lambda) @@ -27,7 +27,8 @@ (cons (car lis) (filter proc (cdr lis)))) (else (filter proc (cdr lis))))) - ;; Read a go.mod file. This returns a processed json object, like `go mod edit -json` outputs by default. + ;; Read a go.mod file. This returns a processed json object, like + ;; `go mod edit -json` outputs by default. (define (read-go-mod mod-file) (call-with-port ;; TODO(puck): don't use /bin/sh here. @@ -38,7 +39,8 @@ (vector-any (lambda (v) (and (string=? (car v) key) (cdr v))) vec)) ;; Reads in the module rooted by the vfs, and resolves its requirements list. - ;; This returns two values: the name of the root module, and a mapping of module name to a pair of its version and the vfs. + ;; This returns two values: the name of the root module, and a mapping of + ;; module name to a pair of its version and the vfs. (define (collect-requirements-for-module vfs replaces) (define sum-lines '()) (define (parse-sumfile go-sum) @@ -135,7 +137,8 @@ (tick) (values root-path-name collected-requires)) - ;; Processes a mapping of module name to a pair of version and vfs, and returns a procedure that takes a package name and returns its go-package. + ;; Processes a mapping of module name to a pair of version and vfs, and + ;; returns a procedure that takes a package name and returns its go-package. (define (collect-packages-from-requires collected-requires) (define (process-package vfs last-part full-path pairs headers) (define name (cdr (assoc "name" pairs))) diff --git a/lang/go/src/package.sld b/lang/go/src/package.sld index 454307d..fa4c2f8 100644 --- a/lang/go/src/package.sld +++ b/lang/go/src/package.sld @@ -17,6 +17,9 @@ (map go-stdlib-ref '("encoding/json" "fmt" "go/build" "io" "io/fs" "os" "path" "path/filepath" "sort" "strings" "time")) (list (cons "main.go" (zfile (foreign-value "parser_source" nonnull-c-string))))))) + ;; Uses IFD to find each Go package defined inside this virtual filesystem, + ;; returning a vector containing pairs, mapping each directory to the + ;; package defined within. (define (find-packages-inside-vfs vfs) (define input #~,(call-with-port diff --git a/lang/go/src/stdlib.sld b/lang/go/src/stdlib.sld index ea60f1a..291f7b9 100644 --- a/lang/go/src/stdlib.sld +++ b/lang/go/src/stdlib.sld @@ -152,7 +152,7 @@ ; Use `++delay++` to resolve the DAG lazily on use. (define stdlib-data (map (lambda (v) (list (cdr (assoc "ImportPath" v)) v (delay (make-stdlib-inner v)))) stdlib-objects)) - ;; Wrapper that forces evaluation of the promise fetching from a Go stdlib entry. + ;; Finds any package contained within Go's standard library. (define (go-stdlib-ref name) (define entry (assoc name stdlib-data)) (unless entry (error (string-append "Could not find package " name " in stdlib"))) diff --git a/lang/go/src/sum.sld b/lang/go/src/sum.sld index ead5471..8a01330 100644 --- a/lang/go/src/sum.sld +++ b/lang/go/src/sum.sld @@ -1,3 +1,4 @@ +;; Parses `go.sum` files. (define-library (zilch lang go sum) (import (scheme base) (scheme write) (scheme read) (scheme file) (scheme process-context) (scheme lazy) (scheme case-lambda) @@ -14,6 +15,7 @@ parse-go-sum-line parse-go-sum-file go-sum-line? go-sum-module go-sum-version go-sum-path go-sum-hash) (begin + ;; Contains the values from a single line from a `go.sum` file. (define-record-type (make-go-sum-line module version path hash) go-sum-line? @@ -35,6 +37,7 @@ ((char=? (string-ref str index) char) index) (else (string-find str (+ index 1) char)))) + ;; Parses a `go.sum` line, and returns a ``. (define (parse-go-sum-line line) (define version-space-index (string-find line 0 #\space)) (unless version-space-index (error "go.sum line contains no space characters")) @@ -52,6 +55,7 @@ (set! version (string-copy version 0 path-index))) (make-go-sum-line module-path version path (base64->bytevector (string-copy hash 3)))) + ;; Parses all the `go.sum` lines from `port`. (define (parse-go-sum-file port) (do ((parsed '()) (line "" (read-line port))) diff --git a/lang/go/src/version.sld b/lang/go/src/version.sld index 4466649..af77235 100644 --- a/lang/go/src/version.sld +++ b/lang/go/src/version.sld @@ -1,9 +1,13 @@ +;; Procedures to deal with Go's semantic versions. (define-library (zilch lang go version) (import (scheme base) (srfi 152)) (export parse-version versionvector (map (lambda (pair) (cons (car pair) (list->vector (cdr pair)))) dirs))) + ;; Generates a full VFS structure from an on-disk directory. (define (vfs-from-directory osdir) (define iter-dir #f) (define output '()) @@ -125,6 +139,8 @@ (iter-dir "") (list->vector output)) + ;; Calls `filter` for each file in the virtual filesystem, replacing its + ;; contents with an empty file if `filter` returns false. (define (filter-vfs vfs filter) (vector-map (lambda (dir) @@ -137,13 +153,15 @@ (cdr dir)))) vfs)) - ; List extracted from go src/go/build/build.go. + ;; List extracted from go src/go/build/build.go. (define good-extensions '("go" "c" "cc" "cpp" "cxx" "m" "h" "hh" "hpp" "hxx" "f" "F" "for" "f90" "s" "S" "sx" "swig" "swigcxx" "syso")) (define (extract-extension name i) (cond ((char=? (string-ref name i) #\.) (string-copy name (+ i 1))) ((= i 0) #f) (else (extract-extension name (- i 1))))) + ;; Returns a VFS, filtered down to only contain the contents of files that + ;; will be read during the processing of Go packages. (define (filter-vfs-for-package-reading vfs) (filter-vfs vfs (lambda (dir fname)