(zilch lang go): document
Change-Id: I6a6a6964558b4fe2f96d78120b2e899f91d48c22
This commit is contained in:
parent
f0ce185d5c
commit
18f2887eba
13 changed files with 457 additions and 96 deletions
|
|
@ -5,6 +5,11 @@
|
|||
** xref:core/zexp.adoc[]
|
||||
** xref:core/vfs.adoc[]
|
||||
** xref:core/zilch-as-lib.adoc[]
|
||||
* Language support
|
||||
** Go
|
||||
*** xref:go/usage.adoc[]
|
||||
*** xref:go/library.adoc[]
|
||||
*** xref:go/man.adoc[Man page]
|
||||
|
||||
* Code reference
|
||||
include::generated:partial$nav.adoc[]
|
||||
|
|
|
|||
210
docs/modules/ROOT/pages/go/library.adoc
Normal file
210
docs/modules/ROOT/pages/go/library.adoc
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
= Writing Go packages directly in Zilch
|
||||
:page-pagination:
|
||||
|
||||
It's possible to use Zilch as generic build system for Go. This page
|
||||
serves to document that, as well as the internals of the Go build
|
||||
library in Zilch.
|
||||
|
||||
== Go packages in Zilch
|
||||
|
||||
Each Go package is represented in Zilch by a xref:generated:zilch.lang.go.adoc#++_go-package_++[`<go-package>`],
|
||||
which represents the compiled API and ABI as separate store paths, plus the
|
||||
name of the package, and its dependencies.
|
||||
|
||||
The simplest possible Go package consists of a single file:
|
||||
|
||||
[,scheme,line-comment=;]
|
||||
----
|
||||
(define example
|
||||
(go-package-compile
|
||||
"example.com/package" ; <1>
|
||||
'() ; <2>
|
||||
`(("example.go" . ,(zfile "package example"))))) ; <3>
|
||||
;; #<go-package example.com/package
|
||||
;; api: #<store path /1ahdindygxf34gjhdj9l6h43i6cwsky0v02ai75g35bp5n1i80gp
|
||||
;; (ca~ /nix/store/f4bj92n5yfhp5f29jmacydghsvkaj7fq-example.com_package-src.drv!api)>
|
||||
;; code: #<store path /0ln85mblval4j4cpjgw65f4phrbiagzh1hd5pryjlnnknbq75agj
|
||||
;; (ca~ /nix/store/8dqr20s72i4w4llp73imm9bkcwmx9via-go-example.com_package-code.drv!code)>
|
||||
;; deps: ()>
|
||||
----
|
||||
<1> Name of the package
|
||||
<2> List of dependency packages (in this case, none)
|
||||
<3> Mapping of filename to in-store files
|
||||
|
||||
To create an executable, though, a bit more work is necessary, which is
|
||||
abstracted by the Go build system by default. The "main" package, which
|
||||
contains the `func main()`, always has the package name "main". However, this
|
||||
is undesirable for the name in panics. For this case, the extended
|
||||
`go-package-compile` procedure allows setting both:
|
||||
|
||||
[,scheme,line-comment=;]
|
||||
----
|
||||
(define binary
|
||||
(go-package-compile
|
||||
"main" ; <1>
|
||||
"example.com/binary" ; <2>
|
||||
(list example) ; <3>
|
||||
`(("main.go" . ,(zfile "package main\nfunc main() { }"))) ; <4>
|
||||
'() '() ; <5>
|
||||
'() '())) ; <6>
|
||||
;; #<go-package
|
||||
;; main (example.com/binary)
|
||||
;; api: #<store path /0p33flbfjc0s9ykgmcyzqbr3dhsq0344zswhl6xjvqdbin9ysqch
|
||||
;; (ca~ /nix/store/hg5j28f63b4czfxm4cg76rjbh4bx6ivy-main-src.drv!api)>
|
||||
;; code: #<store path /15is1garmlclnb06qj9ymxdi8xpsgakrx97qdb2jmx0562wn7kgx
|
||||
;; (ca~ /nix/store/z9w0rccki2rhjrgxr0y063qijz64m6p8-go-example.com_binary-code.drv!code)>
|
||||
;; deps: ("example.com/package")>
|
||||
----
|
||||
<1> The package name, as provided to the compiler (must be `main` for binaries)
|
||||
<2> The package name as shown in stacktraces
|
||||
<3> Direct dependencies for this package
|
||||
<4> The source file for this package
|
||||
<5> Assembly-related files
|
||||
<6> ``go:embed``-related files
|
||||
|
||||
To use this binary, it still needs to be linked. This can be done with
|
||||
`go-package-link`.
|
||||
|
||||
[IMPORTANT]
|
||||
====
|
||||
If you're linking a Go binary, make sure to add the `runtime` package to your
|
||||
dependencies, either directly or indirectly. Otherwise, you will end up with an
|
||||
error like so:
|
||||
|
||||
[,scheme,line-comment=;]
|
||||
----
|
||||
(store-path-realised (go-package-link binary))
|
||||
;> [..building "/nix/store/jr17baky22c0zzpjfi97cw16k5wmqqc3-example.com_binary.drv"]
|
||||
;> [0/1 builds, 1 running]
|
||||
;> loadinternal: cannot find runtime
|
||||
;> panic: could not look up runtime.mapinitnoop
|
||||
----
|
||||
|
||||
This is caused by Zilch not adding any standard library code by default, while
|
||||
the Go compiler depends on structures in the standard library to provide runtime
|
||||
support for the garbage collector and goroutines.
|
||||
====
|
||||
|
||||
== Using the Go standard library
|
||||
The Go standard library, unlike most other languages, is shipped as source code.
|
||||
When you compile Go code, the compiler will also compile the parts of the
|
||||
standard library you use. Zilch does the same.
|
||||
|
||||
`(zilch lang go stdlib)` contains a single procedure, `go-stdlib-ref`, which
|
||||
handles this for you:
|
||||
|
||||
[,scheme,line-comment=;]
|
||||
----
|
||||
(go-stdlib-ref "fmt")
|
||||
;; #<go-package fmt
|
||||
;; api: #<store path /1al016fvsnknhpzdzbcf7kjxyzbf3fj8sbm40p8h32x3kr0aswbv
|
||||
;; (ca~ /nix/store/vpvpmx2d43ix6m142jxxg0z1kqddypzl-fmt-src.drv!api)>
|
||||
;; code: #<store path /0sssb21sd2d398fv4vwlykf0shnlyr31kf372k65djqrgxrz5674
|
||||
;; (ca~ /nix/store/pi9nraymjdn12hg7r18h13js56w0s1b6-go-fmt-code.drv!code)>
|
||||
;; deps: ("errors" "internal/fmtsort" "io" "math" "os" "reflect"
|
||||
;; "slices" "strconv" "sync" "unicode/utf8")>
|
||||
|
||||
(define hello-world
|
||||
(go-package-compile
|
||||
"main"
|
||||
"example.com/binary"
|
||||
(list example (go-stdlib-ref "fmt"))
|
||||
`(("main.go"
|
||||
. ,(zfile "package main
|
||||
import \"fmt\"
|
||||
func main() {
|
||||
fmt.Printf(\"Hello, world!\\n\")
|
||||
}")))
|
||||
'() '()
|
||||
'() '()))
|
||||
|
||||
(define linked (go-package-link hello-world))
|
||||
;; #<store path /1csyxx5m27a4819g5m8bnm88gfgcspxisg19rv2kg6b7cv7wn066
|
||||
;; (ca~ /nix/store/jjqhwl7cq2crhdjdsrkby4ng584n3pc1-example.com_binary.drv!out)>
|
||||
|
||||
(store-path-realised linked)
|
||||
;> [..building "/nix/store/5bgrfjgnvzr8wqksl5j23xr5wm7rahnw-zilchfile.drv"]
|
||||
;> …
|
||||
;> [0/2 builds, 0 running]
|
||||
;> [..building "/nix/store/jdk0wbnrz16sbn41r0y6f82hxl93q87r-example.com_binary.drv"]
|
||||
;; "/nix/store/p71p43d3cv48rgi4yh9dvlzssgfkb5vf-example.com_binary"
|
||||
----
|
||||
|
||||
The binary that is output by this code is incrementally built, one package at a
|
||||
time, including the standard library.
|
||||
|
||||
[#vfs]
|
||||
== Creating virtual filesystems for Go modules
|
||||
Go's `go.sum` files contain enough information to recover the full file
|
||||
structure of the module, through a hash format called `dirhash`. Zilch
|
||||
implements this, and uses it to guide its VFS generating. Once you have a
|
||||
`go.sum` line, e.g. through `parse-go-sum-line`, passing it to
|
||||
`vfs-from-dirhash` will have it generate a VFS from a dirhash, plus the Go
|
||||
module proxy to fetch the zip:
|
||||
|
||||
[,scheme,line-comment=;]
|
||||
----
|
||||
(define sum-line (parse-go-sum-line "golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw="))
|
||||
;; #<go-sum golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=>
|
||||
|
||||
(define x-net-vfs (vfs-from-dirhash sum-line))
|
||||
;> (pre)fetching "module.zip" "https://proxy.golang.org/golang.org/x/net/@v/v0.41.0.zip"
|
||||
;> …
|
||||
;; #<zilch.vfs#<vfs>>
|
||||
|
||||
(store-path-realised
|
||||
(vfs-to-store x-net-vfs))
|
||||
;> [..building "/nix/store/vvs2xsgjpab4gwm530dvzcfzfcs5ak4m-zilchfile.drv"]
|
||||
;> …
|
||||
;; "/nix/store/r1p1y6bnmddmcrl2avsygp659d8iiih1-zilchfile/-"
|
||||
----
|
||||
|
||||
== Dynamically generating ``go-package``s from packages
|
||||
|
||||
Zilch supports generating `go-package` records dynamically, from `go.mod` files
|
||||
and `go.sum` files recursively. To do this, get a VFS for the primary module
|
||||
you want to work with, and any module you want to resolve automatically (Zilch
|
||||
will automatically crawl `go.sum` files to resolve dependencies, where needed.)
|
||||
|
||||
[,scheme,line-comment=;]
|
||||
----
|
||||
(define-values (root-module-path requirements)
|
||||
(collect-requirements-for-module x-net-vfs '()))
|
||||
|
||||
;> Collecting required modules
|
||||
;> [..building "/nix/store/s9fmg08x56px39hjkwlba2pra5g1abgr-go.mod.json.drv"]
|
||||
;> - found "golang.org/x/net" (requires 4 modules)
|
||||
;> - found "golang.org/x/crypto" (requires 4 modules)
|
||||
;> …
|
||||
;; root-module-path -> "golang.org/x/net"
|
||||
;; requirements
|
||||
;; -> #<mapping
|
||||
;; "golang.org/x/net" -> (#f . #<zilch.vfs#<vfs>>)
|
||||
;; "github.com/google/go-cmp" -> ("v0.6.0" . #<zilch.vfs#<vfs>>)
|
||||
;; "golang.org/x/crypto" -> ("v0.39.0" . #<zilch.vfs#<vfs>>)
|
||||
;; …
|
||||
;; "golang.org/x/mod" -> ("v0.25.0" . #<zilch.vfs#<vfs>>)>
|
||||
----
|
||||
|
||||
The resulting values are a comprehensive set of modules mapped to their final
|
||||
versions and the VFS of that module. This information can then be used to
|
||||
create a procedure that returns the `go-package` for any package in this set of
|
||||
modules, similar to how `go-stdlib-ref` works:
|
||||
|
||||
[,scheme,line-comment=;]
|
||||
----
|
||||
(define x-net-ref
|
||||
(collect-packages-from-requires requirements))
|
||||
|
||||
(x-net-ref "golang.org/x/net/idna")
|
||||
;; #<go-package idna (golang.org/x/net/idna)
|
||||
;; api: #<store path /1w32lfajzbs83ibvgnn2cx4zk33nmg3xb20dhra1b89zyg3xin4i
|
||||
;; (ca~ /nix/store/az0g6zjc33mg3kfacy7gnfhkbkps8m2y-golang.org_x_net_idna-src.drv!api)>
|
||||
;; code: #<store path /1r40gj99ycm404ak3f2rkhl4qhy19hx1dkhjpmp0rbbsgy2f4wj7
|
||||
;; (ca~ /nix/store/b93f07hkv2yxf80pdwbra2r760yxkh3r-go-golang.org_x_net_idna-code.drv!code)>
|
||||
;; deps: ("fmt" "bidirule" "bidi" "norm" "math" "strings" "unicode/utf8")>
|
||||
|
||||
(x-net-ref "golang.org/x/text/runes")
|
||||
;; #<go-package runes (golang.org/x/text/runes) …> <1>
|
||||
----
|
||||
<1> Even packages defined in a dependency can be looked up using this method.
|
||||
58
docs/modules/ROOT/pages/go/man.adoc
Normal file
58
docs/modules/ROOT/pages/go/man.adoc
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
= zilch-cli-go(1)
|
||||
Puck Meerburg
|
||||
v0.0.1
|
||||
:doctype: manpage
|
||||
:manmanual: ZILCH-CLI-GO
|
||||
:mansource: ZILCH
|
||||
:page-pagination: prev
|
||||
|
||||
== Name
|
||||
|
||||
zilch-cli-go - builds a Go module using Zilch
|
||||
|
||||
== Synopsis
|
||||
|
||||
*zilch-cli-go* [_OPTION_]... _PACKAGE_...
|
||||
|
||||
== Description
|
||||
|
||||
This command uses Zilch to build one or more Go packages entirely
|
||||
inside Nix, using content-addressed derivations.
|
||||
|
||||
A Go module is recognized by its `go.mod`, which contains information
|
||||
about the dependencies of any Go package. This versioning info is used
|
||||
identically to a normal `go build` call.
|
||||
|
||||
== Options
|
||||
|
||||
*-h*::
|
||||
*--help*::
|
||||
Print a help message
|
||||
|
||||
*-j* _COUNT_::
|
||||
*--max-jobs* _COUNT_::
|
||||
The maximum amount of builds to run. Defaults to the amount of cores.
|
||||
|
||||
*-v*::
|
||||
*--verbose*::
|
||||
Increase the verbosity configured in the Nix daemon. Can be specified
|
||||
multiple times.
|
||||
|
||||
*-L*::
|
||||
*--print-build-logs*::
|
||||
Print derivation logs as they come in.
|
||||
|
||||
*-m* _DIR_::
|
||||
*--module-dir* _DIR_::
|
||||
The directory to use as root module. All packages to be built must
|
||||
come from this module.
|
||||
|
||||
*-r* _DIR_::
|
||||
*--replace* _DIR_::
|
||||
Replace a module from the `go.mod` of the root module with
|
||||
this directory, based on the name of the replaced module's `go.mod`.
|
||||
Can be specified multiple times.
|
||||
|
||||
*--debug*::
|
||||
Crash on the first error, rather than continuing to build the next
|
||||
package.
|
||||
49
docs/modules/ROOT/pages/go/usage.adoc
Normal file
49
docs/modules/ROOT/pages/go/usage.adoc
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
= Usage
|
||||
:page-pagination: next
|
||||
|
||||
Zilch supports compiling any Go code incrementally using `zilch-cli-go`, with
|
||||
no special configuration needed on the part of the Go module being built:
|
||||
|
||||
[,console]
|
||||
----
|
||||
$ git clone https://github.com/tailscale/tailscale
|
||||
|
||||
$ zilch-cli-go --module-dir tailscale/ <1>
|
||||
tailscale.com/client/tailscale/example/servetls /nix/store/727ci6sssga0pbi4aqb08vnhfvmh9nl0-tailscale.com_client_tailscale_example_servetls
|
||||
tailscale.com/cmd/addlicense /nix/store/ckc5bx4s0c29sancd05asvhz0dy43xj7-tailscale.com_cmd_addlicense
|
||||
…
|
||||
tailscale.com/cmd/xdpderper /nix/store/8wn9w91mjipw1dzks4p3kcmkwcn18jp7-tailscale.com_cmd_xdpderper
|
||||
|
||||
$ zilch-cli-go --module-dir tailscale/ \
|
||||
> tailscale.com/cmd/tailscaled <2>
|
||||
tailscale.com/cmd/tailscaled /nix/store/hqrm0f8sd8sx54am921na25w8za67p3m-tailscale.com_cmd_tailscaled
|
||||
----
|
||||
<1> Building all binary packages in a module
|
||||
<2> Building a specifically targeted package
|
||||
|
||||
Right now, the daemon has to be able to build `x86_64-linux` derivations,
|
||||
and `zilch-cli-go` will only output statically linked amd64 binaries as well.
|
||||
|
||||
|
||||
While running, all processing of the Go module, its dependencies, etc, are done
|
||||
inside of Nix. Once all dependencies have been resolved, a series of Nix
|
||||
derivations will be used to then build the requested packages. If any source
|
||||
file's changes do not affect the way dependent packages use it, those packages
|
||||
will not need rebuilding; only the final result will have to be re-linked.
|
||||
|
||||
== Replacing dependencies
|
||||
As part of Zilch, it's also possible to quickly build a module with one of its
|
||||
(transitive) dependencies replaced. This keeps the same guarantees as before:
|
||||
Any changes made that do not involve the output changing will only need
|
||||
relinking of the resulting binary. This is also very simple:
|
||||
|
||||
[,console]
|
||||
----
|
||||
$ git clone https://go.googlesource.com/net x-net # golang.org/x/net
|
||||
$ zilch-cli-go --module-dir tailscale --replace x-net/ tailscale.com/cmd/tailscaled
|
||||
…
|
||||
tailscale.com/cmd/tailscaled /nix/store/hqrm0f8sd8sx54am921na25w8za67p3m-tailscale.com_cmd_tailscaled
|
||||
----
|
||||
|
||||
After editing a file any of the replaced dependencies, if this package is used
|
||||
in the final build, Zilch will apply early-cutoff where possible.
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
;; Defines the baseline definitions needed to compile Go code using Zilch.
|
||||
;; Defines the functions needed to compile Go code using Zilch.
|
||||
;;
|
||||
;; To make incremental builds work, this library uses up to four distinct
|
||||
;; To make incremental builds possible, this library uses up to four distinct
|
||||
;; outputs:
|
||||
;;
|
||||
;; - `api` is any `.a` file needed for compiling any other Go code that depends
|
||||
|
|
@ -42,14 +42,15 @@
|
|||
(define %goarch (make-parameter "amd64"))
|
||||
|
||||
;; The Go toolchain to use to compile this all.
|
||||
(define go-toolchain (cdr (assoc "out" (nixpkgs "go_1_23"))))
|
||||
(define go-toolchain (cdr (assoc "out" (nixpkgs "go_latest"))))
|
||||
|
||||
;; Builds an importcfg file. This file describes the mapping of both
|
||||
;; packages to their api, and the mapping of package name as used in `import`
|
||||
;; to the actual package names (e.g in case of `replace`.)
|
||||
;;
|
||||
;; - `++packagefiles++` is a alist of package name to .a file.
|
||||
;; - `++importmap++` is an alist of package name to actual package name.
|
||||
;; - `++packagefiles++` is an alist mapping a package name to .a file.
|
||||
;; - `++importmap++` is an alist mapping a package name to actual package name.
|
||||
;; This is primarily used by Go's standard library to vendor external packages.
|
||||
(define (build-importcfg packagefiles importmap)
|
||||
(call-with-port (open-output-string)
|
||||
(lambda (outstr)
|
||||
|
|
@ -72,11 +73,12 @@
|
|||
importmap)
|
||||
(get-output-string outstr))))
|
||||
|
||||
;; Builds an embedcfg file, which maps from the pattern used in `go:embed`
|
||||
;; to a list of files, as well as a filename to on-disk file mapping.
|
||||
;; Builds an `embedcfg` file, which maps the pattern used in `//go:embed` directives
|
||||
;; to a list of files that need to be embedded, and a mapping of filename to on-disk
|
||||
;; file path.
|
||||
;;
|
||||
;; - `++patterns++` is an alist of the pattern used to match files (e.g. `++foo/++`, or `++a.*++`) to a list of filenames.
|
||||
;; - `++files++` is an alist of file name to actual path.
|
||||
;; - `patterns` is an alist of the pattern used to match files (e.g. `foo/`, or `a.*`) to a list of filenames.
|
||||
;; - `files` is an alist mapping the file name in `patterns` to an on-disk path.
|
||||
(define (build-embedcfg patterns files)
|
||||
(call-with-port (open-output-string)
|
||||
(lambda (outstr)
|
||||
|
|
@ -88,6 +90,7 @@
|
|||
(get-output-string outstr))))
|
||||
|
||||
;; Clean up the package name to use in drv names.
|
||||
;; Replaces special characters seen in package names to underscores.
|
||||
(define (rewrite-package-name name)
|
||||
(set! name (string-copy name))
|
||||
(do ((x 0 (+ x 1)))
|
||||
|
|
@ -100,22 +103,28 @@
|
|||
;; An empty go_asm.h file used when generating symabis.
|
||||
(define empty-asmhdr (zdir `(("go_asm.h" . ,(zfile "")))))
|
||||
|
||||
;; The environment to append to the build environment for Go.
|
||||
;; The environment to append to the build environment for any Go derivation.
|
||||
(define (env-for-goarch)
|
||||
`(("GOARCH" . ,(%goarch))))
|
||||
|
||||
;; Extra defines to add to `++go tool asm++` uses.
|
||||
;; Extra arguments to add to uses of `++go tool asm++`, to set the expected preprocessor
|
||||
;; variables.
|
||||
(define (defines-for-goarch)
|
||||
`(
|
||||
"-D" "GOOS_linux"
|
||||
"-D" ,(string-append "GOARCH_" (%goarch))
|
||||
,@(if (string=? (%goarch) "amd64") '("-D" "GOAMD64_v1") '())))
|
||||
|
||||
;; Returns an alist of three store paths.
|
||||
;; Calls `go tool compile` with the provided arguments.
|
||||
;;
|
||||
;; - `++api++` containing the compiler's output, used when compiling
|
||||
;; - `++code++` contains the compiled code, used during linking only.
|
||||
;; - `++asmhdr++` contains the headers needed for any assembly code inside this package.
|
||||
;; - `std` must be `#t` when compiling the standard library, and `#f` otherwise.
|
||||
;; - `package-name` is the (zexp) string with the name of the package.
|
||||
;; - `importcfg` must be set, and is a zexp path to a file containing the import map.
|
||||
;; - `symabis` and `embeds` (optionally) point to a file containing their respective configuration.
|
||||
;; - `files` is an alist of all files to compile, mapping filename to on-disk path.
|
||||
;; The filenames are used in `-trimpath`, providing for better runtime traces and error messages.
|
||||
;;
|
||||
;; Returns an alist containing store paths for `api`, `code`, and `asmhdr` outputs.
|
||||
(define (go-compile std package-name importcfg symabis embeds files)
|
||||
(define args
|
||||
#~(
|
||||
|
|
@ -133,13 +142,14 @@
|
|||
"-trimpath" ,(apply string-append (map (lambda (f) (string-append (cdr f) "=>" package-name "/" (car f) ";")) #$files))
|
||||
. ,(map cdr #$files)))
|
||||
|
||||
(store-path-for-ca-drv*
|
||||
(store-path-for-ca-drv
|
||||
(string-append (rewrite-package-name package-name) "-src")
|
||||
"x86_64-linux"
|
||||
#~(,(string-append #$go-toolchain "/bin/go") "tool" "compile" . #$args)
|
||||
(env-for-goarch) '("api" "code" "asmhdr")))
|
||||
|
||||
;; Returns a store path containing the symabi for the assembly files provided.
|
||||
;; Calls `go tool asm -gensymabis`, returning a store path containing the symabi for the assembly files provided.
|
||||
;; `include-path` can be set to add a single path to the include path.
|
||||
(define (go-generate-symabi package-name include-path files)
|
||||
(define args
|
||||
#~(
|
||||
|
|
@ -152,15 +162,17 @@
|
|||
,@(if include-path (list "-I" #$include-path) '())
|
||||
. #$files))
|
||||
|
||||
(cdar (store-path-for-ca-drv*
|
||||
(cdar (store-path-for-ca-drv
|
||||
(string-append (rewrite-package-name package-name) "-asm-symabis")
|
||||
"x86_64-linux"
|
||||
#~(,(string-append #$go-toolchain "/bin/go") "tool" "asm" . #$args)
|
||||
(env-for-goarch) '("symabi"))))
|
||||
|
||||
;; Returns a store path containing the `++code++` of the provided assembly
|
||||
;; files. Assembly files have no `api`, and cannot be directly interacted
|
||||
;; with from other packages.
|
||||
;; Returns a store path containing the `code` of the provided assembly
|
||||
;; files. Assembly files have no `api`, and are used inside the package they're defined in.
|
||||
;;
|
||||
;; - `include-path` and `include-path2` are both added to `-I` arguments if set.
|
||||
;; - `files`, as with `go-compile`, is an alist of file name to on-disk path.
|
||||
(define (go-compile-assembly package-name include-path include-path2 files)
|
||||
(define args
|
||||
#~(
|
||||
|
|
@ -173,7 +185,7 @@
|
|||
"-trimpath" ,(apply string-append (map (lambda (f) (string-append (cdr f) "=>" package-name "/" (car f) ";")) #$files))
|
||||
. ,(map cdr #$files)))
|
||||
|
||||
(cdar (store-path-for-ca-drv*
|
||||
(cdar (store-path-for-ca-drv
|
||||
(string-append (rewrite-package-name package-name) "-asm")
|
||||
"x86_64-linux"
|
||||
#~(,(string-append #$go-toolchain "/bin/go") "tool" "asm" . #$args)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
;; Helpers for fetching files from the Go module proxy, slightly impurely.
|
||||
;; Helpers for fetching files from the Go module and checksum proxy.
|
||||
(define-library (zilch lang go fetch)
|
||||
(import
|
||||
(scheme base) (scheme write) (scheme read) (scheme file) (scheme char)
|
||||
|
|
@ -12,6 +12,10 @@
|
|||
(begin
|
||||
(define fetch-cache-file (string-append (get-environment-variable "HOME") "/.cache/zilch-fetch.scm"))
|
||||
(define fetch-cache (if (file-exists? fetch-cache-file) (call-with-input-file fetch-cache-file read) '()))
|
||||
|
||||
;; Creates a store path that runs `builtin:fetchurl` with specified `name` and `url`.
|
||||
;; If the hash isn't known in the user-wide cache at `~/.cache/zilch-fetch.scm`, shells
|
||||
;; out to `nix-prefetch-url` to fetch the path and calculate its hash.
|
||||
(define (fetch-with-known-url name url)
|
||||
(define cache-entry (assoc url fetch-cache))
|
||||
(define hash (if cache-entry
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
;; Higher-level utilities to write Go compilation instructions inside of Zilch.
|
||||
;; These act on a `<go-package>` record, which is generated by `go-package-compile`,
|
||||
;; and the final program linked together by `go-package-link`.
|
||||
(define-library (zilch lang go)
|
||||
(import
|
||||
(scheme base) (scheme write) (scheme process-context) (scheme lazy)
|
||||
|
|
@ -10,6 +13,7 @@
|
|||
(zilch lang go core))
|
||||
|
||||
(export
|
||||
<go-package>
|
||||
make-go-package go-package?
|
||||
go-package-name go-package-import-path
|
||||
go-package-api go-package-code go-package-dependencies
|
||||
|
|
@ -19,12 +23,12 @@
|
|||
|
||||
|
||||
(begin
|
||||
;; A go package consists of a few separate `++(zilch magic)++` store paths.
|
||||
;; The `++name++` is the package name as compiled, and `++import-path++` is a nicer
|
||||
;; package name for "main" packages. The `++go-package-api++` is a store path consisting
|
||||
;; of a `++.a++` containing the output of the compiler's `++__.PKGDEF++` only, which
|
||||
;; A go package consists of a few separate `(zilch magic)` store paths.
|
||||
;; The `name` is the package name as compiled, and `import-path` is a nicer
|
||||
;; package name for "main" packages. The `go-package-api` is a store path consisting
|
||||
;; of a `.a` containing the output of the compiler's `__.PKGDEF` only, which
|
||||
;; contains the exported types and functions, along with a slight amount of LTO and
|
||||
;; inlining metadata. the `++go-package-code++` store path contains the actual assembly
|
||||
;; inlining metadata. the `go-package-code` store path contains the actual assembly
|
||||
;; of the package.
|
||||
(define-record-type <go-package>
|
||||
(make-go-package name import-path api code dependencies)
|
||||
|
|
@ -44,7 +48,8 @@
|
|||
(go-package-code pkg)
|
||||
(map go-package-name (go-package-dependencies pkg))))
|
||||
|
||||
;; Recursively walk over the dependencies of a `++go-package++`, prepending to the `++vals++` list.
|
||||
;; Recursively walk over the dependencies of a `<go-package>`, prepending to the `vals` list,
|
||||
;; and returning the resulting list.
|
||||
(define (go-dependency-closure package vals)
|
||||
(unless (member package vals)
|
||||
(set! vals (cons package vals))
|
||||
|
|
@ -54,9 +59,16 @@
|
|||
(go-package-dependencies package)))
|
||||
vals)
|
||||
|
||||
;; `(go-package-compile name deps source-files)`
|
||||
;; or `(go-package-compile name path deps source-files assembly-files assembly-includes embed-filenames embed-patterns)`
|
||||
;; Build a Zilch-defined Go package of one store path as source code, and a list of dependencies.
|
||||
;;
|
||||
;; - `name` is the full name of the package, or `main` if the package is the main package.
|
||||
;; - `path` is the full name of the package (e.g. `example.com/foo/bar`).
|
||||
;; - `deps` is a list of `<go-package>` dependencies.
|
||||
;; - `source-files` is a (zexp) alist of file name to their location on disk (or store path).
|
||||
;; - `assembly-files` is identical to `source-files`, but for `.s` files.
|
||||
;; - `assembly-includes` is either a single on-disk path pointing to a directory, or an alist of file name to on-disk location for header files that should be in scope for `#include` in assembly files.
|
||||
;; - `embed-patterns` is an alist of Go embed patterns to the filenames they contain;
|
||||
;; - `embed-filenames` is an alist of filenames used in embed patterns to their on-disk location.
|
||||
(define go-package-compile
|
||||
(case-lambda
|
||||
((name deps source-files) (go-package-compile name name deps source-files '() '() '() '()))
|
||||
|
|
@ -99,26 +111,24 @@
|
|||
; NOTE: .go has to be compiled in one go; but .s is compiled one file at a time.
|
||||
(define all-code (cons (cdr (assoc "code" compiled-go)) compiled-assembly))
|
||||
|
||||
; (printf " -> (store-path-for-ca-drv* meow meow ~S ~S meow)\n" all-code (env-for-goarch))
|
||||
; Use `go tool pack` to merge the code together.
|
||||
(define merged-code
|
||||
(if (length assembly-files)
|
||||
(cdar (store-path-for-ca-drv*
|
||||
(cdar (store-path-for-ca-drv
|
||||
(string-append "go-" (rewrite-package-name path) "-code") "x86_64-linux"
|
||||
#~(,(string-append #$go-toolchain "/bin/go") "tool" "pack" "c" ,(make-placeholder "code") . #$all-code)
|
||||
(env-for-goarch)
|
||||
'("code")))
|
||||
(cdr (assoc "code" compiled-go))))
|
||||
|
||||
; (printf " -> (make-go-package ~S ~S ~S ~S ~S)\n" name path (cdr (assoc "api" compiled-go)) merged-code deps)
|
||||
(make-go-package name path (cdr (assoc "api" compiled-go)) merged-code deps))))
|
||||
|
||||
;; Link a `++go-package++` into a binary that can be (statically) executed.
|
||||
;; Link a `<go-package>` into a binary that can be executed.
|
||||
(define (go-package-link pkg)
|
||||
(define code-importcfg
|
||||
(zfile #~,(build-importcfg #$(map (lambda (pkg) (cons (go-package-import-path pkg) (go-package-code pkg))) (go-dependency-closure pkg '())) '())))
|
||||
|
||||
(cdar (store-path-for-ca-drv* (rewrite-package-name (go-package-import-path pkg)) "x86_64-linux"
|
||||
(cdar (store-path-for-ca-drv (rewrite-package-name (go-package-import-path pkg)) "x86_64-linux"
|
||||
#~(,(string-append #$go-toolchain "/bin/go") "tool" "link" "-buildid" ,(string-append "zilch out=" (make-placeholder "out")) "-importcfg" #$code-importcfg "-o" ,(make-placeholder "out") #$(go-package-code pkg)) (env-for-goarch) '("out"))))))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
;; Processes go module files.
|
||||
;; Resolves dependencies for Go modules and processes them into `<go-package>` files
|
||||
;; that can then be used to compile existing Go modules with Zilch.
|
||||
(define-library (zilch lang go mod)
|
||||
(import
|
||||
(scheme base) (scheme write) (scheme read) (scheme file) (scheme process-context) (scheme lazy) (scheme case-lambda)
|
||||
|
|
@ -33,15 +34,16 @@
|
|||
(define (read-go-mod mod-file)
|
||||
(call-with-port
|
||||
;; TODO(puck): don't use /bin/sh here.
|
||||
(store-path-open (cdar (store-path-for-ca-drv* "go.mod.json" "x86_64-linux" #~("/bin/sh" "-c" ,(string-append #$go-toolchain "/bin/go mod edit -json " #$mod-file " > $out")) '() '("out"))))
|
||||
(store-path-open (cdar (store-path-for-ca-drv "go.mod.json" "x86_64-linux" #~("/bin/sh" "-c" ,(string-append #$go-toolchain "/bin/go mod edit -json " #$mod-file " > $out")) '() '("out"))))
|
||||
(lambda (p) (json-read p))))
|
||||
|
||||
(define (vector-get-kv-value key vec)
|
||||
(vector-any (lambda (v) (and (string=? (car v) key) (cdr v))) vec))
|
||||
|
||||
;; Reads in the module rooted by the vfs, and resolves its requirements list.
|
||||
;; Reads in the module rooted by the vfs in `vfs`, and finds all its requirements.
|
||||
;; This returns two values: the name of the root module, and a mapping of
|
||||
;; module name to a pair of its version and the vfs.
|
||||
;; `replaces` is a list of ``<vfs>``es containing modules that should be prioritized as dependencies.
|
||||
(define (collect-requirements-for-module vfs replaces)
|
||||
(define sum-lines '())
|
||||
(define (parse-sumfile go-sum)
|
||||
|
|
@ -139,7 +141,7 @@
|
|||
(values root-path-name collected-requires))
|
||||
|
||||
;; Processes a mapping of module name to a pair of version and vfs, and
|
||||
;; returns a procedure that takes a package name and returns its go-package.
|
||||
;; returns a procedure that takes a package name and returns a `<go-package>` representing it.
|
||||
(define (collect-packages-from-requires collected-requires)
|
||||
(define (process-package vfs last-part full-path pairs headers)
|
||||
(define name (cdr (assoc "name" pairs)))
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
;; Routines to locate Go packages in a `<vfs>`.
|
||||
(define-library (zilch lang go package)
|
||||
(import
|
||||
(scheme base)
|
||||
|
|
@ -18,9 +19,9 @@
|
|||
(map go-stdlib-ref '("encoding/json" "fmt" "go/build" "io" "io/fs" "os" "path" "path/filepath" "sort" "strings" "time"))
|
||||
(list (cons "main.go" (zfile (foreign-value "parser_source" nonnull-c-string)))))))
|
||||
|
||||
;; Uses IFD to find each Go package defined inside this virtual filesystem,
|
||||
;; returning a vector containing pairs, mapping each directory to the
|
||||
;; package defined within.
|
||||
;; Finds each Go package defined inside this virtual filesystem,
|
||||
;; and returns a vector containing pairs, mapping the name of each directory to the
|
||||
;; a vector-based structure describing the package defined in said directory.
|
||||
(define (find-packages-inside-vfs vfs)
|
||||
(define input
|
||||
#~,(call-with-port
|
||||
|
|
@ -29,6 +30,6 @@
|
|||
(json-write (vector (cons "GOARCH" (%goarch)) (cons "GOOS" "linux") (cons "files" #$(vfs-to-json (vfs-filter-for-go-package vfs)))) bv)
|
||||
(get-output-bytevector bv))))
|
||||
(define input-file (zfile input))
|
||||
(define store-path (cdar (store-path-for-ca-drv* "find-packages" "x86_64-linux" #~(#$go-import-parser #$input-file) '() '("out"))))
|
||||
(define store-path (cdar (store-path-for-ca-drv "find-packages" "x86_64-linux" #~(#$go-import-parser #$input-file) '() '("out"))))
|
||||
(call-with-port (store-path-open store-path)
|
||||
(lambda (p) (json-read p))))))
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
;; Handles the Go standard library.
|
||||
(define-library (zilch lang go stdlib)
|
||||
(import
|
||||
(scheme base) (scheme file) (scheme write) (scheme process-context) (scheme lazy)
|
||||
|
|
@ -20,12 +21,12 @@
|
|||
out
|
||||
(read-all-objects port (cons (json-read port) out))))
|
||||
|
||||
;; Runs `++go list++` (thru `++/bin/sh++`) and reads (IFD) the output to fetch the metadata of the Go standard library and commands.
|
||||
;; Runs `go list` (thru `/bin/sh`) and reads the output to fetch the metadata of the Go standard library and commands.
|
||||
(define stdlib-objects
|
||||
(map vector->list
|
||||
(call-with-port
|
||||
(store-path-open
|
||||
(cdar (store-path-for-ca-drv*
|
||||
(cdar (store-path-for-ca-drv
|
||||
"stdenv"
|
||||
"x86_64-linux"
|
||||
#~("/bin/sh" "-c" ,(string-append "GOCACHE=$TMPDIR/go-cache " #$go-toolchain "/bin/go list -json -deps std cmd > $out"))
|
||||
|
|
@ -37,7 +38,7 @@
|
|||
(if res (cdr res) '()))
|
||||
|
||||
;; Extract everything until the first space.
|
||||
;; Space characters are illegal in Go package names, and `++go list -json std++`
|
||||
;; Space characters are illegal in Go package names, and `go list -json std`
|
||||
;; uses it to disambiguate multiple versions of some internal packages.
|
||||
(define (strip-space-bits name)
|
||||
(do
|
||||
|
|
@ -50,7 +51,7 @@
|
|||
(substring name 0 x)))))
|
||||
|
||||
;; Tail-recursively remove any packages that, if ignoring the postfixed origin
|
||||
;; (e.g. `++unsafe [cmd/compile]++`), match either `++unsafe++` or `++builtin++`;
|
||||
;; (e.g. `unsafe [cmd/compile]`), match either `unsafe` or `builtin`;
|
||||
;; these have no source code and are compiler-internal.
|
||||
(define (remove-builtin-packages pkgs)
|
||||
(if (eq? pkgs '())
|
||||
|
|
@ -72,8 +73,10 @@
|
|||
(cons (car lst) (filter condition (cdr lst)))
|
||||
(filter condition (cdr lst)))))
|
||||
|
||||
;; Helper that parses the JSON returned by `++go list -json -deps++` and builds a `++go-package++` record.
|
||||
;; This is distinct from `go-package-compile` because of format differences,
|
||||
;; Helper that parses the JSON returned by `go list -json -deps` and builds a `go-package` record.
|
||||
;; This is distinct from `go-package-compile` because of format differences.
|
||||
;;
|
||||
;; TODO(puck): Is this still the case?
|
||||
(define (make-stdlib-inner meta)
|
||||
(define files (assoc-or-empty "GoFiles" meta)) ; .go files
|
||||
(define sfiles (assoc-or-empty "SFiles" meta)) ; .s files
|
||||
|
|
@ -140,7 +143,7 @@
|
|||
; Use `go tool pack` to merge the code together.
|
||||
(define merged-code
|
||||
(cdar
|
||||
(store-path-for-ca-drv*
|
||||
(store-path-for-ca-drv
|
||||
(string-append "go-" (rewrite-package-name name) "-code") "x86_64-linux"
|
||||
#~(,(string-append #$go-toolchain "/bin/go") "tool" "pack" "c" ,(make-placeholder "code") . #$all-code)
|
||||
(env-for-goarch)
|
||||
|
|
@ -148,11 +151,11 @@
|
|||
|
||||
(make-go-package name import-path go-api merged-code (map go-stdlib-ref (remove-builtin-packages imports))))
|
||||
|
||||
; Each entry is a list (name metadata (api code)).
|
||||
; Use `++delay++` to resolve the DAG lazily on use.
|
||||
; Each entry is a list of (name metadata (api code)).
|
||||
; Use `delay` to resolve the DAG lazily on use.
|
||||
(define stdlib-data (map (lambda (v) (list (cdr (assoc "ImportPath" v)) v (delay (make-stdlib-inner v)))) stdlib-objects))
|
||||
|
||||
;; Finds any package contained within Go's standard library.
|
||||
;; Finds any package contained within Go's standard library, and returns a `<go-package>` for the package.
|
||||
(define (go-stdlib-ref name)
|
||||
(define entry (assoc name stdlib-data))
|
||||
(unless entry (error (string-append "Could not find package " name " in stdlib")))
|
||||
|
|
|
|||
|
|
@ -12,10 +12,14 @@
|
|||
(chicken foreign))
|
||||
|
||||
(export
|
||||
parse-go-sum-line parse-go-sum-file go-sum-line? go-sum-module go-sum-version go-sum-path go-sum-hash)
|
||||
<go-sum-line> go-sum-line?
|
||||
go-sum-module go-sum-version go-sum-path go-sum-hash
|
||||
|
||||
parse-go-sum-line parse-go-sum-file)
|
||||
|
||||
(begin
|
||||
;; Contains the values from a single line from a `go.sum` file.
|
||||
;; `go-sum-hash` is a bytevector containing the base64-decoded hash.
|
||||
(define-record-type <go-sum-line>
|
||||
(make-go-sum-line module version path hash)
|
||||
go-sum-line?
|
||||
|
|
@ -55,7 +59,7 @@
|
|||
(set! version (string-copy version 0 path-index)))
|
||||
(make-go-sum-line module-path version path (base64->bytevector (string-copy hash 3))))
|
||||
|
||||
;; Parses all the `go.sum` lines from `port`.
|
||||
;; Returns a list of all ``<go-sum-line>``s read from `port`.
|
||||
(define (parse-go-sum-file port)
|
||||
(do ((parsed '())
|
||||
(line "" (read-line port)))
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
;; Procedures to deal with Go's semantic versions.
|
||||
;; Procedures to deal with Go's format of semantic versions.
|
||||
(define-library (zilch lang go version)
|
||||
(import
|
||||
(scheme base) (srfi 152))
|
||||
|
|
|
|||
|
|
@ -44,9 +44,8 @@
|
|||
(if (char-upper-case? ch) (set! out (string-append out (string #\! (char-downcase ch)))) (set! out (string-append out (string ch))))) name)
|
||||
out)
|
||||
|
||||
;; Reads a dirhash from a `go.sum` line. This prefetches the module from
|
||||
;; the go module proxy, and then generates the dirhash without unpacking
|
||||
;; said module file.
|
||||
;; Uses the hash in a `<go-sum-line>` to return a store path containing a list of the hashes of each file,
|
||||
;; following the https://pkg.go.dev/golang.org/x/mod/sumdb/dirhash[`dirhash`] format used by the `go.sum` files.
|
||||
(define (fetch-dirhash-for-sum sum-line)
|
||||
(when (go-sum-path sum-line) (error "go.sum line is invalid for fetch-dirhash-for-sum" sum-line))
|
||||
(define url (string-append "https://proxy.golang.org/" (rewrite-go-package-name-for-url (go-sum-module sum-line)) "/@v/" (go-sum-version sum-line) ".zip"))
|
||||
|
|
@ -54,7 +53,8 @@
|
|||
(store-path-for-fod "module" "x86_64-linux" #~(#$(force dirhash-generator)) #~(("src" . #$known)) "sha256" (go-sum-hash sum-line) #f))
|
||||
|
||||
;; Generates a full VFS structure from a module as described by a `go.sum`
|
||||
;; line.
|
||||
;; line. Uses `fetch-dirhash-for-sum` to generate the dirhash, then creates FODs
|
||||
;; for each file based on said hash; this is then returned as a `<vfs>` structure.
|
||||
(define (vfs-from-dirhash sum-line)
|
||||
(define dirhash-file (fetch-dirhash-for-sum sum-line))
|
||||
(define url (string-append "https://proxy.golang.org/" (rewrite-go-package-name-for-url (go-sum-module sum-line)) "/@v/" (go-sum-version sum-line) ".zip"))
|
||||
|
|
@ -106,14 +106,17 @@
|
|||
((= i 0) #f)
|
||||
(else (extract-extension name (- i 1)))))
|
||||
|
||||
;; Returns a VFS, filtered down to only contain the contents of files that
|
||||
;; will be read during the processing of Go packages.
|
||||
;; Returns a subset of `vfs`, filtered down to only contain the contents of files that
|
||||
;; Go cares about.
|
||||
(define (vfs-filter-for-go-package vfs)
|
||||
(vfs-dir-filter vfs
|
||||
(lambda (dir fname contents)
|
||||
(define extension (extract-extension fname (- (string-length fname) 1)))
|
||||
(member extension good-extensions))))
|
||||
|
||||
;; Generates a representation of the `vfs`, processed to be turned into JSON.
|
||||
;; Primarily used by `(zilch lang go package)` to allow extracting the necessary
|
||||
;; information of each package from a module's vfs.
|
||||
(define (vfs-to-json vfs)
|
||||
(mapping-map->list
|
||||
(lambda (k v) (list (car k) (cdr k) (if (eq? v 'directory) "" v)))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue