puck/zilch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Clean up documentation

Browse source
This commit is contained in:
puck 2024-10-04 01:21:07 +00:00
parent 55a1efa08f
commit 83d41ef778
16 changed files with 146 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

59
lang/go/src/core.sld
View file

@ -1,3 +1,18 @@
;; Defines the baseline definitions needed to compile Go code using Zilch.
;;
;; To make incremental builds work, this library uses up to four distinct
;; outputs:
;;
;; - `api` is any `.a` file needed for compiling any other Go code that depends
;; on said package, and is usually an archive containing a file named
;; `__.PKGDEF`.
;; - `code` is a `.a` file containing the actual binary code for the target
;; arch, and is used only during linking.
;; - `asmhdr` is a header file generated by the Go code, and used during
;; assembly compilation only.
;; - `symabi` is a text file contaning the functions defined by assembly, and
;; their ABI, which is used while compiling the Go code that contains the
;; stubs for any assembly code.
(define-library (zilch lang go core)
(import
(scheme base) (scheme write) (scheme process-context) (scheme lazy)
@ -22,15 +37,19 @@
(begin
;; The architecture to target the Go code at.
;; Equivalent to `GOARCH`.
(define %goarch (make-parameter "amd64"))
; Import the existing Go from nixpkgs.
;; The Go toolchain to use to compile this all.
(define go-toolchain (cdr (assoc "out" (nixpkgs "go_1_23"))))
;; Builds an importcfg file, containing an alist of packages to .a files,
;; and an alist of package names to actual package names.
;; `++packagefiles++` is a alist of package name to .a file (api type),
;; `++importmap++` is an alist of package name to actual package name (used in cases of e.g. replace)
;; Builds an importcfg file. This file describes the mapping of both
;; packages to their api, and the mapping of package name as used in `import`
;; to the actual package names (e.g in case of `replace`.)
;;
;; - `++packagefiles++` is a alist of package name to .a file.
;; - `++importmap++` is an alist of package name to actual package name.
(define (build-importcfg packagefiles importmap)
(call-with-port (open-output-string)
(lambda (outstr)
@ -53,8 +72,11 @@
importmap)
(get-output-string outstr))))
;; `++patterns++` is an alist of the pattern used to match files (e.g. `++foo/++`, or `++a.*++`) to a list of filenames.
;; `++files++` is an alist of file name to actual path.
;; Builds an embedcfg file, which maps from the pattern used in `go:embed`
;; to a list of files, as well as a filename to on-disk file mapping.
;;
;; - `++patterns++` is an alist of the pattern used to match files (e.g. `++foo/++`, or `++a.*++`) to a list of filenames.
;; - `++files++` is an alist of file name to actual path.
(define (build-embedcfg patterns files)
(call-with-port (open-output-string)
(lambda (outstr)
@ -65,7 +87,7 @@
outstr)
(get-output-string outstr))))
; Clean up the package name to use in drv names.
;; Clean up the package name to use in drv names.
(define (rewrite-package-name name)
(set! name (string-copy name))
(do ((x 0 (+ x 1)))
@ -75,10 +97,10 @@
(when (char=? (string-ref name x) #\space) (string-set! name x #\_))
(when (char=? (string-ref name x) #\]) (string-set! name x #\_))))
;; An empty go_asm.h file used to generate symabis.
;; An empty go_asm.h file used when generating symabis.
(define empty-asmhdr (zdir `(("go_asm.h" . ,(zfile "")))))
;; Environment to append to the build environment for Go.
;; The environment to append to the build environment for Go.
(define (env-for-goarch)
`(("GOARCH" . ,(%goarch))))
@ -89,16 +111,19 @@
"-D" ,(string-append "GOARCH_" (%goarch))
,@(if (string=? (%goarch) "amd64") '("-D" "GOAMD64_v1") '())))
;; Returns an alist of three store paths; `++api++` containing the compiler's output,
;; `++code++` containing the linkobj, and `++asmhdr++` containing the headers needed for assembly
;; code to properly use Go functions and variables.
;; Returns an alist of three store paths.
;;
;; - `++api++` containing the compiler's output, used when compiling
;; - `++code++` contains the compiled code, used during linking only.
;; - `++asmhdr++` contains the headers needed for any assembly code inside this package.
(define (go-compile std package-name importcfg symabis embeds files)
(define args
#~(
,@(if std '("-std") '())
#$@(if symabis `("-symabis" ,#$symabis) '())
#$@(if embeds `("-embedcfg" ,#$embeds) '())
"-buildid" "zilch go-compile" ; this goes into both code and __.PKGDEF, so can't be a reference to the code output, sadly
; this goes into both code and __.PKGDEF, so can't be a reference to the code output, sadly
"-buildid" "zilch go-compile"
"-p" #$package-name
"-o" ,(make-placeholder "api")
"-linkobj" ,(make-placeholder "code")
@ -114,7 +139,7 @@
#~(,(string-append #$go-toolchain "/bin/go") "tool" "compile" . #$args)
(env-for-goarch) '("api" "code" "asmhdr")))
;; Returns a store path containing the symabi for the files provided.
;; Returns a store path containing the symabi for the assembly files provided.
(define (go-generate-symabi package-name include-path files)
(define args
#~(
@ -133,7 +158,9 @@
#~(,(string-append #$go-toolchain "/bin/go") "tool" "asm" . #$args)
(env-for-goarch) '("symabi"))))
;; Returns a store path containing the `++code++` of the provided assembly files.
;; Returns a store path containing the `++code++` of the provided assembly
;; files. Assembly files have no `api`, and cannot be directly interacted
;; with from other packages.
(define (go-compile-assembly package-name include-path include-path2 files)
(define args
#~(

14
lang/go/src/fetch.sld
View file

@ -1,26 +1,30 @@
;; Helpers for fetching files from the Go module proxy, slightly impurely.
(define-library (zilch lang go fetch)
(import
(scheme base) (scheme write) (scheme read) (scheme file) (scheme char)
(zilch magic) (zilch zexpr)
(zilch nixpkgs)
(chicken format))
(chicken process-context) (chicken format) (chicken file))
(export
fetch-with-known-url rewrite-module-name-for-url)
(begin
(define fetch-cache (call-with-input-file "/home/.zilchcache" read))
(define fetch-cache-file (string-append (get-environment-variable "HOME") "/.cache/zilch-fetch.scm"))
(define fetch-cache (if (file-exists? fetch-cache-file) (call-with-input-file fetch-cache-file read) '()))
(define (fetch-with-known-url name url)
(define cache-entry (assoc url fetch-cache))
(define hash (if cache-entry
(cdr cache-entry)
(begin (printf "..fetching ~S ~S\n" name url)
(begin (fprintf (current-error-port) "(pre)fetching ~S ~S\n" name url)
(nix-prefetch-url name url #f))))
(unless cache-entry
(set! fetch-cache (cons (cons url hash) fetch-cache))
(call-with-output-file "/home/.zilchcache" (lambda (out) (write fetch-cache out))))
(store-path-for-fod name "builtin" '("builtin:fetchurl") `(("url" . ,url) ("outputHashMode" . "flat")) "sha256" hash #f))
(call-with-output-file fetch-cache-file (lambda (out) (write fetch-cache out))))
(store-path-for-fod name "builtin" '("builtin:fetchurl") `(("url" . ,url)) "sha256" hash #f))
;; Rewrites the module name to prefix all uppercase letters with an
;; exclamation mark instead, as required by the various Go webservices.
(define (rewrite-module-name-for-url name)
(define out "")
(string-for-each

11
lang/go/src/mod.sld
View file

@ -1,4 +1,4 @@
;; Processes go modules.
;; Processes go module files.
(define-library (zilch lang go mod)
(import
(scheme base) (scheme write) (scheme read) (scheme file) (scheme process-context) (scheme lazy) (scheme case-lambda)
@ -27,7 +27,8 @@
(cons (car lis) (filter proc (cdr lis))))
(else (filter proc (cdr lis)))))
;; Read a go.mod file. This returns a processed json object, like `go mod edit -json` outputs by default.
;; Read a go.mod file. This returns a processed json object, like
;; `go mod edit -json` outputs by default.
(define (read-go-mod mod-file)
(call-with-port
;; TODO(puck): don't use /bin/sh here.
@ -38,7 +39,8 @@
(vector-any (lambda (v) (and (string=? (car v) key) (cdr v))) vec))
;; Reads in the module rooted by the vfs, and resolves its requirements list.
;; This returns two values: the name of the root module, and a mapping of module name to a pair of its version and the vfs.
;; This returns two values: the name of the root module, and a mapping of
;; module name to a pair of its version and the vfs.
(define (collect-requirements-for-module vfs replaces)
(define sum-lines '())
(define (parse-sumfile go-sum)
@ -135,7 +137,8 @@
(tick)
(values root-path-name collected-requires))
;; Processes a mapping of module name to a pair of version and vfs, and returns a procedure that takes a package name and returns its go-package.
;; Processes a mapping of module name to a pair of version and vfs, and
;; returns a procedure that takes a package name and returns its go-package.
(define (collect-packages-from-requires collected-requires)
(define (process-package vfs last-part full-path pairs headers)
(define name (cdr (assoc "name" pairs)))

3
lang/go/src/package.sld
View file

@ -17,6 +17,9 @@
(map go-stdlib-ref '("encoding/json" "fmt" "go/build" "io" "io/fs" "os" "path" "path/filepath" "sort" "strings" "time"))
(list (cons "main.go" (zfile (foreign-value "parser_source" nonnull-c-string)))))))
;; Uses IFD to find each Go package defined inside this virtual filesystem,
;; returning a vector containing pairs, mapping each directory to the
;; package defined within.
(define (find-packages-inside-vfs vfs)
(define input
#~,(call-with-port

2
lang/go/src/stdlib.sld
View file

@ -152,7 +152,7 @@
; Use `++delay++` to resolve the DAG lazily on use.
(define stdlib-data (map (lambda (v) (list (cdr (assoc "ImportPath" v)) v (delay (make-stdlib-inner v)))) stdlib-objects))
;; Wrapper that forces evaluation of the promise fetching from a Go stdlib entry.
;; Finds any package contained within Go's standard library.
(define (go-stdlib-ref name)
(define entry (assoc name stdlib-data))
(unless entry (error (string-append "Could not find package " name " in stdlib")))

4
lang/go/src/sum.sld
View file

@ -1,3 +1,4 @@
;; Parses `go.sum` files.
(define-library (zilch lang go sum)
(import
(scheme base) (scheme write) (scheme read) (scheme file) (scheme process-context) (scheme lazy) (scheme case-lambda)
@ -14,6 +15,7 @@
parse-go-sum-line parse-go-sum-file go-sum-line? go-sum-module go-sum-version go-sum-path go-sum-hash)
(begin
;; Contains the values from a single line from a `go.sum` file.
(define-record-type <go-sum-line>
(make-go-sum-line module version path hash)
go-sum-line?
@ -35,6 +37,7 @@
((char=? (string-ref str index) char) index)
(else (string-find str (+ index 1) char))))
;; Parses a `go.sum` line, and returns a `<go-sum-line>`.
(define (parse-go-sum-line line)
(define version-space-index (string-find line 0 #\space))
(unless version-space-index (error "go.sum line contains no space characters"))
@ -52,6 +55,7 @@
(set! version (string-copy version 0 path-index)))
(make-go-sum-line module-path version path (base64->bytevector (string-copy hash 3))))
;; Parses all the `go.sum` lines from `port`.
(define (parse-go-sum-file port)
(do ((parsed '())
(line "" (read-line port)))

6
lang/go/src/version.sld
View file

@ -1,9 +1,13 @@
;; Procedures to deal with Go's semantic versions.
(define-library (zilch lang go version)
(import
(scheme base) (srfi 152))
(export parse-version version<?)
(begin
;; Returns a list containing five values parsed from the version string:
;; `(major minor patch prerelease build)`
(define (parse-version vstr)
(unless (char=? (string-ref vstr 0) #\v) (error "not a valid version" vstr))
(define first-period (string-index vstr (lambda (ch) (char=? ch #\.)) 1))
@ -17,6 +21,7 @@
(define build (and build-dash (string-copy vstr (+ build-dash 1))))
(list major minor patch prerelease build))
;; Returns `#t` if `left` is a smaller version than `right`.
(define (version<? left right)
(set! left (parse-version left))
(set! right (parse-version right))
@ -43,4 +48,3 @@
(and (list-ref left 3) (not (list-ref right 3)))
; or both have a prerelease and it's comparable
(and (list-ref left 3) (string<? (list-ref left 3) (list-ref right 3)))))))))))))

22
lang/go/src/vfs.sld
View file

@ -1,3 +1,10 @@
;; Contains procedures to work with a very simple virtual filesystem,
;; abstracting between local and in-store store paths.
;;
;; A VFS is defined as a vector containing pairs consisting of the directory's
;; name, with a forward slash prefixed and postfixed (e.g. `/` or `/foo/bar/`).
;; Each pair then contains another vector, mapping filename to any value that
;; can be used as a z-expression (e.g. `store-path-for-fod` or `zfile`).
(define-library (zilch lang go vfs)
(import
(scheme base) (scheme write) (scheme read) (scheme file) (scheme process-context) (scheme lazy) (scheme case-lambda)
@ -42,6 +49,8 @@
(if (char-upper-case? ch) (set! out (string-append out (string #\! (char-downcase ch)))) (set! out (string-append out (string ch))))) name)
out)
;; Takes a VFS and writes its directory structure into the Nix store,
;; returning a zdir describing the root directory.
(define (vfs-to-store vfs)
(define dirmap (mapping (make-default-comparator)))
(vector-for-each
@ -62,13 +71,17 @@
(map (lambda (k) (cons (car k) (translate-dir (cdr k)))) dirs))))
(translate-dir "/"))
;; Reads a dirhash from a `go.sum` line. This prefetches the module from
;; the go module proxy, and then generates the dirhash without unpacking
;; said module file.
(define (fetch-dirhash-for-sum sum-line)
(when (go-sum-path sum-line) (error "go.sum line is invalid for fetch-dirhash-for-sum" sum-line))
(define url (string-append "https://proxy.golang.org/" (rewrite-name (go-sum-module sum-line)) "/@v/" (go-sum-version sum-line) ".zip"))
(define known (fetch-with-known-url "module.zip" url))
(store-path-for-fod "module" "x86_64-linux" #~(#$dirhash-generator) #~(("src" . #$known)) "sha256" (go-sum-hash sum-line) #f))
;; Generates a full VFS structure from a module as described by a `go.sum`
;; line.
(define (vfs-from-dirhash sum-line)
(define dirhash-file (fetch-dirhash-for-sum sum-line))
(define url (string-append "https://proxy.golang.org/" (rewrite-name (go-sum-module sum-line)) "/@v/" (go-sum-version sum-line) ".zip"))
@ -106,6 +119,7 @@
lines)
(list->vector (map (lambda (pair) (cons (car pair) (list->vector (cdr pair)))) dirs)))
;; Generates a full VFS structure from an on-disk directory.
(define (vfs-from-directory osdir)
(define iter-dir #f)
(define output '())
@ -125,6 +139,8 @@
(iter-dir "")
(list->vector output))
;; Calls `filter` for each file in the virtual filesystem, replacing its
;; contents with an empty file if `filter` returns false.
(define (filter-vfs vfs filter)
(vector-map
(lambda (dir)
@ -137,13 +153,15 @@
(cdr dir))))
vfs))
; List extracted from go src/go/build/build.go.
;; List extracted from go src/go/build/build.go.
(define good-extensions '("go" "c" "cc" "cpp" "cxx" "m" "h" "hh" "hpp" "hxx" "f" "F" "for" "f90" "s" "S" "sx" "swig" "swigcxx" "syso"))
(define (extract-extension name i)
(cond ((char=? (string-ref name i) #\.) (string-copy name (+ i 1)))
((= i 0) #f)
(else (extract-extension name (- i 1)))))
;; Returns a VFS, filtered down to only contain the contents of files that
;; will be read during the processing of Go packages.
(define (filter-vfs-for-package-reading vfs)
(filter-vfs vfs
(lambda (dir fname)