jit

README

JIT Compiler for Go

This package attempts to streamline the runtime build/load process for arbitrary Go code.

It assumes the existence of a working Go toolchain on the path.

It automatically resolves package dependencies recursively, and provides a type safe way of interacting with the built functions.

Go compiler patch

To allow the loader to know the types of exported functions, this package will attempt to patch the Go compiler (gc) to emit these if not already patched.

This patch can be found in gc.patch.

Usage and Configuration

package main

import (
	"fmt"
	"github.com/eh-steve/goloader/jit"
)

func main() {
	conf := jit.BuildConfig{
		KeepTempFiles:    false,          // Files are copied/written to a temp dir to ensure it is writable. This retains the temporary copies
		ExtraBuildFlags:  []string{"-x"}, // Flags passed to go build command
		BuildEnv:         nil,            // Env vars to set for go build toolchain
		TmpDir:           "",             // To control where temporary files are copied
		DebugLog:         true,           //
	}

	loadable, err := jit.BuildGoFiles(conf, "./path/to/file1.go", "/path/to/file2.go")
	if err != nil {
		panic(err)
	}
	// or
	loadable, err = jit.BuildGoPackage(conf, "./path/to/package")
	if err != nil {
		panic(err)
	}
	// or
	loadable, err = jit.BuildGoPackageRemote(conf, "github.com/some/package/v4", "latest")
	if err != nil {
		panic(err)
	}
	// or
	loadable, err = jit.BuildGoText(conf, `
package mypackage

import "encoding/json"

func MyFunc(input []byte) (interface{}, error) {
	var output interface{}
	err := json.Unmarshal(input, &output)
	return output, err
}
`)

	if err != nil {
		panic(err)
	}

	module, err := loadable.Load()
    symbols := module.SymbolsByPkg[loadable.ImportPath] 
	if err != nil {
		panic(err)
	}
	defer func() {
		err = module.Unload()
		if err != nil {
			panic(err)
		}
	}()
	switch f := symbols["MyFunc"].(type) {
	case func([]byte) (interface{}, error):
		result, err := f([]byte(`{"k":"v"}`))
		if err != nil {
			panic(err)
		}
		fmt.Println(result)
	default:
		panic("Function signature was not what was expected")
	}
}

Documentation

Index

• func GlobalPkgSet() map[string]struct{}
• func GlobalSymPtr() map[string]uintptr
• func GoGet(goCmd, packagePath, workDir string, verbose bool) error
• func GoListStd(goCmd string) map[string]struct{}
• func GoModDownload(goCmd, workDir string, verbose bool, args ...string) error
• func PatchGC(goBinary string, debugLog bool) error
• func RegisterCGoSymbol(symNameC string, symNameGo string) bool
• func RegisterSymbols() error
• func RegisterTypes(types ...interface{})
• type BuildConfig
• type LoadableUnit
  • func BuildGoFiles(config BuildConfig, pathToGoFile string, extraFiles ...string) (*LoadableUnit, error)
  • func BuildGoPackage(config BuildConfig, pathToGoPackage string) (*LoadableUnit, error)
  • func BuildGoPackageRemote(config BuildConfig, goPackage string, version string) (*LoadableUnit, error)
  • func BuildGoText(config BuildConfig, goText string) (*LoadableUnit, error)
  • func (l *LoadableUnit) Load() (module *goloader.CodeModule, err error)
• type Module
• type ModuleError
• type Package
  • func GoList(goCmd, absPath, workDir string, verbose bool) (*Package, error)
• type PackageError

Functions

func GlobalPkgSet

func GlobalPkgSet() map[string]struct{}

func GlobalSymPtr

func GlobalSymPtr() map[string]uintptr

func GoGet

func GoGet(goCmd, packagePath, workDir string, verbose bool) error

func GoListStd

func GoListStd(goCmd string) map[string]struct{}

func GoModDownload

func GoModDownload(goCmd, workDir string, verbose bool, args ...string) error

func PatchGC

func PatchGC(goBinary string, debugLog bool) error

PatchGC checks whether the go compiler at a given GOROOT requires patching to emit export types and if so, applies a patch and rebuilds it and tests again

func RegisterCGoSymbol

func RegisterCGoSymbol(symNameC string, symNameGo string) bool

func RegisterSymbols

func RegisterSymbols() error

func RegisterTypes

func RegisterTypes(types ...interface{})

Types

type BuildConfig

type BuildConfig struct {
	GoBinary                         string // Path to go binary, defaults to "go"
	KeepTempFiles                    bool
	ExtraBuildFlags                  []string
	BuildEnv                         []string
	TmpDir                           string
	DebugLog                         bool
	SymbolNameOrder                  []string // Control the layout of symbols in the linker's linear memory - useful for reproducing bugs
	RandomSymbolNameOrder            bool     // Randomise the order of linker symbols (may identify linker bugs)
	RelocationDebugWriter            io.Writer
	DumpTextBeforeAfterRelocation    bool
	SkipTypeDeduplicationForPackages []string
	UnsafeBlindlyUseFirstmoduleTypes bool
	Dynlink                          bool
}

type LoadableUnit

type LoadableUnit struct {
	Linker     *goloader.Linker
	ImportPath string
	Module     *goloader.CodeModule
	Package    *Package
}

func BuildGoFiles

func BuildGoFiles(config BuildConfig, pathToGoFile string, extraFiles ...string) (*LoadableUnit, error)

func BuildGoPackage

func BuildGoPackage(config BuildConfig, pathToGoPackage string) (*LoadableUnit, error)

func BuildGoPackageRemote

func BuildGoPackageRemote(config BuildConfig, goPackage string, version string) (*LoadableUnit, error)

func BuildGoText

func BuildGoText(config BuildConfig, goText string) (*LoadableUnit, error)

func (*LoadableUnit) Load

func (l *LoadableUnit) Load() (module *goloader.CodeModule, err error)

type Module

type Module struct {
	Path       string       // module path
	Query      string       // version query corresponding to this version
	Version    string       // module version
	Versions   []string     // available module versions
	Replace    *Module      // replaced by this module
	Time       *time.Time   // time version was created
	Update     *Module      // available update (with -u)
	Main       bool         // is this the main module?
	Indirect   bool         // module is only indirectly needed by main module
	Dir        string       // directory holding local copy of files, if any
	GoMod      string       // path to go.mod file describing module, if any
	GoVersion  string       // go version used in module
	Retracted  []string     // retraction information, if any (with -retracted or -u)
	Deprecated string       // deprecation message, if any (with -u)
	Error      *ModuleError // error loading module
	Origin     interface{}  // provenance of module
	Reuse      bool         // reuse of old module info is safe
}

type ModuleError

type ModuleError struct {
	Err string // the error itself
}

type Package

type Package struct {
	Dir           string   // directory containing package sources
	ImportPath    string   // import path of package in dir
	ImportComment string   // path in import comment on package statement
	Name          string   // package name
	Doc           string   // package documentation string
	Target        string   // install path
	Shlib         string   // the shared library that contains this package (only set when -linkshared)
	Goroot        bool     // is this package in the Go root?
	Standard      bool     // is this package part of the standard Go library?
	Stale         bool     // would 'go install' do anything for this package?
	StaleReason   string   // explanation for Stale==true
	Root          string   // Go root or Go path dir containing this package
	ConflictDir   string   // this directory shadows Dir in $GOPATH
	BinaryOnly    bool     // binary-only package (no longer supported)
	ForTest       string   // package is only for use in named test
	Export        string   // file containing export data (when using -export)
	BuildID       string   // build ID of the compiled package (when using -export)
	Module        *Module  // info about package's containing module, if any (can be nil)
	Match         []string // command-line patterns matching this package
	DepOnly       bool     // package is only a dependency, not explicitly listed

	// Source files
	GoFiles           []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
	CgoFiles          []string // .go source files that import "C"
	CompiledGoFiles   []string // .go files presented to compiler (when using -compiled)
	IgnoredGoFiles    []string // .go source files ignored due to build constraints
	IgnoredOtherFiles []string // non-.go source files ignored due to build constraints
	CFiles            []string // .c source files
	CXXFiles          []string // .cc, .cxx and .cpp source files
	MFiles            []string // .m source files
	HFiles            []string // .h, .hh, .hpp and .hxx source files
	FFiles            []string // .f, .F, .for and .f90 Fortran source files
	SFiles            []string // .s source files
	SwigFiles         []string // .swig files
	SwigCXXFiles      []string // .swigcxx files
	SysoFiles         []string // .syso object files to add to archive
	TestGoFiles       []string // _test.go files in package
	XTestGoFiles      []string // _test.go files outside package

	// Embedded files
	EmbedPatterns      []string // //go:embed patterns
	EmbedFiles         []string // files matched by EmbedPatterns
	TestEmbedPatterns  []string // //go:embed patterns in TestGoFiles
	TestEmbedFiles     []string // files matched by TestEmbedPatterns
	XTestEmbedPatterns []string // //go:embed patterns in XTestGoFiles
	XTestEmbedFiles    []string // files matched by XTestEmbedPatterns

	// Cgo directives
	CgoCFLAGS    []string // cgo: flags for C compiler
	CgoCPPFLAGS  []string // cgo: flags for C preprocessor
	CgoCXXFLAGS  []string // cgo: flags for C++ compiler
	CgoFFLAGS    []string // cgo: flags for Fortran compiler
	CgoLDFLAGS   []string // cgo: flags for linker
	CgoPkgConfig []string // cgo: pkg-config names

	// Dependency information
	Imports      []string          // import paths used by this package
	ImportMap    map[string]string // map from source import to ImportPath (identity entries omitted)
	Deps         []string          // all (recursively) imported dependencies
	TestImports  []string          // imports from TestGoFiles
	XTestImports []string          // imports from XTestGoFiles

	// Error information
	Incomplete bool            // this package or a dependency has an error
	Error      *PackageError   // error loading package
	DepsErrors []*PackageError // errors loading dependencies
}

func GoList

func GoList(goCmd, absPath, workDir string, verbose bool) (*Package, error)

type PackageError

type PackageError struct {
	ImportStack []string // shortest path from package named on command line to this one
	Pos         string   // position of error (if present, file:line:col)
	Err         string   // the error itself
}