README
¶
runn ( means "Run N" ) is a package/tool for running operations following a scenario.
Key features of runn are:
- As a test helper package for the Go language.
- As a tool for scenario based testing.
- As a tool for automation.
- Support HTTP request, gRPC request, DB query and command execution
- OpenAPI Document-like syntax for HTTP request testing.
Usage
runn can run a multi-step scenario following a runbook written in YAML format.
As a test helper package for the Go language.
runn can also behave as a test helper for the Go language.
Run N runbooks using httptest.Server and sql.DB
func TestRouter(t *testing.T) {
ctx := context.Background()
db, err := sql.Open("mysql", "username:password@tcp(localhost:3306)/testdb")
if err != nil {
log.Fatal(err)
}
ts := httptest.NewServer(NewRouter(db))
t.Cleanup(func() {
ts.Close()
db.Close()
})
opts := []runn.Option{
runn.T(t),
runn.Runner("req", ts.URL),
runn.DBRunner("db", db),
}
o, err := runn.Load("testdata/books/**/*.yml", opts...)
if err != nil {
t.Fatal(err)
}
if err := o.RunN(ctx); err != nil {
t.Fatal(err)
}
}
Run single runbook using httptest.Server and sql.DB
func TestRouter(t *testing.T) {
ctx := context.Background()
db, err := sql.Open("mysql", "username:password@tcp(localhost:3306)/testdb")
if err != nil {
log.Fatal(err)
}
ts := httptest.NewServer(NewRouter(db))
t.Cleanup(func() {
ts.Close()
db.Close()
})
opts := []runn.Option{
runn.T(t),
runn.Book("testdata/books/login.yml"),
runn.Runner("req", ts.URL),
runn.DBRunner("db", db),
}
o, err := runn.New(opts...)
if err != nil {
t.Fatal(err)
}
if err := o.Run(ctx); err != nil {
t.Fatal(err)
}
}
Run N runbooks using grpc.Server
func TestServer(t *testing.T) {
addr := "127.0.0.1:8080"
l, err := net.Listen("tcp", addr)
if err != nil {
t.Fatal(err)
}
ts := grpc.NewServer()
myapppb.RegisterMyappServiceServer(s, NewMyappServer())
reflection.Register(s)
go func() {
s.Serve(l)
}()
t.Cleanup(func() {
ts.GracefulStop()
})
opts := []runn.Option{
runn.T(t),
runn.Runner("greq", fmt.Sprintf("grpc://%s", addr),
}
o, err := runn.Load("testdata/books/**/*.yml", opts...)
if err != nil {
t.Fatal(err)
}
if err := o.RunN(ctx); err != nil {
t.Fatal(err)
}
}
Run N runbooks with http.Handler and sql.DB
func TestRouter(t *testing.T) {
ctx := context.Background()
db, err := sql.Open("mysql", "username:password@tcp(localhost:3306)/testdb")
if err != nil {
log.Fatal(err)
}
t.Cleanup(func() {
db.Close()
})
opts := []runn.Option{
runn.T(t),
runn.HTTPRunnerWithHandler("req", NewRouter(db)),
runn.DBRunner("db", db),
}
o, err := runn.Load("testdata/books/**/*.yml", opts...)
if err != nil {
t.Fatal(err)
}
if err := o.RunN(ctx); err != nil {
t.Fatal(err)
}
}
As a tool for scenario based testing / As a tool for automation.
runn can run one or more runbooks as a CLI tool.
$ runn list path/to/**/*.yml
Desc Path If
---------------------------------------------------------------------------------
Login and get projects. pato/to/book/projects.yml
Login and logout. pato/to/book/logout.yml
Only if included. pato/to/book/only_if_included.yml included
$ runn run path/to/**/*.yml
Login and get projects. ... ok
Login and logout. ... ok
Only if included. ... skip
3 scenarios, 1 skipped, 0 failures
Runbook ( runn scenario file )
The runbook file has the following format.
step: section accepts array or ordered map.
Array:
desc: Login and get projects.
runners:
req: https://example.com/api/v1
db: mysql://root:mypass@localhost:3306/testdb
vars:
username: alice
password: ${TEST_PASS}
steps:
-
db:
query: SELECT * FROM users WHERE name = '{{ vars.username }}'
-
req:
/login:
post:
body:
application/json:
email: "{{ steps[0].rows[0].email }}"
password: "{{ vars.password }}"
test: steps[1].res.status == 200
-
req:
/projects:
get:
headers:
Authorization: "token {{ steps[1].res.body.session_token }}"
body: null
test: steps[2].res.status == 200
-
test: len(steps[2].res.body.projects) > 0
Map:
desc: Login and get projects.
runners:
req: https://example.com/api/v1
db: mysql://root:mypass@localhost:3306/testdb
vars:
username: alice
password: ${TEST_PASS}
steps:
find_user:
db:
query: SELECT * FROM users WHERE name = '{{ vars.username }}'
login:
req:
/login:
post:
body:
application/json:
email: "{{ steps.find_user.rows[0].email }}"
password: "{{ vars.password }}"
test: steps.login.res.status == 200
list_projects:
req:
/projects:
get:
headers:
Authorization: "token {{ steps.login.res.body.session_token }}"
body: null
test: steps.list_projects.res.status == 200
count_projects:
test: len(steps.list_projects.res.body.projects) > 0
Grouping of related parts by color
Array:
Map:
desc:
Description of runbook.
runners:
Mapping of runners that run steps: of runbook.
In the steps: section, call the runner with the key specified in the runners: section.
Built-in runners such as test runner do not need to be specified in this section.
runners:
ghapi: ${GITHUB_API_ENDPOINT}
idp: https://auth.example.com
db: my:dbuser:${DB_PASS}@hostname:3306/dbname
In the example, each runner can be called by ghapi:, idp: or db: in steps:.
vars:
Mapping of variables available in the steps: of runbook.
vars:
username: alice@example.com
token: ${SECRET_TOKEN}
In the example, each variable can be used in {{ vars.username }} or {{ vars.token }} in steps:.
debug:
Enable debug output for runn.
debug: true
if:
Conditions for skip all steps.
if: included # Run steps only if included
skipTest:
Skip all test: sections
skipTest: true
steps:
Steps to run in runbook.
The steps are invoked in order from top to bottom.
Any return values are recorded for each step.
When steps: is array, recorded values can be retrieved with {{ steps[*].* }}.
steps:
-
db:
query: SELECT * FROM users WHERE name = '{{ vars.username }}'
-
req:
/users/{{ steps[0].rows[0].id }}:
get:
body: null
When steps: is map, recorded values can be retrieved with {{ steps.<key>.* }}.
steps:
find_user:
db:
query: SELECT * FROM users WHERE name = '{{ vars.username }}'
user_info:
req:
/users/{{ steps.find_user.rows[0].id }}:
get:
body: null
steps[*].desc: steps.<key>.desc:
Description of step.
steps[*].if: steps.<key>.if:
Conditions for skip step.
steps:
login:
if: 'len(vars.token) == 0' # Run step only if var.token is not set
req:
/login:
post:
body:
[...]
steps[*].loop: steps.<key>.loop:
Loop settings for steps.
Simple loop usage
steps:
multicartin:
loop: 10
req:
/cart/in:
post:
body:
application/json:
product_id: "{{ i }}" # The loop count (0..9) is assigned to `i`.
[...]
or
steps:
multicartin:
loop:
count: 10
req:
/cart/in:
post:
body:
application/json:
product_id: "{{ i }}" # The loop count (0..9) is assigned to `i`.
[...]
Retry
It can be used as a retry mechanism by setting a condition in the until: section.
If the condition of until: is met, the loop is broken without waiting for the number of count: to be run.
Also, if the run of the number of count: completes but does not satisfy the condition of until:, then the step is considered to be failed.
steps:
waitingroom:
loop:
count: 10
until: 'steps.waitingroom.res.status == "201"' # Store values of latest loop
minInterval: 0.5 # sec
maxInterval: 10 # sec
# jitter: 0.0
# interval: 5
# multiplier: 1.5
req:
/cart/in:
post:
body:
[...]
( steps[*].retry: steps.<key>.retry: are deprecated )
Runner
HTTP Runner: Do HTTP request
Use https:// or http:// scheme to specify HTTP Runner.
When the step is invoked, it sends the specified HTTP Request and records the response.
runners:
ghapi: https://api.github.com
Validation of HTTP request and HTTP response
HTTP requests sent by runn and their HTTP responses can be validated.
OpenAPI v3:
runners:
myapi:
endpoint: https://api.github.com
openapi3: path/to/openapi.yaml
# skipValidateRequest: false
# skipValidateResponse: false
gRPC Runner: Do gRPC request
Use grpc:// scheme to specify gRPC Runner.
When the step is invoked, it sends the specified gRPC Request and records the response.
runners:
greq: grpc://grpc.example.com:80
runners:
greq:
addr: grpc.example.com:8080
tls: true
cacert: path/to/cacert.pem
cert: path/to/cert.pem
key: path/to/key.pem
# skipVerify: false
DB Runner: Query a database
Use dsn (Data Source Name) to specify DB Runner.
When step is executed, it executes the specified query the database.
If the query is a SELECT clause, it records the selected rows, otherwise it records last_insert_id and rows_affected .
Support Databases
PostgreSQL:
runners:
mydb: postgres://dbuser:dbpass@hostname:5432/dbname
runners:
db: pg://dbuser:dbpass@hostname:5432/dbname
MySQL:
runners:
testdb: mysql://dbuser:dbpass@hostname:3306/dbname
runners:
db: my://dbuser:dbpass@hostname:3306/dbname
SQLite3:
runners:
db: sqlite:///path/to/dbname.db
runners:
local: sq://dbname.db
Exec Runner: execute command
The exec runner is a built-in runner, so there is no need to specify it in the runners: section.
It execute command using command: and stdin:
-
exec:
command: grep error
stdin: '{{ steps[3].res.rawBody }}'
Test Runner: test using recorded values
The test runner is a built-in runner, so there is no need to specify it in the runners: section.
It evaluates the conditional expression using the recorded values.
-
test: steps[3].res.status == 200
The test runner can run in the same steps as the other runners.
Dump Runner: dump recorded values
The dump runner is a built-in runner, so there is no need to specify it in the runners: section.
It dumps the specified recorded values.
-
dump: steps[4].rows
The dump runner can run in the same steps as the other runners.
Include Runner: include other runbook
The include runner is a built-in runner, so there is no need to specify it in the runners: section.
Include runner reads and runs the runbook in the specified path.
Recorded values are nested.
-
include: path/to/get_token.yml
It is also possible to override vars: of included runbook.
-
include:
path: path/to/login.yml
vars:
username: alice
password: alicepass
-
include:
path: path/to/login.yml
vars:
username: bob
password: bobpass
It is also possible to skip all test: sections in the included runbook.
-
include:
path: path/to/signup.yml
skipTest: true
Bind Runner: bind variables
The bind runner is a built-in runner, so there is no need to specify it in the runners: section.
It bind runner binds any values with another key.
-
req:
/users/k1low:
get:
body: null
-
bind:
user_id: steps[0].res.body.data.id
-
dump: user_id
The bind runner can run in the same steps as the other runners.
Option
See https://pkg.go.dev/github.com/k1LoW/runn#Option
Example: Run as a test helper ( func T )
https://pkg.go.dev/github.com/k1LoW/runn#T
o, err := runn.Load("testdata/**/*.yml", runn.T(t))
if err != nil {
t.Fatal(err)
}
if err := o.RunN(ctx); err != nil {
t.Fatal(err)
}
Example: Add custom function ( func Func )
https://pkg.go.dev/github.com/k1LoW/runn#Func
desc: Test using GitHub
runners:
req:
endpoint: https://github.com
steps:
-
req:
/search?l={{ urlencode('C++') }}&q=runn&type=Repositories:
get:
body:
application/json:
null
test: 'steps[0].res.status == 200'
o, err := runn.Load("testdata/**/*.yml", runn.Func("urlencode", url.QueryEscape))
if err != nil {
t.Fatal(err)
}
if err := o.RunN(ctx); err != nil {
t.Fatal(err)
}
Filter runbooks to be executed by the environment variable RUNN_RUN
Run only runbooks matching the filename "login".
$ env RUNN_RUN=login go test ./... -run TestRouter
Measure elapsed time as profile
opts := []runn.Option{
runn.T(t),
runn.Book("testdata/books/login.yml"),
runn.Profile(true)
}
o, err := runn.New(opts...)
if err != nil {
t.Fatal(err)
}
if err := o.Run(ctx); err != nil {
t.Fatal(err)
}
f, err := os.Open("profile.json")
if err != nil {
t.Fatal(err)
}
if err := o.DumpProfile(f); err != nil {
t.Fatal(err)
}
Install
As a CLI tool
deb:
$ export RUNN_VERSION=X.X.X
$ curl -o runn.deb -L https://github.com/k1LoW/runn/releases/download/v$RUNN_VERSION/runn_$RUNN_VERSION-1_amd64.deb
$ dpkg -i runn.deb
RPM:
$ export RUNN_VERSION=X.X.X
$ yum install https://github.com/k1LoW/runn/releases/download/v$RUNN_VERSION/runn_$RUNN_VERSION-1_amd64.rpm
apk:
$ export RUNN_VERSION=X.X.X
$ curl -o runn.apk -L https://github.com/k1LoW/runn/releases/download/v$RUNN_VERSION/runn_$RUNN_VERSION-1_amd64.apk
$ apk add runn.apk
homebrew tap:
$ brew install k1LoW/tap/runn
manually:
Download binary from releases page
docker:
$ docker pull ghcr.io/k1low/runn:latest
go install:
$ go install github.com/k1LoW/runn/cmd/runn@latest
As a test helper
$ go get github.com/k1LoW/runn
Alternatives
- zoncoen/scenarigo: An end-to-end scenario testing tool for HTTP/gRPC server.
References
- zoncoen/scenarigo: An end-to-end scenario testing tool for HTTP/gRPC server.
- fullstorydev/grpcurl: Like cURL, but for gRPC: Command-line tool for interacting with gRPC servers
Documentation
¶
Index ¶
- Constants
- Variables
- func CACert(path string) grpcRunnerOption
- func CACertFromData(b []byte) grpcRunnerOption
- func Cert(path string) grpcRunnerOption
- func CertFromData(b []byte) grpcRunnerOption
- func GetDesc(opt Option) string
- func Key(path string) grpcRunnerOption
- func KeyFromData(b []byte) grpcRunnerOption
- func Load(pathp string, opts ...Option) (*operators, error)
- func LoadBook(path string) (*book, error)
- func New(opts ...Option) (*operator, error)
- func OpenApi3(l string) httpRunnerOption
- func OpenApi3FromData(d []byte) httpRunnerOption
- func Paths(pathp string) ([]string, error)
- func SkipValidateRequest(skip bool) httpRunnerOption
- func SkipValidateResponse(skip bool) httpRunnerOption
- func TLS(useTLS bool) grpcRunnerOption
- type Loop
- type Option
- func AfterFunc(fn func() error) Option
- func BeforeFunc(fn func() error) Option
- func Book(path string) Option
- func Books(pathp string) ([]Option, error)
- func DBRunner(name string, client *sql.DB) Option
- func Debug(debug bool) Option
- func Desc(desc string) Option
- func FailFast(enable bool) Option
- func Func(k string, v interface{}) Option
- func GrpcRunner(name string, cc *grpc.ClientConn, opts ...grpcRunnerOption) Option
- func HTTPRunner(name, endpoint string, client *http.Client, opts ...httpRunnerOption) Option
- func HTTPRunnerWithHandler(name string, h http.Handler, opts ...httpRunnerOption) Option
- func Interval(d time.Duration) Option
- func Profile(profile bool) Option
- func RunMatch(m string) Option
- func RunSample(n int) Option
- func RunShard(n, i int) Option
- func Runner(name, dsn string, opts ...httpRunnerOption) Option
- func SkipIncluded(enable bool) Option
- func SkipTest(enable bool) Option
- func T(t *testing.T) Option
- func Var(k string, v interface{}) Option
- type UnsupportedError
Constants ¶
const ( MediaTypeApplicationJSON = "application/json" MediaTypeTextPlain = "text/plain" MediaTypeApplicationFormUrlencoded = "application/x-www-form-urlencoded" )
Variables ¶
var ( AsTestHelper = T Runbook = Book RunPart = RunShard )
Functions ¶
func CACertFromData ¶ added in v0.26.0
func CACertFromData(b []byte) grpcRunnerOption
func CertFromData ¶ added in v0.26.0
func CertFromData(b []byte) grpcRunnerOption
func KeyFromData ¶ added in v0.26.0
func KeyFromData(b []byte) grpcRunnerOption
func OpenApi3FromData ¶ added in v0.13.0
func OpenApi3FromData(d []byte) httpRunnerOption
func SkipValidateRequest ¶ added in v0.13.0
func SkipValidateRequest(skip bool) httpRunnerOption
func SkipValidateResponse ¶ added in v0.13.0
func SkipValidateResponse(skip bool) httpRunnerOption
Types ¶
type Loop ¶ added in v0.29.0
type Loop struct {
Count string `yaml:"count,omitempty"`
Interval *float64 `yaml:"interval,omitempty"`
MinInterval *float64 `yaml:"minInterval,omitempty"`
MaxInterval *float64 `yaml:"maxInterval,omitempty"`
Jitter *float64 `yaml:"jitter,omitempty"`
Multiplier *float64 `yaml:"multiplier,omitempty"`
Until string `yaml:"until"`
// contains filtered or unexported fields
}
type Option ¶
type Option func(*book) error
func AfterFunc ¶ added in v0.20.0
AfterFunc - Register the function to be run after the runbook is run.
func BeforeFunc ¶ added in v0.20.0
BeforeFunc - Register the function to be run before the runbook is run.
func GrpcRunner ¶ added in v0.24.0
func GrpcRunner(name string, cc *grpc.ClientConn, opts ...grpcRunnerOption) Option
GrpcRunner - Set grpc runner to runbook
func HTTPRunner ¶
HTTPRunner - Set http runner to runbook
func HTTPRunnerWithHandler ¶ added in v0.6.0
HTTPRunnerWithHandler - Set http runner to runbook with http.Handler
func RunShard ¶ added in v0.23.0
RunShard - Distribute runbooks into a specified number of shards and run the specified shard of them.
func SkipIncluded ¶ added in v0.19.0
SkipIncluded - Skip running the included step by itself.
type UnsupportedError ¶ added in v0.17.1
type UnsupportedError struct {
Cause error
}
func (*UnsupportedError) Error ¶ added in v0.17.1
func (e *UnsupportedError) Error() string
func (UnsupportedError) Unwrap ¶ added in v0.17.1
func (e UnsupportedError) Unwrap() error
Source Files
Directories