spdk

README

Go language bindings for the SPDK API

Go bindings for the SPDK native-C library to facilitate management of NVMe SSDs from an application written in Go.

The bindings require libspdk-devel (or distro equivalent) package to be installed. To install please follow steps in the SPDK github instructions.

These bindings are currently working against SPDK 21.07-pre and DPDK 21.02.0 which are the versions pinned in the DAOS 2.0 release.

This is not a general purpose set of SPDK go bindings but provides a set of capabilities tailored to the specific needs of DAOS, the NVMe SSD related features are as follows:

• device discovery (SPDK environment initialization and device probing)
• device firmware update
• VMD enablement and discovery
• format (wipe) of device namespaces

How to build these bindings

Using these bindings assumes SPDK shared lib is installed in ${SPDK_REPO}/build/lib/libspdk.so, please also follow Hugepages and Device Binding.

Setup environment:

export GOSPDK=${DAOS_ROOT}/src/control/lib/spdk
export SPDK_LIB=${DAOS_ROOT}/opt/spdk
export LD_LIBRARY_PATH=${SPDK_LIB}/build/lib:${SPDK_LIB}/include:${GOSPDK}/spdk:${LD_LIBRARY_PATH}
export CGO_CFLAGS="-I${SPDK_LIB}/include"
LIBS="-lspdk_nvme -lnvme_control -lspdk_env_dpdk -lspdk_vmd -lrte_mempool -lrte_mempool_ring -lrte_bus_pci"
export CGO_LDFLAGS="-L${SPDK_LIB}/build/lib ${LIBS}"

Build NVMe libs:

cd ${GOSPDK}
gcc ${CGO_LDFLAGS} ${CGO_CFLAGS} -Werror -g -Wshadow -Wall -Wno-missing-braces -c -fpic -Iinclude src/*.c ${libs}
gcc ${CGO_LDFLAGS} ${CGO_CFLAGS} -shared -o libnvme_control.so *.o

Build go spdk bindings:

cd ${GOSPDK}
sudo CGO_LDFLAGS=${CGO_LDFLAGS} CGO_CFLAGS=${CGO_CFLAGS} go build -v -i

Run Go Unit Tests:

cd ${GOSPDK}
go test -v

cmocka Test Env:

export TESTLIBS="-lspdk -lcmocka -lnvme_control"
export GCCFLAGS="-g -fpic -Wall -Werror -Wshadow -Wno-missing-braces"
export TESTFLAGS="${CGO_LDFLAGS} ${CGO_CFLAGS} ${GCCFLAGS} ${TESTLIBS}"

Build cmocka Tests:

cd ${GOSPDK}/ctests
gcc ${TESTFLAGS} -I../include -L../. nvme_control_ut.c ../src/*.c -o nvme_control_ctests

Documentation

Overview

Package spdk provides Go bindings for SPDK

Package spdk provides Go bindings for SPDK

Index

• Variables
• func FaultBindingFailed(rc int, msg string) *fault.Fault
• func FaultBindingRetNull(msg string) *fault.Fault
• type Env
• type EnvImpl
  • func (ei *EnvImpl) FiniSPDKEnv(log logging.Logger, opts *EnvOptions)
  • func (ei *EnvImpl) InitSPDKEnv(log logging.Logger, opts *EnvOptions) error
• type EnvOptions
• type FormatResult
• type MockEnvCfg
• type MockEnvImpl
  • func (e *MockEnvImpl) FiniSPDKEnv(log logging.Logger, opts *EnvOptions)
  • func (e *MockEnvImpl) InitSPDKEnv(log logging.Logger, opts *EnvOptions) error
• type MockNvmeCfg
• type MockNvmeImpl
  • func (n *MockNvmeImpl) Discover(log logging.Logger) (storage.NvmeControllers, error)
  • func (n *MockNvmeImpl) Format(log logging.Logger) ([]*FormatResult, error)
  • func (n *MockNvmeImpl) Update(log logging.Logger, ctrlrPciAddr string, path string, slot int32) error
• type Nvme
• type NvmeImpl
  • func (n *NvmeImpl) Discover(log logging.Logger) (storage.NvmeControllers, error)
  • func (n *NvmeImpl) Format(log logging.Logger) ([]*FormatResult, error)
  • func (n *NvmeImpl) Update(log logging.Logger, ctrlrPciAddr string, path string, slot int32) error

Variables

var (
	FaultUnknown = spdkFault(
		code.SpdkUnknown,
		"unknown SPDK bindings error",
		"",
	)
	FaultCtrlrNoHealth = spdkFault(
		code.SpdkCtrlrNoHealth,
		"NVMe controller details are missing health statistics",
		"",
	)
)

Functions

func FaultBindingFailed

func FaultBindingFailed(rc int, msg string) *fault.Fault

func FaultBindingRetNull

func FaultBindingRetNull(msg string) *fault.Fault

Types

type Env

type Env interface {
	InitSPDKEnv(logging.Logger, *EnvOptions) error
	FiniSPDKEnv(logging.Logger, *EnvOptions)
}

Env is the interface that provides SPDK environment management.

type EnvImpl

type EnvImpl struct{}

EnvImpl is a an implementation of the Env interface.

func (*EnvImpl) FiniSPDKEnv

func (ei *EnvImpl) FiniSPDKEnv(log logging.Logger, opts *EnvOptions)

FiniSPDKEnv initializes the SPDK environment.

func (*EnvImpl) InitSPDKEnv

func (ei *EnvImpl) InitSPDKEnv(log logging.Logger, opts *EnvOptions) error

InitSPDKEnv initializes the SPDK environment.

type EnvOptions

type EnvOptions struct {
	PCIAllowList *hardware.PCIAddressSet // restrict SPDK device access
	EnableVMD    bool                    // flag if VMD functionality should be enabled
}

EnvOptions describe parameters to be used when initializing a processes SPDK environment.

type FormatResult

type FormatResult struct {
	CtrlrPCIAddr string
	NsID         uint32
	Err          error
}

FormatResult struct mirrors C.struct_wipe_res_t and describes the results of a format operation on an NVMe controller namespace.

type MockEnvCfg

type MockEnvCfg struct {
	InitErr error
}

MockEnvCfg controls the behavior of the MockEnvImpl.

type MockEnvImpl

type MockEnvImpl struct {
	sync.RWMutex
	Cfg       MockEnvCfg
	InitCalls []*EnvOptions
	FiniCalls []*EnvOptions
}

MockEnvImpl is a mock implementation of the Env interface.

func (*MockEnvImpl) FiniSPDKEnv

func (e *MockEnvImpl) FiniSPDKEnv(log logging.Logger, opts *EnvOptions)

FiniSPDKEnv finalizes the SPDK environment.

func (*MockEnvImpl) InitSPDKEnv

func (e *MockEnvImpl) InitSPDKEnv(log logging.Logger, opts *EnvOptions) error

InitSPDKEnv initializes the SPDK environment.

type MockNvmeCfg

type MockNvmeCfg struct {
	DiscoverCtrlrs storage.NvmeControllers
	DiscoverErr    error
	FormatRes      []*FormatResult
	FormatErr      error
	UpdateErr      error
}

MockNvmeCfg controls the behavior of the MockNvmeImpl.

type MockNvmeImpl

type MockNvmeImpl struct {
	Cfg MockNvmeCfg
}

MockNvmeImpl is an implementation of the Nvme interface.

func (*MockNvmeImpl) Discover

func (n *MockNvmeImpl) Discover(log logging.Logger) (storage.NvmeControllers, error)

Discover NVMe devices, including NVMe devices behind VMDs if enabled, accessible by SPDK on a given host.

func (*MockNvmeImpl) Format

func (n *MockNvmeImpl) Format(log logging.Logger) ([]*FormatResult, error)

Format device at given pci address, destructive operation!

func (*MockNvmeImpl) Update

func (n *MockNvmeImpl) Update(log logging.Logger, ctrlrPciAddr string, path string, slot int32) error

Update calls C.nvme_fwupdate to update controller firmware image.

type Nvme

type Nvme interface {
	// Discover NVMe controllers and namespaces, and device health info
	Discover(logging.Logger) (storage.NvmeControllers, error)
	// Format NVMe controller namespaces
	Format(logging.Logger) ([]*FormatResult, error)
	// Update updates the firmware on a specific PCI address and slot
	Update(log logging.Logger, ctrlrPciAddr string, path string, slot int32) error
}

Nvme is the interface that provides SPDK NVMe functionality.

type NvmeImpl

type NvmeImpl struct{}

NvmeImpl is an implementation of the Nvme interface.

func (*NvmeImpl) Discover

func (n *NvmeImpl) Discover(log logging.Logger) (storage.NvmeControllers, error)

Discover NVMe devices.

func (*NvmeImpl) Format

func (n *NvmeImpl) Format(log logging.Logger) ([]*FormatResult, error)

Format devices available through SPDK.

func (*NvmeImpl) Update

func (n *NvmeImpl) Update(log logging.Logger, ctrlrPciAddr string, path string, slot int32) error

Update updates the firmware image via SPDK in a given slot on the device.