wal

package
v0.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 14, 2023 License: MIT Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source 
const ChunkHeaderSize = 7

ChunkHeaderSize +-----------+--------------+------------+------------+ | check sum | chunk length | chunk type | chunk data | +-----------+--------------+------------+------------+ | 4 Bytes | 2 Bytes | 1 Bytes | <= 32KB-7B | +-----------+--------------+------------+------------+

View Source 
const DefaultWalSuffix = "wal"

DefaultWalSuffix is the extension name of wal file, default is wal. eg: 001.wal

View Source 
const MaxBlockSize = 32 * types.KB

Variables

View Source 
var (
	ErrAlreadyClosed    = errors.New("file already closed")
	ErrBlockExceeded    = errors.New("block size exceeded")
	ErrWriteToImmutable = errors.New("write to immutable")
)
View Source 
var (
	ErrFileNotFound   = errors.New("wal file not found")
	ErrDataExceedFile = errors.New("data size has exceeded the max file limit")
)

Functions

func EstimateBlockSize added in v0.3.0 

func EstimateBlockSize(dataSize int64) int64

EstimateBlockSize estimate the possible max size of data will be written in wal file if the first written block has already contains chunks, maybe occurs the first data chunk could be hold by the block but no left space to hold the second chunk header, in that case, the block need padding to fill the left space. padding + block * header + data size

func WalFileName 

func WalFileName(dir string, fid uint32, ext string) string

WalFileName returns the name of wal file

Types

type ChunkHeader 

type ChunkHeader struct {
	Length    uint16
	ChunkType ChunkType
	// contains filtered or unexported fields
}

type ChunkIterator 

type ChunkIterator interface {
	// File returns file that iterator belongs to
	File() File
	// Index returns current chunk position
	Index() ChunkPos
	// Next returns the next chunk data bytes, chunk info, and error
	Next() ([]byte, ChunkPos, error)
}

ChunkIterator iterate chunk data for a log file

type ChunkPos 

type ChunkPos struct {
	// file id
	Fid uint32
	// chunk in which block
	Block uint32
	// chunk offset in block
	Offset int64
	// chunk size
	Size int64
}

type ChunkType 

type ChunkType = byte
const (
	ChunkFullType ChunkType = 1 + iota
	ChunkFirstType
	ChunkMiddleType
	ChunkLastType
)

type F 

type F struct {
	Fid  uint32
	File File
}

func (F) Less 

func (f F) Less(nf F) bool

type File 

type File interface {
	// Fid return file id
	Fid() uint32
	// Read bytes with specified block and offset
	// chunkOffset is offset by the start of the specified block
	Read(block uint32, chunkOffset int64) ([]byte, error)
	// Write bytes and returns ChunkPos
	Write(data []byte) (ChunkPos, error)
	// WriteAll write bytes in batch and returns ChunkPos
	WriteAll(data [][]byte) ([]ChunkPos, error)
	// Iterator iterate per chunk in file
	Iterator() (ChunkIterator, error)
	// Size return size of file
	Size() int64
	// Remove remove file
	Remove() error
	// Sync buffer to disk
	Sync() error
	// MarkImmutable mark this file as immutable
	MarkImmutable()
	// Close file and release resource
	Close() error
}

File represents a wal file

type FileIterator 

type FileIterator struct {
	// contains filtered or unexported fields
}

FileIterator iterates over all the files and chunks

func (*FileIterator) IndexFid 

func (f *FileIterator) IndexFid() uint32

IndexFid returns the current fid

func (*FileIterator) IndexPos 

func (f *FileIterator) IndexPos() ChunkPos

IndexPos returns curren index position

func (*FileIterator) NextData 

func (f *FileIterator) NextData() ([]byte, ChunkPos, error)

func (*FileIterator) NextFile 

func (f *FileIterator) NextFile()

type LogFile 

type LogFile struct {
	// contains filtered or unexported fields
}

func (*LogFile) Close 

func (log *LogFile) Close() error

func (*LogFile) Fid 

func (log *LogFile) Fid() uint32

func (*LogFile) Iterator 

func (log *LogFile) Iterator() (ChunkIterator, error)

func (*LogFile) MarkImmutable 

func (log *LogFile) MarkImmutable()

func (*LogFile) Read 

func (log *LogFile) Read(block uint32, chunkOffset int64) ([]byte, error)

func (*LogFile) Remove 

func (log *LogFile) Remove() error

func (*LogFile) Size 

func (log *LogFile) Size() int64

func (*LogFile) Sync 

func (log *LogFile) Sync() error

func (*LogFile) Write 

func (log *LogFile) Write(data []byte) (ChunkPos, error)

func (*LogFile) WriteAll 

func (log *LogFile) WriteAll(data [][]byte) ([]ChunkPos, error)

type LogFileChunkIterator 

type LogFileChunkIterator struct {
	// contains filtered or unexported fields
}

func (*LogFileChunkIterator) File 

func (l *LogFileChunkIterator) File() File

func (*LogFileChunkIterator) Index 

func (l *LogFileChunkIterator) Index() ChunkPos

func (*LogFileChunkIterator) Next 

func (l *LogFileChunkIterator) Next() ([]byte, ChunkPos, error)

type Option 

type Option struct {
	// data dir
	DataDir string
	// max log file size
	MaxFileSize int64
	// log file extension
	Ext string
	// how many block will be cached, off if it is 0
	BlockCache uint32
	// sync buffer to disk per write, if not enabled data maybe not save when machine crashed
	FsyncPerWrite bool
	// specify the written threshold for triggering sync
	FsyncThreshold int64
}

func DefaultOption added in v0.2.0 

func DefaultOption(dir string) Option

type Pending added in v0.2.0 

type Pending struct {
	// contains filtered or unexported fields
}

Pending provides a way to write data in batches

func (*Pending) Flush added in v0.2.0 

func (p *Pending) Flush(sync bool) ([]ChunkPos, error)

func (*Pending) Reset added in v0.2.0 

func (p *Pending) Reset()

func (*Pending) Write added in v0.2.0 

func (p *Pending) Write(data []byte) error

type Stat added in v0.2.0 

type Stat struct {
	NumOfWal  int64
	SizeOfWal int64
}

type Wal 

type Wal struct {
	// contains filtered or unexported fields
}

Wal instance represents a data directory and its sub-files that store write-ahead-log format data

func Open 

func Open(option Option) (*Wal, error)

Open returns a new wal instance with the give Option

func (*Wal) ActiveFid 

func (w *Wal) ActiveFid() uint32

ActiveFid returns current active file id

func (*Wal) Close 

func (w *Wal) Close() error

Close will close all the immutable files sync the active file and close it

func (*Wal) IsEmpty 

func (w *Wal) IsEmpty() bool

IsEmpty check wal data if is empty

func (*Wal) Iterator 

func (w *Wal) Iterator(minFid, maxFid uint32, pos ChunkPos) (FileIterator, error)

Iterator returns an FileIterator maxFid determines the file range for the iterator pos determines the chunk position for the iterator

func (*Wal) Pending added in v0.2.0 

func (w *Wal) Pending(capacity int) *Pending

func (*Wal) Purge 

func (w *Wal) Purge() error

Purge close all the file and remove them

func (*Wal) Read 

func (w *Wal) Read(pos ChunkPos) ([]byte, error)

Read reads data at the specified position

func (*Wal) Rotate 

func (w *Wal) Rotate() error

func (*Wal) Stat added in v0.2.0 

func (w *Wal) Stat() Stat

Stat return a simple statistics of wal instance

func (*Wal) Sync 

func (w *Wal) Sync() error

Sync call fsync on the active file

func (*Wal) Write 

func (w *Wal) Write(data []byte) (ChunkPos, error)

Write writes data bytes to the active file

func (*Wal) WriteAll added in v0.2.0 

func (w *Wal) WriteAll(datas [][]byte, needSync bool) ([]ChunkPos, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.