codec

Documentation

Index

• Variables
• func BorrowEmptyIndexBuf() []byte
• func ReadInt(b []byte, offset uint32) uint32
• func ReturnIndexBuf(buf *[]byte)
• type Codec
  • func GetOrCreate(basePath string) (_codec Codec, exist bool, err error)
• type Metadata
• type V1
  • func (v *V1) GetHeaderSize() uint32
  • func (v *V1) GetIdxExtension() string
  • func (v *V1) GetIndexHeaderSize() uint32
  • func (v *V1) GetRecordSize(buf []byte, startFileOffset uint32) (payloadSize uint32, err error)
  • func (v *V1) GetTxnExtension() string
  • func (v *V1) ReadHeaderWithValidation(buf []byte, startFileOffset uint32) (payloadSize uint32, previousCrc uint32, payloadCrc uint32, err error)
  • func (*V1) ReadIndex(path string) ([]byte, error)
  • func (v *V1) ReadRecordWithValidation(buf []byte, startFileOffset uint32) (payload []byte, err error)
  • func (v *V1) RecoverIndex(buf []byte, startFileOffset uint32, baseEntryOffset int64, _ *int64) (index []byte, lastCrc uint32, newFileOffset uint32, lastEntryOffset int64, ...)
  • func (*V1) WriteIndex(path string, index []byte) error
  • func (*V1) WriteRecord(buf []byte, startOffset uint32, _ uint32, payload []byte) (recordSize uint32, payloadCrc uint32)
• type V2
  • func (v *V2) GetHeaderSize() uint32
  • func (v *V2) GetIdxExtension() string
  • func (v *V2) GetIndexHeaderSize() uint32
  • func (v *V2) GetRecordSize(buf []byte, startFileOffset uint32) (uint32, error)
  • func (v *V2) GetTxnExtension() string
  • func (v *V2) ReadHeaderWithValidation(buf []byte, startFileOffset uint32) (payloadSize uint32, previousCrc uint32, payloadCrc uint32, err error)
  • func (v *V2) ReadIndex(path string) ([]byte, error)
  • func (v *V2) ReadRecordWithValidation(buf []byte, startFileOffset uint32) (payload []byte, err error)
  • func (v *V2) RecoverIndex(buf []byte, startFileOffset uint32, baseEntryOffset int64, commitOffset *int64) (index []byte, lastCrc uint32, newFileOffset uint32, lastEntryOffset int64, ...)
  • func (v *V2) WriteIndex(path string, index []byte) error
  • func (*V2) WriteRecord(buf []byte, startOffset uint32, previousCrc uint32, payload []byte) (recordSize uint32, payloadCrc uint32)

Variables

var (
	ErrOffsetOutOfBounds = errors.New("oxia: offset out of bounds")
	ErrEmptyPayload      = errors.New("oxia: empty payload")
	ErrDataCorrupted     = errors.New("oxia: data corrupted")
)

var SupportedCodecs = []Codec{latestCodec, v1} // the latest codec should be always first element

Functions

func BorrowEmptyIndexBuf

func BorrowEmptyIndexBuf() []byte

func ReadInt

func ReadInt(b []byte, offset uint32) uint32

ReadInt read unsigned int from buf with big endian.

func ReturnIndexBuf

func ReturnIndexBuf(buf *[]byte)

Types

type Codec

type Codec interface {
	// GetHeaderSize returns the fixed size of the header in bytes
	// for each record. This value is used to understand where the
	// payload starts after the header.
	GetHeaderSize() uint32

	// GetIdxExtension returns the index file extension. THis value is used to help compatible with
	// multiple versions for index file.
	GetIdxExtension() string

	// GetTxnExtension returns the txn file extension. THis value is used to help compatible with
	// multiple versions for txn file.
	GetTxnExtension() string

	// GetRecordSize returns the size of the record in bytes which includes the header.
	GetRecordSize(buf []byte, startFileOffset uint32) (payloadSize uint32, err error)

	// ReadRecordWithValidation reads a record starting at the specified
	// file offset in the buffer. It also validates the record's integrity
	// (e.g., CRC checks) before returning the payload.
	//
	// Parameters:
	// - buf: The buffer containing the data to read from.
	// - startFileOffset: The file offset to start reading from.
	//
	// Returns:
	// - payload: The actual data (payload) of the record.
	// - err: Error if any issues occur during reading or validation.
	ReadRecordWithValidation(buf []byte, startFileOffset uint32) (payload []byte, err error)

	// ReadHeaderWithValidation reads the header of a record at the specified
	// offset and validates the integrity of the header data (e.g., CRC checks).
	//
	// Parameters:
	// - buf: The buffer containing the data to read from.
	// - startFileOffset: The file offset to start reading from.
	//
	// Returns:
	// - payloadSize: The size of the payload.
	// - previousCrc: The CRC value of the previous record.
	// - payloadCrc: The CRC value of the current payload.
	// - err: Error if any issues occur during reading or validation.
	ReadHeaderWithValidation(buf []byte, startFileOffset uint32) (payloadSize uint32, previousCrc uint32, payloadCrc uint32, err error)

	// WriteRecord writes a record to the buffer, starting at the specified
	// offset, and includes a header with metadata like CRC.
	//
	// Parameters:
	// - buf: The buffer where the record will be written.
	// - startFileOffset: The file offset to start reading from.
	// - previousCrc: The CRC value of the previous record to maintain consistency.
	// - payload: The actual data (payload) to write as part of the record.
	//
	// Returns:
	// - recordSize: The total size of the written record, including the header.
	// - payloadCrc: The CRC value of the written payload.
	WriteRecord(buf []byte, startFileOffset uint32, previousCrc uint32, payload []byte) (recordSize uint32, payloadCrc uint32)

	// GetIndexHeaderSize returns the size of the index header in bytes.
	// The header size is typically a fixed value representing the metadata at the
	// beginning of an index file.
	GetIndexHeaderSize() uint32

	// WriteIndex writes the provided index data to the specified file path.
	// Parameters:
	// - path: is the location where the index file will be written.
	// - index: is the byte slice that contains the index data.
	// Returns an error if the file cannot be written or if any I/O issues occur.
	WriteIndex(path string, index []byte) error

	// ReadIndex reads the index data from the specified file path.
	// Parameters
	// - path is the location of the index file to be read.
	// Returns the index data as a byte slice and an error if any I/O issues occur.
	ReadIndex(path string) ([]byte, error)

	// RecoverIndex attempts to recover the index from a txn byte buffer.
	//
	// Parameters:
	//   - buf: the byte slice containing the raw data.
	//   - startFileOffset: the starting file offset from which recovery begins.
	//   - baseEntryOffset: the base offset for the index entries, used to adjust entry offsets.
	//   - commitOffset: a pointer to the commit offset, which is using for auto-discard uncommited corruption data
	//
	// Returns:
	//   - index: the recovered index data as a byte slice.
	//   - lastCrc: the CRC of the last valid entry in the index, used to verify data corruption.
	//   - newFileOffset: the new file offset after recovery, indicating where the next data should be written.
	//   - lastEntryOffset: the offset of the last valid entry in the recovered index.
	//   - err: an error if the recovery process encounters issues, such as data corruption or invalid entries.
	RecoverIndex(buf []byte, startFileOffset uint32, baseEntryOffset int64, commitOffset *int64) (index []byte,
		lastCrc uint32, newFileOffset uint32, lastEntryOffset int64, err error)
}

func GetOrCreate

func GetOrCreate(basePath string) (_codec Codec, exist bool, err error)

GetOrCreate checks if a file with the specified extension exists at the basePath to support compatible with the old codec versions.

type Metadata

type Metadata struct {
	TxnExtension  string
	IdxExtension  string
	HeaderSize    uint32
	IdxHeaderSize uint32
}

type V1

type V1 struct {
	Metadata
}

func (*V1) GetHeaderSize

func (v *V1) GetHeaderSize() uint32

func (*V1) GetIdxExtension

func (v *V1) GetIdxExtension() string

func (*V1) GetIndexHeaderSize

func (v *V1) GetIndexHeaderSize() uint32

func (*V1) GetRecordSize

func (v *V1) GetRecordSize(buf []byte, startFileOffset uint32) (payloadSize uint32, err error)

func (*V1) GetTxnExtension

func (v *V1) GetTxnExtension() string

func (*V1) ReadHeaderWithValidation

func (v *V1) ReadHeaderWithValidation(buf []byte, startFileOffset uint32) (payloadSize uint32, previousCrc uint32, payloadCrc uint32, err error)

func (*V1) ReadIndex

func (*V1) ReadIndex(path string) ([]byte, error)

func (*V1) ReadRecordWithValidation

func (v *V1) ReadRecordWithValidation(buf []byte, startFileOffset uint32) (payload []byte, err error)

func (*V1) RecoverIndex

func (v *V1) RecoverIndex(buf []byte, startFileOffset uint32, baseEntryOffset int64,
	_ *int64) (index []byte, lastCrc uint32, newFileOffset uint32, lastEntryOffset int64, err error)

func (*V1) WriteIndex

func (*V1) WriteIndex(path string, index []byte) error

func (*V1) WriteRecord

func (*V1) WriteRecord(buf []byte, startOffset uint32, _ uint32, payload []byte) (recordSize uint32, payloadCrc uint32)

type V2

type V2 struct {
	Metadata
}

func (*V2) GetHeaderSize

func (v *V2) GetHeaderSize() uint32

func (*V2) GetIdxExtension

func (v *V2) GetIdxExtension() string

func (*V2) GetIndexHeaderSize

func (v *V2) GetIndexHeaderSize() uint32

func (*V2) GetRecordSize

func (v *V2) GetRecordSize(buf []byte, startFileOffset uint32) (uint32, error)

func (*V2) GetTxnExtension

func (v *V2) GetTxnExtension() string

func (*V2) ReadHeaderWithValidation

func (v *V2) ReadHeaderWithValidation(buf []byte, startFileOffset uint32) (payloadSize uint32, previousCrc uint32, payloadCrc uint32, err error)

func (*V2) ReadIndex

func (v *V2) ReadIndex(path string) ([]byte, error)

func (*V2) ReadRecordWithValidation

func (v *V2) ReadRecordWithValidation(buf []byte, startFileOffset uint32) (payload []byte, err error)

func (*V2) RecoverIndex

func (v *V2) RecoverIndex(buf []byte, startFileOffset uint32, baseEntryOffset int64,
	commitOffset *int64) (index []byte, lastCrc uint32,
	newFileOffset uint32, lastEntryOffset int64, err error)

func (*V2) WriteIndex

func (v *V2) WriteIndex(path string, index []byte) error

func (*V2) WriteRecord

func (*V2) WriteRecord(buf []byte, startOffset uint32, previousCrc uint32, payload []byte) (recordSize uint32, payloadCrc uint32)