blockio

package
v1.2.3-hotfix-20241010 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 11, 2024 License: Apache-2.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

View Source
const (
	AsyncIo = 1
	SyncIo  = 2
)
View Source
const (
	JTLoad tasks.JobType = 200 + iota
	JTFlush
)
View Source
const (
	CheckpointExt = "ckp"
	GCFullExt     = "fgc"
	SnapshotExt   = "snap"
	AcctExt       = "acct"
	TmpExt        = "tmp"
)

Variables

View Source
var IoModel = SyncIo

Functions

func BlockCompactionRead

func BlockCompactionRead(
	ctx context.Context,
	location objectio.Location,
	deletes []int64,
	seqnums []uint16,
	colTypes []types.Type,
	fs fileservice.FileService,
	mp *mpool.MPool,
) (*batch.Batch, error)

func BlockPrefetch

func BlockPrefetch(idxes []uint16, service fileservice.FileService, infos [][]*objectio.BlockInfo, prefetchFile bool) error

BlockPrefetch is the interface for cn to call read ahead columns Which columns should be taken for columns service fileservice infos [s3object name][block]

func BlockRead

func BlockRead(
	ctx context.Context,
	info *objectio.BlockInfo,
	inputDeletes []int64,
	columns []uint16,
	colTypes []types.Type,
	ts timestamp.Timestamp,
	filterSeqnums []uint16,
	filterColTypes []types.Type,
	filter BlockReadFilter,
	fs fileservice.FileService,
	mp *mpool.MPool,
	vp engine.VectorPool,
	policy fileservice.Policy,
) (*batch.Batch, error)

BlockRead read block data from storage and apply deletes according given timestamp. Caller make sure metaloc is not empty

func BlockReadInner

func BlockReadInner(
	ctx context.Context,
	info *objectio.BlockInfo,
	inputDeleteRows []int64,
	columns []uint16,
	colTypes []types.Type,
	ts types.TS,
	selectRows []int32,
	fs fileservice.FileService,
	mp *mpool.MPool,
	vp engine.VectorPool,
	policy fileservice.Policy,
) (result *batch.Batch, err error)

func DecodeCheckpointMetadataFileName

func DecodeCheckpointMetadataFileName(name string) (start, end types.TS)

func DecodeGCMetadataFileName

func DecodeGCMetadataFileName(name string) (start, end types.TS, ext string)

func EncodeCheckpointMetadataFileName

func EncodeCheckpointMetadataFileName(dir, prefix string, start, end types.TS) string

func EncodeCheckpointMetadataFileNameWithoutDir added in v1.2.1

func EncodeCheckpointMetadataFileNameWithoutDir(prefix string, start, end types.TS) string

func EncodeGCMetadataFileName

func EncodeGCMetadataFileName(dir, prefix string, start, end types.TS) string

func EncodeLocation

func EncodeLocation(
	name objectio.ObjectName,
	extent objectio.Extent,
	rows uint32,
	id uint16) objectio.Location

EncodeLocation Generate a metaloc

func EncodeLocationFromString

func EncodeLocationFromString(info string) (objectio.Location, error)

EncodeLocationFromString Generate a metaloc from an info string

func EncodeSnapshotMetadataFileName added in v1.2.0

func EncodeSnapshotMetadataFileName(dir, prefix string, start, end types.TS) string

func EncodeTableMetadataFileName added in v1.2.0

func EncodeTableMetadataFileName(dir, prefix string, start, end types.TS) string

func EncodeTmpFileName added in v1.2.3

func EncodeTmpFileName(dir, prefix string, ts int64) string

func EvalDeleteRowsByTimestamp added in v1.2.0

func EvalDeleteRowsByTimestamp(deletes *batch.Batch, ts types.TS, blockid *types.Blockid) (rows *nulls.Bitmap)

func EvalDeleteRowsByTimestampForDeletesPersistedByCN added in v1.2.0

func EvalDeleteRowsByTimestampForDeletesPersistedByCN(deletes *batch.Batch, ts types.TS, committs types.TS) (rows *nulls.Bitmap)

func ExportSelectivityString

func ExportSelectivityString() string

func FindIntervalForBlock added in v1.0.0

func FindIntervalForBlock(rowids []types.Rowid, id *types.Blockid) (start int, end int)

func GetObjectSizeWithBlocks

func GetObjectSizeWithBlocks(blocks []objectio.BlockObject) (uint32, error)

func IsPersistedByCN added in v1.2.0

func IsPersistedByCN(ctx context.Context, deltaloc objectio.Location, fs fileservice.FileService) (bool, error)

func LoadColumns

func LoadColumns(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	m *mpool.MPool,
	policy fileservice.Policy,
) (bat *batch.Batch, release func(), err error)

func LoadColumns2 added in v1.2.0

func LoadColumns2(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	policy fileservice.Policy,
	needCopy bool,
	vPool *containers.VectorPool,
) (vectors []containers.Vector, release func(), err error)

LoadColumns2 load columns data from file service for TN need to copy data from vPool to avoid releasing cache

func LoadColumnsData added in v1.0.0

func LoadColumnsData(
	ctx context.Context,
	metaType objectio.DataMetaType,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	m *mpool.MPool,
	policy fileservice.Policy,
) (bat *batch.Batch, release func(), err error)

func LoadColumnsData2 added in v1.2.0

func LoadColumnsData2(
	ctx context.Context,
	metaType objectio.DataMetaType,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	policy fileservice.Policy,
	needCopy bool,
	vPool *containers.VectorPool,
) (vectors []containers.Vector, release func(), err error)

func LoadOneBlock added in v1.1.0

func LoadOneBlock(
	ctx context.Context,
	fs fileservice.FileService,
	key objectio.Location,
	metaType objectio.DataMetaType,
) (*batch.Batch, error)

func LoadTombstoneColumns added in v1.0.0

func LoadTombstoneColumns(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	m *mpool.MPool,
) (bat *batch.Batch, release func(), err error)

func LoadTombstoneColumns2 added in v1.2.0

func LoadTombstoneColumns2(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	needCopy bool,
	vPool *containers.VectorPool,
) (vectors []containers.Vector, release func(), err error)

LoadTombstoneColumns2 load tombstone data from file service for TN need to copy data from vPool to avoid releasing cache

func Prefetch

func Prefetch(idxes []uint16, ids []uint16, service fileservice.FileService, key objectio.Location) error

func PrefetchFile

func PrefetchFile(service fileservice.FileService, name string) error

func PrefetchMeta

func PrefetchMeta(service fileservice.FileService, key objectio.Location) error

func PrefetchTombstone added in v1.0.0

func PrefetchTombstone(idxes []uint16, ids []uint16, service fileservice.FileService, key objectio.Location) error

func PrefetchWithMerged

func PrefetchWithMerged(params PrefetchParams) error

The caller has merged the block information that needs to be prefetched

func ReadBlockDelete added in v1.0.0

func ReadBlockDelete(ctx context.Context, deltaloc objectio.Location, fs fileservice.FileService) (bat *batch.Batch, isPersistedByCN bool, release func(), err error)

func ReadBlockDeleteBySchema added in v1.2.0

func ReadBlockDeleteBySchema(ctx context.Context, deltaloc objectio.Location, fs fileservice.FileService, isPersistedByCN bool) (bat *batch.Batch, release func(), err error)

func ReadByFilter added in v1.0.0

func ReadByFilter(
	ctx context.Context,
	info *objectio.BlockInfo,
	inputDeletes []int64,
	columns []uint16,
	colTypes []types.Type,
	ts types.TS,
	searchFunc ReadFilterSearchFuncType,
	fs fileservice.FileService,
	mp *mpool.MPool,
) (sels []int32, err error)

func RecordBlockSelectivity

func RecordBlockSelectivity(hit, total int)

func RecordColumnSelectivity

func RecordColumnSelectivity(hit, total int)

func RecordReadDel added in v1.0.0

func RecordReadDel(total, read, bisect time.Duration)

func RecordReadFilterSelectivity

func RecordReadFilterSelectivity(hit, total int)

func ResetPipeline

func ResetPipeline()

func Start

func Start()

func Stop

func Stop()

func UpdateGCMetadataFileName added in v1.2.2

func UpdateGCMetadataFileName(name string, start, end types.TS) string

Types

type BlockReadFilter added in v1.2.1

type BlockReadFilter struct {
	HasFakePK          bool
	Valid              bool
	SortedSearchFunc   ReadFilterSearchFuncType
	UnSortedSearchFunc ReadFilterSearchFuncType
}

type BlockReader

type BlockReader struct {
	// contains filtered or unexported fields
}

func NewFileReader

func NewFileReader(service fileservice.FileService, name string) (*BlockReader, error)

func NewFileReaderNoCache

func NewFileReaderNoCache(service fileservice.FileService, name string) (*BlockReader, error)

func NewObjectReader

func NewObjectReader(
	service fileservice.FileService,
	key objectio.Location,
	opts ...objectio.ReaderOptionFunc,
) (*BlockReader, error)

func (*BlockReader) GetName

func (r *BlockReader) GetName() string

func (*BlockReader) GetObjectName

func (r *BlockReader) GetObjectName() *objectio.ObjectName

func (*BlockReader) GetObjectReader

func (r *BlockReader) GetObjectReader() *objectio.ObjectReader

func (*BlockReader) LoadAllBF

func (r *BlockReader) LoadAllBF(
	ctx context.Context,
) (objectio.BloomFilter, uint32, error)

func (*BlockReader) LoadAllBlocks

func (r *BlockReader) LoadAllBlocks(ctx context.Context, m *mpool.MPool) ([]objectio.BlockObject, error)

func (*BlockReader) LoadAllColumns

func (r *BlockReader) LoadAllColumns(
	ctx context.Context,
	idxs []uint16,
	m *mpool.MPool,
) ([]*batch.Batch, func(), error)

func (*BlockReader) LoadColumns

func (r *BlockReader) LoadColumns(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	blk uint16,
	m *mpool.MPool,
) (bat *batch.Batch, release func(), err error)

LoadColumns needs typs to generate columns, if the target table has no schema change, nil can be passed.

func (*BlockReader) LoadObjectMeta

func (r *BlockReader) LoadObjectMeta(ctx context.Context, m *mpool.MPool) (objectio.ObjectDataMeta, error)

func (*BlockReader) LoadOneBF

func (r *BlockReader) LoadOneBF(
	ctx context.Context,
	blk uint16,
) (objectio.StaticFilter, uint32, error)

func (*BlockReader) LoadOneSubColumns added in v1.0.0

func (r *BlockReader) LoadOneSubColumns(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	dataType uint16,
	blk uint16,
	m *mpool.MPool,
) (bat *batch.Batch, release func(), err error)

LoadColumns needs typs to generate columns, if the target table has no schema change, nil can be passed.

func (*BlockReader) LoadSubColumns added in v1.0.0

func (r *BlockReader) LoadSubColumns(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	blk uint16,
	m *mpool.MPool,
) (bats []*batch.Batch, releases func(), err error)

LoadColumns needs typs to generate columns, if the target table has no schema change, nil can be passed.

func (*BlockReader) LoadZoneMap

func (r *BlockReader) LoadZoneMap(
	ctx context.Context,
	seqnums []uint16,
	block objectio.BlockObject,
	m *mpool.MPool) ([]objectio.ZoneMap, error)

func (*BlockReader) LoadZoneMaps

func (r *BlockReader) LoadZoneMaps(
	ctx context.Context,
	seqnums []uint16,
	id uint16,
	m *mpool.MPool,
) ([]objectio.ZoneMap, error)

type BlockWriter

type BlockWriter struct {
	// contains filtered or unexported fields
}

func NewBlockWriter

func NewBlockWriter(fs fileservice.FileService, name string) (*BlockWriter, error)

func NewBlockWriterNew

func NewBlockWriterNew(fs fileservice.FileService, name objectio.ObjectName, schemaVer uint32, seqnums []uint16) (*BlockWriter, error)

seqnums is the column's seqnums of the batch written by `WriteBatch`. `WriteBatchWithoutIndex` will ignore the seqnums

func (*BlockWriter) GetName

func (w *BlockWriter) GetName() objectio.ObjectName

func (*BlockWriter) GetObjectStats added in v1.1.0

func (w *BlockWriter) GetObjectStats() []objectio.ObjectStats

func (*BlockWriter) SetAppendable added in v1.1.0

func (w *BlockWriter) SetAppendable()

func (*BlockWriter) SetPrimaryKey

func (w *BlockWriter) SetPrimaryKey(idx uint16)

func (*BlockWriter) SetSortKey added in v1.1.0

func (w *BlockWriter) SetSortKey(idx uint16)

func (*BlockWriter) Stats added in v1.1.0

func (w *BlockWriter) Stats() objectio.ObjectStats

func (*BlockWriter) String

func (w *BlockWriter) String(
	blocks []objectio.BlockObject) string

func (*BlockWriter) Sync

func (*BlockWriter) WriteBatch

func (w *BlockWriter) WriteBatch(batch *batch.Batch) (objectio.BlockObject, error)

WriteBatch write a batch whose schema is decribed by seqnum in NewBlockWriterNew

func (*BlockWriter) WriteSubBatch added in v1.0.0

func (w *BlockWriter) WriteSubBatch(batch *batch.Batch, dataType objectio.DataMetaType) (objectio.BlockObject, int, error)

func (*BlockWriter) WriteTombstoneBatch added in v1.0.0

func (w *BlockWriter) WriteTombstoneBatch(batch *batch.Batch) (objectio.BlockObject, error)

type FetchFunc

type FetchFunc = func(ctx context.Context, params fetchParams) (any, error)

type IOJobFactory

type IOJobFactory func(context.Context, fetchParams) *tasks.Job

type IoPipeline

type IoPipeline struct {
	// contains filtered or unexported fields
}

func NewIOPipeline

func NewIOPipeline(
	opts ...Option,
) *IoPipeline

func (*IoPipeline) Fetch

func (p *IoPipeline) Fetch(
	ctx context.Context,
	params fetchParams,
) (res any, err error)

func (*IoPipeline) Prefetch

func (p *IoPipeline) Prefetch(params PrefetchParams) (err error)

func (*IoPipeline) Start

func (p *IoPipeline) Start()

func (*IoPipeline) Stop

func (p *IoPipeline) Stop()

type ObjectColumnMetasBuilder

type ObjectColumnMetasBuilder struct {
	// contains filtered or unexported fields
}

func NewObjectColumnMetasBuilder

func NewObjectColumnMetasBuilder(colIdx int) *ObjectColumnMetasBuilder

func (*ObjectColumnMetasBuilder) AddPKData

func (b *ObjectColumnMetasBuilder) AddPKData(data containers.Vector)

func (*ObjectColumnMetasBuilder) AddRowCnt

func (b *ObjectColumnMetasBuilder) AddRowCnt(rows int)

func (*ObjectColumnMetasBuilder) Build

func (*ObjectColumnMetasBuilder) GetPKData

func (b *ObjectColumnMetasBuilder) GetPKData() []containers.Vector

func (*ObjectColumnMetasBuilder) GetTotalRow

func (b *ObjectColumnMetasBuilder) GetTotalRow() uint32

func (*ObjectColumnMetasBuilder) InspectVector

func (b *ObjectColumnMetasBuilder) InspectVector(idx int, vec containers.Vector, isPK bool)

func (*ObjectColumnMetasBuilder) SetPKNdv

func (b *ObjectColumnMetasBuilder) SetPKNdv(idx uint16, ndv uint32)

func (*ObjectColumnMetasBuilder) UpdateZm

func (b *ObjectColumnMetasBuilder) UpdateZm(idx int, zm index.ZM)

type Option

type Option func(*IoPipeline)

func WithFetchParallism

func WithFetchParallism(num int) Option

func WithJobFactory

func WithJobFactory(factory IOJobFactory) Option

func WithPrefetchParallism

func WithPrefetchParallism(num int) Option

type PrefetchFunc

type PrefetchFunc = func(params PrefetchParams) error

type PrefetchParams added in v1.0.0

type PrefetchParams struct {
	// contains filtered or unexported fields
}

PrefetchParams is the parameter of the executed IoPipeline.PrefetchParams, which provides the merge function, which can merge the PrefetchParams requests of multiple blocks in an object/file

func BuildPrefetchParams

func BuildPrefetchParams(service fileservice.FileService, key objectio.Location) (PrefetchParams, error)

func (*PrefetchParams) AddBlock added in v1.0.0

func (p *PrefetchParams) AddBlock(idxes []uint16, ids []uint16)

func (*PrefetchParams) AddBlockWithType added in v1.0.0

func (p *PrefetchParams) AddBlockWithType(idxes []uint16, ids []uint16, dataType uint16)

type ReadFilterSearchFuncType added in v1.2.1

type ReadFilterSearchFuncType func([]*vector.Vector) []int32

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL