blockio

package
v1.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 17, 2024 License: Apache-2.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	AsyncIo = 1
	SyncIo  = 2
)
View Source 
const (
	JTLoad tasks.JobType = 200 + iota
	JTFlush
)
View Source 
const (
	CheckpointExt = "ckp"
	GCFullExt     = "fgc"
	SnapshotExt   = "snap"
	AcctExt       = "acct"
)

Variables

View Source 
var IoModel = SyncIo

Functions

func BlockCompactionRead 

func BlockCompactionRead(
	ctx context.Context,
	location objectio.Location,
	deletes []int64,
	seqnums []uint16,
	colTypes []types.Type,
	fs fileservice.FileService,
	mp *mpool.MPool,
) (*batch.Batch, error)

func BlockPrefetch 

func BlockPrefetch(idxes []uint16, service fileservice.FileService, infos [][]*objectio.BlockInfo, prefetchFile bool) error

BlockPrefetch is the interface for cn to call read ahead columns Which columns should be taken for columns service fileservice infos [s3object name][block]

func BlockRead 

func BlockRead(
	ctx context.Context,
	info *objectio.BlockInfo,
	inputDeletes []int64,
	columns []uint16,
	colTypes []types.Type,
	ts timestamp.Timestamp,
	filterSeqnums []uint16,
	filterColTypes []types.Type,
	filter ReadFilter,
	fs fileservice.FileService,
	mp *mpool.MPool,
	vp engine.VectorPool,
	policy fileservice.Policy,
) (*batch.Batch, error)

BlockRead read block data from storage and apply deletes according given timestamp. Caller make sure metaloc is not empty

func BlockReadInner 

func BlockReadInner(
	ctx context.Context,
	info *objectio.BlockInfo,
	inputDeleteRows []int64,
	columns []uint16,
	colTypes []types.Type,
	ts types.TS,
	selectRows []int32,
	fs fileservice.FileService,
	mp *mpool.MPool,
	vp engine.VectorPool,
	policy fileservice.Policy,
) (result *batch.Batch, err error)

func DecodeCheckpointMetadataFileName 

func DecodeCheckpointMetadataFileName(name string) (start, end types.TS)

func DecodeGCMetadataFileName 

func DecodeGCMetadataFileName(name string) (start, end types.TS, ext string)

func EncodeCheckpointMetadataFileName 

func EncodeCheckpointMetadataFileName(dir, prefix string, start, end types.TS) string

func EncodeGCMetadataFileName 

func EncodeGCMetadataFileName(dir, prefix string, start, end types.TS) string

func EncodeLocation 

func EncodeLocation(
	name objectio.ObjectName,
	extent objectio.Extent,
	rows uint32,
	id uint16) objectio.Location

EncodeLocation Generate a metaloc

func EncodeLocationFromString 

func EncodeLocationFromString(info string) (objectio.Location, error)

EncodeLocationFromString Generate a metaloc from an info string

func EncodeSnapshotMetadataFileName added in v1.2.0 

func EncodeSnapshotMetadataFileName(dir, prefix string, start, end types.TS) string

func EncodeTableMetadataFileName added in v1.2.0 

func EncodeTableMetadataFileName(dir, prefix string, start, end types.TS) string

func EvalDeleteRowsByTimestamp added in v1.2.0 

func EvalDeleteRowsByTimestamp(deletes *batch.Batch, ts types.TS, blockid *types.Blockid) (rows *nulls.Bitmap)

func EvalDeleteRowsByTimestampForDeletesPersistedByCN added in v1.2.0 

func EvalDeleteRowsByTimestampForDeletesPersistedByCN(deletes *batch.Batch, ts types.TS, committs types.TS) (rows *nulls.Bitmap)

func ExportSelectivityString 

func ExportSelectivityString() string

func FindIntervalForBlock added in v1.0.0 

func FindIntervalForBlock(rowids []types.Rowid, id *types.Blockid) (start int, end int)

func GetObjectSizeWithBlocks 

func GetObjectSizeWithBlocks(blocks []objectio.BlockObject) (uint32, error)

func IsPersistedByCN added in v1.2.0 

func IsPersistedByCN(ctx context.Context, deltaloc objectio.Location, fs fileservice.FileService) (bool, error)

func LoadColumns 

func LoadColumns(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	m *mpool.MPool,
	policy fileservice.Policy,
) (bat *batch.Batch, release func(), err error)

func LoadColumns2 added in v1.2.0 

func LoadColumns2(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	policy fileservice.Policy,
	needCopy bool,
	vPool *containers.VectorPool,
) (vectors []containers.Vector, release func(), err error)

LoadColumns2 load columns data from file service for TN need to copy data from vPool to avoid releasing cache

func LoadColumnsData added in v1.0.0 

func LoadColumnsData(
	ctx context.Context,
	metaType objectio.DataMetaType,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	m *mpool.MPool,
	policy fileservice.Policy,
) (bat *batch.Batch, release func(), err error)

func LoadColumnsData2 added in v1.2.0 

func LoadColumnsData2(
	ctx context.Context,
	metaType objectio.DataMetaType,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	policy fileservice.Policy,
	needCopy bool,
	vPool *containers.VectorPool,
) (vectors []containers.Vector, release func(), err error)

func LoadOneBlock added in v1.1.0 

func LoadOneBlock(
	ctx context.Context,
	fs fileservice.FileService,
	key objectio.Location,
	metaType objectio.DataMetaType,
) (*batch.Batch, error)

func LoadTombstoneColumns added in v1.0.0 

func LoadTombstoneColumns(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	m *mpool.MPool,
) (bat *batch.Batch, release func(), err error)

func LoadTombstoneColumns2 added in v1.2.0 

func LoadTombstoneColumns2(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	fs fileservice.FileService,
	location objectio.Location,
	needCopy bool,
	vPool *containers.VectorPool,
) (vectors []containers.Vector, release func(), err error)

LoadTombstoneColumns2 load tombstone data from file service for TN need to copy data from vPool to avoid releasing cache

func Prefetch 

func Prefetch(idxes []uint16, ids []uint16, service fileservice.FileService, key objectio.Location) error

func PrefetchFile 

func PrefetchFile(service fileservice.FileService, name string) error

func PrefetchMeta 

func PrefetchMeta(service fileservice.FileService, key objectio.Location) error

func PrefetchTombstone added in v1.0.0 

func PrefetchTombstone(idxes []uint16, ids []uint16, service fileservice.FileService, key objectio.Location) error

func PrefetchWithMerged 

func PrefetchWithMerged(params PrefetchParams) error

The caller has merged the block information that needs to be prefetched

func ReadBlockDelete added in v1.0.0 

func ReadBlockDelete(ctx context.Context, deltaloc objectio.Location, fs fileservice.FileService) (bat *batch.Batch, isPersistedByCN bool, release func(), err error)

func ReadBlockDeleteBySchema added in v1.2.0 

func ReadBlockDeleteBySchema(ctx context.Context, deltaloc objectio.Location, fs fileservice.FileService, isPersistedByCN bool) (bat *batch.Batch, release func(), err error)

func ReadByFilter added in v1.0.0 

func ReadByFilter(
	ctx context.Context,
	info *objectio.BlockInfo,
	inputDeletes []int64,
	columns []uint16,
	colTypes []types.Type,
	ts types.TS,
	filter ReadFilter,
	fs fileservice.FileService,
	mp *mpool.MPool,
) (sels []int32, err error)

func RecordBlockSelectivity 

func RecordBlockSelectivity(hit, total int)

func RecordColumnSelectivity 

func RecordColumnSelectivity(hit, total int)

func RecordReadDel added in v1.0.0 

func RecordReadDel(total, read, bisect time.Duration)

func RecordReadFilterSelectivity 

func RecordReadFilterSelectivity(hit, total int)

func ResetPipeline 

func ResetPipeline()

func Start 

func Start()

func Stop 

func Stop()

Types

type BlockReader 

type BlockReader struct {
	// contains filtered or unexported fields
}

func NewFileReader 

func NewFileReader(service fileservice.FileService, name string) (*BlockReader, error)

func NewFileReaderNoCache 

func NewFileReaderNoCache(service fileservice.FileService, name string) (*BlockReader, error)

func NewObjectReader 

func NewObjectReader(
	service fileservice.FileService,
	key objectio.Location,
	opts ...objectio.ReaderOptionFunc,
) (*BlockReader, error)

func (*BlockReader) GetName 

func (r *BlockReader) GetName() string

func (*BlockReader) GetObjectName 

func (r *BlockReader) GetObjectName() *objectio.ObjectName

func (*BlockReader) GetObjectReader 

func (r *BlockReader) GetObjectReader() *objectio.ObjectReader

func (*BlockReader) LoadAllBF 

func (r *BlockReader) LoadAllBF(
	ctx context.Context,
) (objectio.BloomFilter, uint32, error)

func (*BlockReader) LoadAllBlocks 

func (r *BlockReader) LoadAllBlocks(ctx context.Context, m *mpool.MPool) ([]objectio.BlockObject, error)

func (*BlockReader) LoadAllColumns 

func (r *BlockReader) LoadAllColumns(
	ctx context.Context,
	idxs []uint16,
	m *mpool.MPool,
) ([]*batch.Batch, func(), error)

func (*BlockReader) LoadColumns 

func (r *BlockReader) LoadColumns(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	blk uint16,
	m *mpool.MPool,
) (bat *batch.Batch, release func(), err error)

LoadColumns needs typs to generate columns, if the target table has no schema change, nil can be passed.

func (*BlockReader) LoadObjectMeta 

func (r *BlockReader) LoadObjectMeta(ctx context.Context, m *mpool.MPool) (objectio.ObjectDataMeta, error)

func (*BlockReader) LoadOneBF 

func (r *BlockReader) LoadOneBF(
	ctx context.Context,
	blk uint16,
) (objectio.StaticFilter, uint32, error)

func (*BlockReader) LoadOneSubColumns added in v1.0.0 

func (r *BlockReader) LoadOneSubColumns(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	dataType uint16,
	blk uint16,
	m *mpool.MPool,
) (bat *batch.Batch, release func(), err error)

LoadColumns needs typs to generate columns, if the target table has no schema change, nil can be passed.

func (*BlockReader) LoadSubColumns added in v1.0.0 

func (r *BlockReader) LoadSubColumns(
	ctx context.Context,
	cols []uint16,
	typs []types.Type,
	blk uint16,
	m *mpool.MPool,
) (bats []*batch.Batch, releases func(), err error)

LoadColumns needs typs to generate columns, if the target table has no schema change, nil can be passed.

func (*BlockReader) LoadZoneMap 

func (r *BlockReader) LoadZoneMap(
	ctx context.Context,
	seqnums []uint16,
	block objectio.BlockObject,
	m *mpool.MPool) ([]objectio.ZoneMap, error)

func (*BlockReader) LoadZoneMaps 

func (r *BlockReader) LoadZoneMaps(
	ctx context.Context,
	seqnums []uint16,
	id uint16,
	m *mpool.MPool,
) ([]objectio.ZoneMap, error)

type BlockWriter 

type BlockWriter struct {
	// contains filtered or unexported fields
}

func NewBlockWriter 

func NewBlockWriter(fs fileservice.FileService, name string) (*BlockWriter, error)

func NewBlockWriterNew 

func NewBlockWriterNew(fs fileservice.FileService, name objectio.ObjectName, schemaVer uint32, seqnums []uint16) (*BlockWriter, error)

seqnums is the column's seqnums of the batch written by `WriteBatch`. `WriteBatchWithoutIndex` will ignore the seqnums

func (*BlockWriter) GetName 

func (w *BlockWriter) GetName() objectio.ObjectName

func (*BlockWriter) GetObjectStats added in v1.1.0 

func (w *BlockWriter) GetObjectStats() []objectio.ObjectStats

func (*BlockWriter) SetAppendable added in v1.1.0 

func (w *BlockWriter) SetAppendable()

func (*BlockWriter) SetPrimaryKey 

func (w *BlockWriter) SetPrimaryKey(idx uint16)

func (*BlockWriter) SetSortKey added in v1.1.0 

func (w *BlockWriter) SetSortKey(idx uint16)

func (*BlockWriter) Stats added in v1.1.0 

func (w *BlockWriter) Stats() objectio.ObjectStats

func (*BlockWriter) String 

func (w *BlockWriter) String(
	blocks []objectio.BlockObject) string

func (*BlockWriter) Sync 

func (w *BlockWriter) Sync(ctx context.Context) ([]objectio.BlockObject, objectio.Extent, error)

func (*BlockWriter) WriteBatch 

func (w *BlockWriter) WriteBatch(batch *batch.Batch) (objectio.BlockObject, error)

WriteBatch write a batch whose schema is decribed by seqnum in NewBlockWriterNew

func (*BlockWriter) WriteSubBatch added in v1.0.0 

func (w *BlockWriter) WriteSubBatch(batch *batch.Batch, dataType objectio.DataMetaType) (objectio.BlockObject, int, error)

func (*BlockWriter) WriteTombstoneBatch added in v1.0.0 

func (w *BlockWriter) WriteTombstoneBatch(batch *batch.Batch) (objectio.BlockObject, error)

type FetchFunc 

type FetchFunc = func(ctx context.Context, params fetchParams) (any, error)

type IOJobFactory 

type IOJobFactory func(context.Context, fetchParams) *tasks.Job

type IoPipeline 

type IoPipeline struct {
	// contains filtered or unexported fields
}

func NewIOPipeline 

func NewIOPipeline(
	opts ...Option,
) *IoPipeline

func (*IoPipeline) Fetch 

func (p *IoPipeline) Fetch(
	ctx context.Context,
	params fetchParams,
) (res any, err error)

func (*IoPipeline) Prefetch 

func (p *IoPipeline) Prefetch(params PrefetchParams) (err error)

func (*IoPipeline) Start 

func (p *IoPipeline) Start()

func (*IoPipeline) Stop 

func (p *IoPipeline) Stop()

type ObjectColumnMetasBuilder 

type ObjectColumnMetasBuilder struct {
	// contains filtered or unexported fields
}

func NewObjectColumnMetasBuilder 

func NewObjectColumnMetasBuilder(colIdx int) *ObjectColumnMetasBuilder

func (*ObjectColumnMetasBuilder) AddPKData 

func (b *ObjectColumnMetasBuilder) AddPKData(data containers.Vector)

func (*ObjectColumnMetasBuilder) AddRowCnt 

func (b *ObjectColumnMetasBuilder) AddRowCnt(rows int)

func (*ObjectColumnMetasBuilder) Build 

func (b *ObjectColumnMetasBuilder) Build() (uint32, []objectio.ColumnMeta)

func (*ObjectColumnMetasBuilder) GetPKData 

func (b *ObjectColumnMetasBuilder) GetPKData() []containers.Vector

func (*ObjectColumnMetasBuilder) GetTotalRow 

func (b *ObjectColumnMetasBuilder) GetTotalRow() uint32

func (*ObjectColumnMetasBuilder) InspectVector 

func (b *ObjectColumnMetasBuilder) InspectVector(idx int, vec containers.Vector, isPK bool)

func (*ObjectColumnMetasBuilder) SetPKNdv 

func (b *ObjectColumnMetasBuilder) SetPKNdv(idx uint16, ndv uint32)

func (*ObjectColumnMetasBuilder) UpdateZm 

func (b *ObjectColumnMetasBuilder) UpdateZm(idx int, zm index.ZM)

type Option 

type Option func(*IoPipeline)

func WithFetchParallism 

func WithFetchParallism(num int) Option

func WithJobFactory 

func WithJobFactory(factory IOJobFactory) Option

func WithPrefetchParallism 

func WithPrefetchParallism(num int) Option

type PrefetchFunc 

type PrefetchFunc = func(params PrefetchParams) error

type PrefetchParams added in v1.0.0 

type PrefetchParams struct {
	// contains filtered or unexported fields
}

PrefetchParams is the parameter of the executed IoPipeline.PrefetchParams, which provides the merge function, which can merge the PrefetchParams requests of multiple blocks in an object/file

func BuildPrefetchParams 

func BuildPrefetchParams(service fileservice.FileService, key objectio.Location) (PrefetchParams, error)

func (*PrefetchParams) AddBlock added in v1.0.0 

func (p *PrefetchParams) AddBlock(idxes []uint16, ids []uint16)

func (*PrefetchParams) AddBlockWithType added in v1.0.0 

func (p *PrefetchParams) AddBlockWithType(idxes []uint16, ids []uint16, dataType uint16)

type ReadFilter 

type ReadFilter = func([]*vector.Vector) []int32

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.