objectio

package
v0.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 29, 2022 License: Apache-2.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source 
const ExtentTypeSize = 4 * 3
View Source 
const ExtentsLength = 20
View Source 
const FSName = "local"
View Source 
const FooterSize = 8 + 4
View Source 
const Magic = 0xFFFFFFFF
View Source 
const Version = 1
View Source 
const ZoneMapMaxSize = 32
View Source 
const ZoneMapMinSize = 32

Variables

This section is empty.

Functions

func TmpNewFileservice 

func TmpNewFileservice(dir string) fileservice.FileService

Types

type Block 

type Block struct {
	// contains filtered or unexported fields
}

Block is the organizational structure of a batch in objectio Write one batch at a time, and batch and block correspond one-to-one

func (*Block) GetColumn 

func (b *Block) GetColumn(idx uint16) (ColumnObject, error)

func (*Block) GetExtent 

func (b *Block) GetExtent() Extent

func (*Block) GetID 

func (b *Block) GetID() uint32

func (*Block) GetMeta 

func (b *Block) GetMeta() BlockMeta

func (*Block) GetRows 

func (b *Block) GetRows() (uint32, error)

func (*Block) MarshalMeta 

func (b *Block) MarshalMeta() ([]byte, error)

func (*Block) UnMarshalMeta 

func (b *Block) UnMarshalMeta(data []byte) error

type BlockHeader 

type BlockHeader struct {
	// contains filtered or unexported fields
}

func (*BlockHeader) GetBlockId 

func (bh *BlockHeader) GetBlockId() uint64

func (*BlockHeader) GetColumnCount 

func (bh *BlockHeader) GetColumnCount() uint16

func (*BlockHeader) GetSegmentId 

func (bh *BlockHeader) GetSegmentId() uint64

func (*BlockHeader) GetTableId 

func (bh *BlockHeader) GetTableId() uint64

type BlockMeta 

type BlockMeta struct {
	// contains filtered or unexported fields
}

+---------------------------------------------------------------------------------------------+ | Header | +-------------+---------------+--------------+---------------+---------------+----------------+ | TableID(8B) | SegmentID(8B) | BlockID(8B) | ColumnCnt(2B) | Reserved(34B) | Chksum(4B) | +-------------+---------------+--------------+---------------+---------------+----------------+ | ColumnMeta | +---------------------------------------------------------------------------------------------+ | ColumnMeta | +---------------------------------------------------------------------------------------------+ | ColumnMeta | +---------------------------------------------------------------------------------------------+ | .......... | +---------------------------------------------------------------------------------------------+ Header Size = 64B TableID = Table ID for Block SegmentID = Segment ID BlockID = Block ID ColumnCnt = The number of column in the block Chksum = Block metadata checksum Reserved = 34 bytes reserved space

func (*BlockMeta) GetHeader 

func (bm *BlockMeta) GetHeader() BlockHeader

func (*BlockMeta) GetName 

func (bm *BlockMeta) GetName() string

type BlockObject 

type BlockObject interface {
	// GetColumn gets a ColumnObject with idx
	GetColumn(idx uint16) (ColumnObject, error)

	// GetRows gets the rows of the BlockObject
	GetRows() (uint32, error)

	// GetMeta gets the meta of the BlockObject
	GetMeta() BlockMeta

	// GetExtent gets the metadata location of BlockObject in fileservice
	GetExtent() Extent

	// GetID is to get the serial number of the block in the object
	GetID() uint32
}

BlockObject is a batch written to fileservice

func NewBlock 

func NewBlock(colCnt uint16, object *Object, name string) BlockObject

type BloomFilter 

type BloomFilter struct {
	// contains filtered or unexported fields
}

func (*BloomFilter) GetData 

func (b *BloomFilter) GetData() []byte

func (*BloomFilter) GetIdx 

func (b *BloomFilter) GetIdx() uint16

func (*BloomFilter) Write 

func (b *BloomFilter) Write(writer *ObjectWriter, block *Block) error

type ColumnBlock 

type ColumnBlock struct {
	// contains filtered or unexported fields
}

ColumnBlock is the organizational structure of a vector in objectio One batch can be written at a time, and a batch can contain multiple vectors. It is a child of the block

func (*ColumnBlock) GetData 

func (cb *ColumnBlock) GetData(ctx context.Context, m *mpool.MPool) (*fileservice.IOVector, error)

func (*ColumnBlock) GetIndex 

func (cb *ColumnBlock) GetIndex(ctx context.Context, dataType IndexDataType, m *mpool.MPool) (IndexData, error)

func (*ColumnBlock) GetMeta 

func (cb *ColumnBlock) GetMeta() *ColumnMeta

func (*ColumnBlock) MarshalMeta 

func (cb *ColumnBlock) MarshalMeta() ([]byte, error)

func (*ColumnBlock) UnMarshalMate 

func (cb *ColumnBlock) UnMarshalMate(cache *bytes.Buffer) error

type ColumnMeta 

type ColumnMeta struct {
	// contains filtered or unexported fields
}

+---------------------------------------------------------------------------------------------------------------+ | ColumnMeta | +--------+-------+----------+--------+---------+--------+--------+------------+---------+----------+------------+ |Type(1B)|Idx(2B)| Algo(1B) |Offset(4B)|Size(4B)|oSize(4B)|Min(32B)|Max(32B)|BFoffset(4b)|BFlen(4b)|BFoSize(4B) | +--------+-------+----------+--------+---------+--------+--------+------------+---------+----------+------------+ | Reserved(32B) | Chksum(4B) | +---------------------------------------------------------------------------------------------------------------+ ColumnMeta Size = 128B Type = Metadata type, always 0, representing column meta, used for extension. Idx = Column index Algo = Type of compression algorithm for column data Offset = Offset of column data Size = Size of column data oSize = Original data size Min = Column min value Max = Column Max value BFoffset = Bloomfilter data offset Bflen = Bloomfilter data size BFoSize = Bloomfilter original data size Chksum = Data checksum Reserved = 32 bytes reserved space

func (*ColumnMeta) GetAlg 

func (cm *ColumnMeta) GetAlg() uint8

func (*ColumnMeta) GetBloomFilter 

func (cm *ColumnMeta) GetBloomFilter() Extent

func (*ColumnMeta) GetIdx 

func (cm *ColumnMeta) GetIdx() uint16

func (*ColumnMeta) GetLocation 

func (cm *ColumnMeta) GetLocation() Extent

func (*ColumnMeta) GetType 

func (cm *ColumnMeta) GetType() uint8

func (*ColumnMeta) GetZoneMap 

func (cm *ColumnMeta) GetZoneMap() ZoneMap

type ColumnObject 

type ColumnObject interface {
	// GetData gets the data of ColumnObject
	// Returns an IOVector, the caller needs to traverse the IOVector
	// to get all the structures required for data generation
	GetData(ctx context.Context, m *mpool.MPool) (*fileservice.IOVector, error)

	// GetIndex gets the index of ColumnObject
	GetIndex(ctx context.Context, dataType IndexDataType, m *mpool.MPool) (IndexData, error)

	// GetMeta gets the metadata of ColumnObject
	GetMeta() *ColumnMeta
}

ColumnObject is a vector in a batch written to fileservice

func NewColumnBlock 

func NewColumnBlock(idx uint16, object *Object) ColumnObject

type Extent 

type Extent struct {
	// contains filtered or unexported fields
}

func NewExtent 

func NewExtent(offset, length, originSize uint32) Extent

func (Extent) End 

func (ex Extent) End() uint32

func (Extent) Id 

func (ex Extent) Id() uint64

func (Extent) Length 

func (ex Extent) Length() uint32

func (Extent) Offset 

func (ex Extent) Offset() uint32

func (Extent) OriginSize 

func (ex Extent) OriginSize() uint32
type Footer struct {
	// contains filtered or unexported fields
}

func (*Footer) UnMarshalFooter 

func (f *Footer) UnMarshalFooter(data []byte) error
type Header struct {
	// contains filtered or unexported fields
}

type IndexData 

type IndexData interface {
	Write(writer *ObjectWriter, block *Block) error
	GetIdx() uint16
}

func NewBloomFilter 

func NewBloomFilter(idx uint16, alg uint8, buf []byte) IndexData

func NewZoneMap 

func NewZoneMap(idx uint16, buf []byte) (IndexData, error)

type IndexDataType 

type IndexDataType uint8
const (
	ZoneMapType IndexDataType = iota
	BloomFilterType
)

type Object 

type Object struct {
	// contains filtered or unexported fields
}

func NewObject 

func NewObject(name string, fs fileservice.FileService) *Object

type ObjectBuffer 

type ObjectBuffer struct {
	// contains filtered or unexported fields
}

ObjectBuffer is the buffer prepared before writing to the object file, all data written to the object needs to be filled in it, and then written to the object file at one time

func NewObjectBuffer 

func NewObjectBuffer(name string) *ObjectBuffer

func (*ObjectBuffer) GetData 

func (b *ObjectBuffer) GetData() fileservice.IOVector

func (*ObjectBuffer) Length 

func (b *ObjectBuffer) Length() int

func (*ObjectBuffer) Write 

func (b *ObjectBuffer) Write(buf []byte) (int, int, error)

type ObjectFS 

type ObjectFS struct {
	Service fileservice.FileService
	Dir     string
}

func NewObjectFS 

func NewObjectFS(service fileservice.FileService, dir string) *ObjectFS

func (*ObjectFS) ListDir 

func (o *ObjectFS) ListDir(dir string) ([]fileservice.DirEntry, error)

type ObjectReader 

type ObjectReader struct {
	// contains filtered or unexported fields
}

func (*ObjectReader) Read 

func (r *ObjectReader) Read(ctx context.Context, extent Extent, idxs []uint16, m *mpool.MPool) (*fileservice.IOVector, error)

func (*ObjectReader) ReadAllMeta 

func (r *ObjectReader) ReadAllMeta(ctx context.Context, fileSize int64, m *mpool.MPool) ([]BlockObject, error)

func (*ObjectReader) ReadIndex 

func (r *ObjectReader) ReadIndex(ctx context.Context, extent Extent, idxs []uint16, typ IndexDataType, m *mpool.MPool) ([]IndexData, error)

func (*ObjectReader) ReadMeta 

func (r *ObjectReader) ReadMeta(ctx context.Context, extents []Extent, m *mpool.MPool) ([]BlockObject, error)

type ObjectWriter 

type ObjectWriter struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func (*ObjectWriter) AddBlock 

func (w *ObjectWriter) AddBlock(block *Block)

func (*ObjectWriter) GetBlock 

func (w *ObjectWriter) GetBlock(id uint32) *Block

func (*ObjectWriter) Sync 

func (w *ObjectWriter) Sync(ctx context.Context) error

Sync is for testing

func (*ObjectWriter) Write 

func (w *ObjectWriter) Write(batch *batch.Batch) (BlockObject, error)

func (*ObjectWriter) WriteEnd 

func (w *ObjectWriter) WriteEnd(ctx context.Context) ([]BlockObject, error)

func (*ObjectWriter) WriteHeader 

func (w *ObjectWriter) WriteHeader() error

func (*ObjectWriter) WriteIndex 

func (w *ObjectWriter) WriteIndex(fd BlockObject, index IndexData) error

type Reader 

type Reader interface {
	// Read is to read columns data of a block from fileservice at one time
	// extent is location of the block meta
	// idxs is the column serial number of the data to be read
	Read(ctx context.Context, extents Extent, idxs []uint16, m *mpool.MPool) (*fileservice.IOVector, error)

	// ReadMeta is the meta that reads a block
	// extent is location of the block meta
	ReadMeta(ctx context.Context, extent []Extent, m *mpool.MPool) ([]BlockObject, error)

	// ReadIndex is the index data of the read columns
	ReadIndex(ctx context.Context, extent Extent, idxs []uint16, dataType IndexDataType, m *mpool.MPool) ([]IndexData, error)

	// ReadAllMeta is read the meta of all blocks in an object
	ReadAllMeta(ctx context.Context, fileSize int64, m *mpool.MPool) ([]BlockObject, error)
}

Reader is to read data from fileservice

func NewObjectReader 

func NewObjectReader(name string, fs fileservice.FileService) (Reader, error)

type ToObjectFunc 

type ToObjectFunc = func(r io.Reader, buf []byte) (any, int64, error)

type Writer 

type Writer interface {
	// Write writes one batch to the Buffer at a time,
	// one batch corresponds to a virtual block,
	// and returns the handle of the block.
	Write(batch *batch.Batch) (BlockObject, error)

	// WriteIndex is the index of the column in the block written to the block's handle.
	// block is the handle of the block
	// idx is the column to which the index is written
	// buf is the data to write to the index
	WriteIndex(block BlockObject, index IndexData) error

	// WriteEnd is to write multiple batches written to
	// the buffer to the fileservice at one time
	WriteEnd(ctx context.Context) ([]BlockObject, error)
}

Writer is to virtualize batches into multiple blocks and write them into filefservice at one time

func NewObjectWriter 

func NewObjectWriter(name string, fs fileservice.FileService) (Writer, error)

type ZoneMap 

type ZoneMap struct {
	// contains filtered or unexported fields
}

func (*ZoneMap) GetData 

func (z *ZoneMap) GetData() []byte

func (*ZoneMap) GetIdx 

func (z *ZoneMap) GetIdx() uint16

func (*ZoneMap) Write 

func (z *ZoneMap) Write(_ *ObjectWriter, block *Block) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.