codec

package
v0.0.0-...-2588cc6 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 13, 2024 License: Apache-2.0 Imports: 12 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// The magic number of the committed data file.
	CommittedDataFileMagicNumber = uint32(0x30541989)
	CommittedDataFileVersion     = uint32(1)
)
View Source
const (
	CommittedManifestName    string = "manifest"
	CommittedManifestMagic   string = "TISTREAM-COMMITTED-MANIFEST"
	CommittedManifestVersion uint32 = 1
)
View Source
const (
	CommittedRangeIndexName = "committed_range_index"
	// The magic number of the committed range index file.
	CommittedRangeIndexMagicNumber = uint32(0x30541990)
	CommittedRangeIndexVersion     = uint32(1)
)
View Source
const (
	// The magic number of the prewrite buffer file.
	PrewriteBufferSnapFileMagicNumber = uint32(0x21647911)
	PrewriteBufferSnapFileVersion     = uint32(1)
)
View Source
const (
	RangeWatermarksSnapMagicNumber = uint32(0x81081793)
	RangeWatermarksSnapFileVersion = uint32(1)
)

The magic number of the range watermarks snap file.

View Source
const (
	SchemaFileMagicHeader = string("TiStreamSchemaSnap")
	SchemaFileVersion     = 1
)
View Source
const (
	// The magic number of the sorter buffer snap file.
	SorterSnapMagicNumber = uint32(0x81081792)
	SorterSnapFileVersion = uint32(1)
)

Variables

This section is empty.

Functions

func CalcChecksum

func CalcChecksum(data []byte) uint32

func GetCommittedRangeIndex

func GetCommittedRangeIndex(fdata []byte, tenantId uint64, range_ *pb.Task_Range) ([]uint64, error)

Types

type CommittedDataDecoder

type CommittedDataDecoder struct {
	// The tenant id.
	TenantID uint64

	// range of the file.
	Range *pb.Task_Range
}

func NewCommittedDataDecoder

func NewCommittedDataDecoder(tenantID uint64, range_ *pb.Task_Range) *CommittedDataDecoder

func (*CommittedDataDecoder) DecodeFrom

func (c *CommittedDataDecoder) DecodeFrom(r io.Reader) ([]*pb.EventRow, error)

type CommittedDataEncoder

type CommittedDataEncoder struct {

	// The range of the file.
	Range *pb.Task_Range
	// contains filtered or unexported fields
}

func NewCommittedDataEncoder

func NewCommittedDataEncoder(tenantID uint64, range_ *pb.Task_Range, lowWatermark, highWatermark uint64) *CommittedDataEncoder

func (*CommittedDataEncoder) Encode

func (c *CommittedDataEncoder) Encode(w io.Writer, events []*pb.EventRow) error

type CommittedManifest

type CommittedManifest struct {
	sync.RWMutex
	// The tenant id.
	TenantID uint64

	// Root directory of the committed data pool for the tenant.
	// In above file org it should be "{prefix}committed_data_pool/Tenant-{1}/"
	RootDir string

	// Backend Storage
	BackendStorage storage.ExternalStorage

	// Ranges Snapshots in different ts.
	// ts -> *pb.RangesSnapshot
	Snapshots []*pb.RangesSnapshot
	// contains filtered or unexported fields
}

func NewCommittedDataPoolManifest

func NewCommittedDataPoolManifest(tenantID uint64, rootDir string, backendStorage storage.ExternalStorage) *CommittedManifest

NewCommittedDataPoolManifest creates a new CommittedDataPoolManifest.

func (*CommittedManifest) AddSnapshot

func (m *CommittedManifest) AddSnapshot(ts uint64, snapshot *pb.RangesSnapshot) error

AddSnapshot adds a snapshot to the committed manifest.

func (*CommittedManifest) CompactTo

func (m *CommittedManifest) CompactTo(ts uint64) bool

Compact the committed manifest. Return true if the manifest is compacted, otherwise return false.

func (*CommittedManifest) Load

func (m *CommittedManifest) Load() error

Load from file.

func (*CommittedManifest) Save

func (m *CommittedManifest) Save() error

Save to file.

type CommittedManifestDecoder

type CommittedManifestDecoder struct {
	// The tenant id.
	TenantID uint64
}

func NewCommittedManifestDecoder

func NewCommittedManifestDecoder(tenantID uint64) *CommittedManifestDecoder

func (*CommittedManifestDecoder) Decode

func (*CommittedManifestDecoder) DecodeV1

type CommittedManifestEncoder

type CommittedManifestEncoder struct {
	// The tenant id.
	TenantID uint64
}

func NewCommittedManifestEncoder

func NewCommittedManifestEncoder(tenantID uint64) *CommittedManifestEncoder

func (*CommittedManifestEncoder) Encode

func (e *CommittedManifestEncoder) Encode(w io.Writer, snapshots []*pb.RangesSnapshot) error

type CommittedRangeIndexDecoder

type CommittedRangeIndexDecoder struct {
	// The tenant id.
	TenantID uint64

	// The range of the file.
	Range *pb.Task_Range
}

func NewCommittedRangeIndexDecoder

func NewCommittedRangeIndexDecoder(tenantID uint64, range_ *pb.Task_Range) *CommittedRangeIndexDecoder

func (*CommittedRangeIndexDecoder) Decode

func (c *CommittedRangeIndexDecoder) Decode(r io.Reader) ([]uint64, error)

type CommittedRangeIndexEncoder

type CommittedRangeIndexEncoder struct {
	// The tenant id.
	TenantID uint64

	// The range of the file.
	Range *pb.Task_Range

	// All sub folders
	TsArray []uint64
}

func NewCommittedRangeIndexEncoder

func NewCommittedRangeIndexEncoder(tenantID uint64, range_ *pb.Task_Range, tsArray []uint64) *CommittedRangeIndexEncoder

func (*CommittedRangeIndexEncoder) Encode

type DataChangesFileDecoder

type DataChangesFileDecoder struct {
	TenantID uint64

	// All the event rows in the file.
	EventRows []*pb.EventRow

	// The event watermark in the file.
	EventWatermarks []*pb.EventWatermark

	// DDL changes
	DdlChanges []*pb.DDLChange
}

Data change file is the file to store the data change. The file payload is as follows: Header (24 bytes) | File version (4 bytes) | Event size (4 bytes) | Event batch (event size bytes) | ... | Checksum (4 bytes)

func NewDataChangesFileDecoder

func NewDataChangesFileDecoder(tenantId uint64) *DataChangesFileDecoder

NewDataChangesFileDecoder creates a new DataChanges.

func (*DataChangesFileDecoder) DecodeFrom

func (f *DataChangesFileDecoder) DecodeFrom(reader io.Reader) error

DecodeFrom decodes the data change from the reader.

type FileEvent

type FileEvent = DataChangesFileDecoder

type PrewriteBufferSnapDecoder

type PrewriteBufferSnapDecoder struct {
	// The tenant id.
	TenantID uint64

	// range of the buffer.
	Range *pb.Task_Range
}

func NewPrewriteBufferSnapDecoder

func NewPrewriteBufferSnapDecoder(tenantID uint64, range_ *pb.Task_Range) *PrewriteBufferSnapDecoder

func (*PrewriteBufferSnapDecoder) Decode

func (p *PrewriteBufferSnapDecoder) Decode(r io.Reader) ([]*pb.EventRow, []*pb.EventRow, error)

type PrewriteBufferSnapEncoder

type PrewriteBufferSnapEncoder struct {
	// The tenant id.
	TenantID uint64

	// range of the buffer.
	Range *pb.Task_Range
}

func NewPrewriteBufferSnapEncoder

func NewPrewriteBufferSnapEncoder(tenantID uint64, range_ *pb.Task_Range) *PrewriteBufferSnapEncoder

func (*PrewriteBufferSnapEncoder) Encode

func (p *PrewriteBufferSnapEncoder) Encode(w io.Writer, prewrites map[string]*pb.EventRow, commits map[string]*pb.EventRow) error

type RangeWatermarksSnapDecoder

type RangeWatermarksSnapDecoder struct {
	// The tenant id.
	TenantID uint64

	// range of the buffer.
	Range *pb.Task_Range
}

RangeWatermarksSnapDecoder is used to decode the range watermarks snap file.

func NewRangeWatermarksSnapDecoder

func NewRangeWatermarksSnapDecoder(tenantID uint64, range_ *pb.Task_Range) *RangeWatermarksSnapDecoder

NewRangeWatermarksSnapDecoder creates a new RangeWatermarksSnapDecoder.

func (*RangeWatermarksSnapDecoder) Decode

func (r *RangeWatermarksSnapDecoder) Decode(reader io.Reader) ([]*pb.EventWatermark, error)

Decode decodes the range watermarks snap file.

type RangeWatermarksSnapEncoder

type RangeWatermarksSnapEncoder struct {
	// The tenant id.
	TenantID uint64

	// range of the buffer.
	Range *pb.Task_Range
}

RangeWatermarksSnapEncoder is used to encode the range watermarks snap file.

func NewRangeWatermarksSnapEncoder

func NewRangeWatermarksSnapEncoder(tenantID uint64, range_ *pb.Task_Range) *RangeWatermarksSnapEncoder

NewRangeWatermarksSnapEncoder creates a new RangeWatermarksSnapEncoder.

func (*RangeWatermarksSnapEncoder) Encode

func (r *RangeWatermarksSnapEncoder) Encode(w io.Writer, watermarks []*pb.EventWatermark) error

Encode encodes the range watermarks snap file.

type SchemaSnapFile

type SchemaSnapFile struct {
	// contains filtered or unexported fields
}

func NewEmptySchemaSnap

func NewEmptySchemaSnap() *SchemaSnapFile

NewSchemaSnap creates a new SchemaSnap.

func (*SchemaSnapFile) DecodeFrom

func (s *SchemaSnapFile) DecodeFrom(reader io.Reader) error

func (*SchemaSnapFile) EncodeTo

func (s *SchemaSnapFile) EncodeTo(writer io.Writer) error

type SorterBufferSnapDecoder

type SorterBufferSnapDecoder struct {
	// The tenant id.
	TenantID uint64

	// range of the buffer.
	Range *pb.Task_Range
}

func NewSorterBufferSnapDecoder

func NewSorterBufferSnapDecoder(tenantID uint64, range_ *pb.Task_Range) *SorterBufferSnapDecoder

func (*SorterBufferSnapDecoder) Decode

func (d *SorterBufferSnapDecoder) Decode(r io.Reader) ([]*pb.EventRow, error)

type SorterBufferSnapEncoder

type SorterBufferSnapEncoder struct {
	// The tenant id.
	TenantID uint64

	// range of the buffer.
	Range *pb.Task_Range
}

func NewSorterBufferSnapEncoder

func NewSorterBufferSnapEncoder(tenantID uint64, range_ *pb.Task_Range) *SorterBufferSnapEncoder

func (*SorterBufferSnapEncoder) Encode

func (s *SorterBufferSnapEncoder) Encode(w io.Writer, events []*pb.EventRow) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL