Documentation ¶
Index ¶
- Constants
- func CalcChecksum(data []byte) uint32
- func GetCommittedRangeIndex(fdata []byte, tenantId uint64, range_ *pb.Task_Range) ([]uint64, error)
- type CommittedDataDecoder
- type CommittedDataEncoder
- type CommittedManifest
- type CommittedManifestDecoder
- type CommittedManifestEncoder
- type CommittedRangeIndexDecoder
- type CommittedRangeIndexEncoder
- type DataChangesFileDecoder
- type FileEvent
- type PrewriteBufferSnapDecoder
- type PrewriteBufferSnapEncoder
- type RangeWatermarksSnapDecoder
- type RangeWatermarksSnapEncoder
- type SchemaSnapFile
- type SorterBufferSnapDecoder
- type SorterBufferSnapEncoder
Constants ¶
const ( // The magic number of the committed data file. CommittedDataFileMagicNumber = uint32(0x30541989) CommittedDataFileVersion = uint32(1) )
const ( CommittedManifestName string = "manifest" CommittedManifestMagic string = "TISTREAM-COMMITTED-MANIFEST" CommittedManifestVersion uint32 = 1 )
const ( CommittedRangeIndexName = "committed_range_index" // The magic number of the committed range index file. CommittedRangeIndexMagicNumber = uint32(0x30541990) CommittedRangeIndexVersion = uint32(1) )
const ( // The magic number of the prewrite buffer file. PrewriteBufferSnapFileMagicNumber = uint32(0x21647911) PrewriteBufferSnapFileVersion = uint32(1) )
const ( RangeWatermarksSnapMagicNumber = uint32(0x81081793) RangeWatermarksSnapFileVersion = uint32(1) )
The magic number of the range watermarks snap file.
const ( SchemaFileMagicHeader = string("TiStreamSchemaSnap") SchemaFileVersion = 1 )
const ( // The magic number of the sorter buffer snap file. SorterSnapMagicNumber = uint32(0x81081792) SorterSnapFileVersion = uint32(1) )
Variables ¶
This section is empty.
Functions ¶
func CalcChecksum ¶
func GetCommittedRangeIndex ¶
Types ¶
type CommittedDataDecoder ¶
type CommittedDataDecoder struct { // The tenant id. TenantID uint64 // range of the file. Range *pb.Task_Range }
func NewCommittedDataDecoder ¶
func NewCommittedDataDecoder(tenantID uint64, range_ *pb.Task_Range) *CommittedDataDecoder
func (*CommittedDataDecoder) DecodeFrom ¶
type CommittedDataEncoder ¶
type CommittedDataEncoder struct { // The range of the file. Range *pb.Task_Range // contains filtered or unexported fields }
func NewCommittedDataEncoder ¶
func NewCommittedDataEncoder(tenantID uint64, range_ *pb.Task_Range, lowWatermark, highWatermark uint64) *CommittedDataEncoder
type CommittedManifest ¶
type CommittedManifest struct { sync.RWMutex // The tenant id. TenantID uint64 // Root directory of the committed data pool for the tenant. // In above file org it should be "{prefix}committed_data_pool/Tenant-{1}/" RootDir string // Backend Storage BackendStorage storage.ExternalStorage // Ranges Snapshots in different ts. // ts -> *pb.RangesSnapshot Snapshots []*pb.RangesSnapshot // contains filtered or unexported fields }
func NewCommittedDataPoolManifest ¶
func NewCommittedDataPoolManifest(tenantID uint64, rootDir string, backendStorage storage.ExternalStorage) *CommittedManifest
NewCommittedDataPoolManifest creates a new CommittedDataPoolManifest.
func (*CommittedManifest) AddSnapshot ¶
func (m *CommittedManifest) AddSnapshot(ts uint64, snapshot *pb.RangesSnapshot) error
AddSnapshot adds a snapshot to the committed manifest.
func (*CommittedManifest) CompactTo ¶
func (m *CommittedManifest) CompactTo(ts uint64) bool
Compact the committed manifest. Return true if the manifest is compacted, otherwise return false.
type CommittedManifestDecoder ¶
type CommittedManifestDecoder struct { // The tenant id. TenantID uint64 }
func NewCommittedManifestDecoder ¶
func NewCommittedManifestDecoder(tenantID uint64) *CommittedManifestDecoder
func (*CommittedManifestDecoder) Decode ¶
func (d *CommittedManifestDecoder) Decode(r io.Reader) ([]*pb.RangesSnapshot, error)
func (*CommittedManifestDecoder) DecodeV1 ¶
func (d *CommittedManifestDecoder) DecodeV1(r io.Reader) ([]*pb.RangesSnapshot, error)
type CommittedManifestEncoder ¶
type CommittedManifestEncoder struct { // The tenant id. TenantID uint64 }
func NewCommittedManifestEncoder ¶
func NewCommittedManifestEncoder(tenantID uint64) *CommittedManifestEncoder
func (*CommittedManifestEncoder) Encode ¶
func (e *CommittedManifestEncoder) Encode(w io.Writer, snapshots []*pb.RangesSnapshot) error
type CommittedRangeIndexDecoder ¶
type CommittedRangeIndexDecoder struct { // The tenant id. TenantID uint64 // The range of the file. Range *pb.Task_Range }
func NewCommittedRangeIndexDecoder ¶
func NewCommittedRangeIndexDecoder(tenantID uint64, range_ *pb.Task_Range) *CommittedRangeIndexDecoder
type CommittedRangeIndexEncoder ¶
type CommittedRangeIndexEncoder struct { // The tenant id. TenantID uint64 // The range of the file. Range *pb.Task_Range // All sub folders TsArray []uint64 }
func NewCommittedRangeIndexEncoder ¶
func NewCommittedRangeIndexEncoder(tenantID uint64, range_ *pb.Task_Range, tsArray []uint64) *CommittedRangeIndexEncoder
type DataChangesFileDecoder ¶
type DataChangesFileDecoder struct { TenantID uint64 // All the event rows in the file. EventRows []*pb.EventRow // The event watermark in the file. EventWatermarks []*pb.EventWatermark // DDL changes DdlChanges []*pb.DDLChange }
Data change file is the file to store the data change. The file payload is as follows: Header (24 bytes) | File version (4 bytes) | Event size (4 bytes) | Event batch (event size bytes) | ... | Checksum (4 bytes)
func NewDataChangesFileDecoder ¶
func NewDataChangesFileDecoder(tenantId uint64) *DataChangesFileDecoder
NewDataChangesFileDecoder creates a new DataChanges.
func (*DataChangesFileDecoder) DecodeFrom ¶
func (f *DataChangesFileDecoder) DecodeFrom(reader io.Reader) error
DecodeFrom decodes the data change from the reader.
type FileEvent ¶
type FileEvent = DataChangesFileDecoder
type PrewriteBufferSnapDecoder ¶
type PrewriteBufferSnapDecoder struct { // The tenant id. TenantID uint64 // range of the buffer. Range *pb.Task_Range }
func NewPrewriteBufferSnapDecoder ¶
func NewPrewriteBufferSnapDecoder(tenantID uint64, range_ *pb.Task_Range) *PrewriteBufferSnapDecoder
type PrewriteBufferSnapEncoder ¶
type PrewriteBufferSnapEncoder struct { // The tenant id. TenantID uint64 // range of the buffer. Range *pb.Task_Range }
func NewPrewriteBufferSnapEncoder ¶
func NewPrewriteBufferSnapEncoder(tenantID uint64, range_ *pb.Task_Range) *PrewriteBufferSnapEncoder
type RangeWatermarksSnapDecoder ¶
type RangeWatermarksSnapDecoder struct { // The tenant id. TenantID uint64 // range of the buffer. Range *pb.Task_Range }
RangeWatermarksSnapDecoder is used to decode the range watermarks snap file.
func NewRangeWatermarksSnapDecoder ¶
func NewRangeWatermarksSnapDecoder(tenantID uint64, range_ *pb.Task_Range) *RangeWatermarksSnapDecoder
NewRangeWatermarksSnapDecoder creates a new RangeWatermarksSnapDecoder.
func (*RangeWatermarksSnapDecoder) Decode ¶
func (r *RangeWatermarksSnapDecoder) Decode(reader io.Reader) ([]*pb.EventWatermark, error)
Decode decodes the range watermarks snap file.
type RangeWatermarksSnapEncoder ¶
type RangeWatermarksSnapEncoder struct { // The tenant id. TenantID uint64 // range of the buffer. Range *pb.Task_Range }
RangeWatermarksSnapEncoder is used to encode the range watermarks snap file.
func NewRangeWatermarksSnapEncoder ¶
func NewRangeWatermarksSnapEncoder(tenantID uint64, range_ *pb.Task_Range) *RangeWatermarksSnapEncoder
NewRangeWatermarksSnapEncoder creates a new RangeWatermarksSnapEncoder.
func (*RangeWatermarksSnapEncoder) Encode ¶
func (r *RangeWatermarksSnapEncoder) Encode(w io.Writer, watermarks []*pb.EventWatermark) error
Encode encodes the range watermarks snap file.
type SchemaSnapFile ¶
type SchemaSnapFile struct {
// contains filtered or unexported fields
}
func NewEmptySchemaSnap ¶
func NewEmptySchemaSnap() *SchemaSnapFile
NewSchemaSnap creates a new SchemaSnap.
func (*SchemaSnapFile) DecodeFrom ¶
func (s *SchemaSnapFile) DecodeFrom(reader io.Reader) error
type SorterBufferSnapDecoder ¶
type SorterBufferSnapDecoder struct { // The tenant id. TenantID uint64 // range of the buffer. Range *pb.Task_Range }
func NewSorterBufferSnapDecoder ¶
func NewSorterBufferSnapDecoder(tenantID uint64, range_ *pb.Task_Range) *SorterBufferSnapDecoder
type SorterBufferSnapEncoder ¶
type SorterBufferSnapEncoder struct { // The tenant id. TenantID uint64 // range of the buffer. Range *pb.Task_Range }
func NewSorterBufferSnapEncoder ¶
func NewSorterBufferSnapEncoder(tenantID uint64, range_ *pb.Task_Range) *SorterBufferSnapEncoder