gc

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 13, 2023 License: Apache-2.0 Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source
const (
	MessgeReplay = iota
	MessgeNormal
)
View Source
const (
	PrefixGCMeta = "gc"
	GCMetaDir    = "gc/"
)
View Source
const (
	GCAttrObjectName = "name"
	GCAttrBlockId    = "block_id"
	GCAttrTableId    = "table_id"
	GCAttrDBId       = "db_id"
)
View Source
const MinMergeCount = 20

Variables

View Source
var (
	BlockSchemaAttr = []string{
		GCAttrBlockId,
		GCAttrTableId,
		GCAttrDBId,
		GCAttrObjectName,
	}
	BlockSchemaTypes = []types.Type{
		types.New(types.T_Blockid, 0, 0),
		types.New(types.T_uint64, 0, 0),
		types.New(types.T_uint64, 0, 0),
		types.New(types.T_varchar, 5000, 0),
	}

	DropTableSchemaAttr = []string{
		GCAttrTableId,
		GCAttrDBId,
	}
	DropTableSchemaTypes = []types.Type{
		types.New(types.T_uint64, 0, 0),
		types.New(types.T_uint64, 0, 0),
	}

	DropDBSchemaAtt = []string{
		GCAttrDBId,
	}
	DropDBSchemaTypes = []types.Type{
		types.New(types.T_uint64, 0, 0),
	}

	DeleteFileSchemaAtt = []string{
		GCAttrObjectName,
	}
	DeleteFileSchemaTypes = []types.Type{
		types.New(types.T_varchar, 5000, 0),
	}
)

Functions

func NewDropDB

func NewDropDB(id uint64) *dropDB

func NewDropTable

func NewDropTable(id uint64) *dropTable

Types

type BatchType

type BatchType int8
const (
	CreateBlock BatchType = iota
	DeleteBlock
	DropTable
	DropDB
	DeleteFile
)

type CleanerState

type CleanerState int8
const (
	Idle CleanerState = iota
	Running
)

type DiskCleaner

type DiskCleaner struct {
	// contains filtered or unexported fields
}

DiskCleaner is the main structure of gc operation, and provides "JobFactory" to let tae notify itself to perform a gc

func NewDiskCleaner

func NewDiskCleaner(
	ctx context.Context,
	fs *objectio.ObjectFS,
	ckpClient checkpoint.RunnerReader,
	catalog *catalog.Catalog,
) *DiskCleaner

func (*DiskCleaner) AddChecker

func (cleaner *DiskCleaner) AddChecker(checker func(item any) bool)

func (*DiskCleaner) CheckGC

func (cleaner *DiskCleaner) CheckGC() error

func (*DiskCleaner) GC

func (cleaner *DiskCleaner) GC(ctx context.Context) (err error)

func (*DiskCleaner) GetAndClearOutputs

func (cleaner *DiskCleaner) GetAndClearOutputs() []string

func (*DiskCleaner) GetInputs

func (cleaner *DiskCleaner) GetInputs() *GCTable

func (*DiskCleaner) GetMaxConsumed

func (cleaner *DiskCleaner) GetMaxConsumed() *checkpoint.CheckpointEntry

func (*DiskCleaner) GetMinMerged

func (cleaner *DiskCleaner) GetMinMerged() *checkpoint.CheckpointEntry

func (*DiskCleaner) SetMinMergeCountForTest

func (cleaner *DiskCleaner) SetMinMergeCountForTest(count int)

func (*DiskCleaner) Start

func (cleaner *DiskCleaner) Start()

func (*DiskCleaner) Stop

func (cleaner *DiskCleaner) Stop()

type GCTable

type GCTable struct {
	sync.Mutex
	// contains filtered or unexported fields
}

GCTable is a data structure in memory after consuming checkpoint

func NewGCTable

func NewGCTable() *GCTable

func (*GCTable) Compare

func (t *GCTable) Compare(table *GCTable) bool

For test

func (*GCTable) Merge

func (t *GCTable) Merge(GCTable *GCTable)

Merge can merge two GCTables

func (*GCTable) Prefetch added in v0.8.0

func (t *GCTable) Prefetch(ctx context.Context, name string, size int64, fs *objectio.ObjectFS) error

func (*GCTable) ReadTable

func (t *GCTable) ReadTable(ctx context.Context, name string, size int64, fs *objectio.ObjectFS) error

ReadTable reads an s3 file and replays a GCTable in memory

func (*GCTable) SaveFullTable

func (t *GCTable) SaveFullTable(start, end types.TS, fs *objectio.ObjectFS, files []string) ([]objectio.BlockObject, error)

SaveFullTable is to write data to s3

func (*GCTable) SaveTable

func (t *GCTable) SaveTable(start, end types.TS, fs *objectio.ObjectFS, files []string) ([]objectio.BlockObject, error)

SaveTable is to write data to s3

func (*GCTable) SoftGC

func (t *GCTable) SoftGC() []string

SoftGC is to remove objectentry that can be deleted from GCTable

func (*GCTable) String

func (t *GCTable) String() string

func (*GCTable) UpdateTable

func (t *GCTable) UpdateTable(data *logtail.CheckpointData)

type GCWorker

type GCWorker struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewGCWorker

func NewGCWorker(fs *objectio.ObjectFS, cleaner *DiskCleaner) *GCWorker

func (*GCWorker) ExecDelete

func (g *GCWorker) ExecDelete(ctx context.Context, names []string) error

func (*GCWorker) Start

func (g *GCWorker) Start() bool

type ObjectEntry

type ObjectEntry struct {
	// contains filtered or unexported fields
}

func NewObjectEntry

func NewObjectEntry() *ObjectEntry

func (*ObjectEntry) AddBlock

func (o *ObjectEntry) AddBlock(block common.ID)

func (*ObjectEntry) AllowGC

func (o *ObjectEntry) AllowGC() bool

func (*ObjectEntry) Compare

func (o *ObjectEntry) Compare(object *ObjectEntry) bool

func (*ObjectEntry) DelBlock

func (o *ObjectEntry) DelBlock(block common.ID)

func (*ObjectEntry) MergeEntry

func (o *ObjectEntry) MergeEntry(entry *ObjectEntry)

func (*ObjectEntry) Refs

func (o *ObjectEntry) Refs(n int)

func (*ObjectEntry) String

func (o *ObjectEntry) String() string

func (*ObjectEntry) UnRefs

func (o *ObjectEntry) UnRefs(n int)

type TableEntry

type TableEntry struct {
	// contains filtered or unexported fields
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL