gc

package
v1.0.0-rc2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 24, 2023 License: Apache-2.0 Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	MessgeReplay = iota
	MessgeNormal
)
View Source 
const (
	PrefixGCMeta = "gc"
	GCMetaDir    = "gc/"
)
View Source 
const (
	GCAttrObjectName = "name"
	GCAttrBlockId    = "block_id"
	GCAttrTableId    = "table_id"
	GCAttrDBId       = "db_id"
)
View Source 
const MinMergeCount = 20

Variables

View Source 
var (
	BlockSchemaAttr = []string{
		GCAttrBlockId,
		GCAttrTableId,
		GCAttrDBId,
		GCAttrObjectName,
	}
	BlockSchemaTypes = []types.Type{
		types.New(types.T_Blockid, 0, 0),
		types.New(types.T_uint64, 0, 0),
		types.New(types.T_uint64, 0, 0),
		types.New(types.T_varchar, 5000, 0),
	}

	DropTableSchemaAttr = []string{
		GCAttrTableId,
		GCAttrDBId,
	}
	DropTableSchemaTypes = []types.Type{
		types.New(types.T_uint64, 0, 0),
		types.New(types.T_uint64, 0, 0),
	}

	DropDBSchemaAtt = []string{
		GCAttrDBId,
	}
	DropDBSchemaTypes = []types.Type{
		types.New(types.T_uint64, 0, 0),
	}

	DeleteFileSchemaAtt = []string{
		GCAttrObjectName,
	}
	DeleteFileSchemaTypes = []types.Type{
		types.New(types.T_varchar, 5000, 0),
	}
)

Functions

func NewDropDB 

func NewDropDB(id uint64) *dropDB

func NewDropTable 

func NewDropTable(id uint64) *dropTable

Types

type BatchType 

type BatchType int8
const (
	CreateBlock BatchType = iota
	DeleteBlock
	DropTable
	DropDB
	DeleteFile
)

type CleanerState 

type CleanerState int8
const (
	Idle CleanerState = iota
	Running
)

type DiskCleaner 

type DiskCleaner struct {
	// contains filtered or unexported fields
}

DiskCleaner is the main structure of gc operation, and provides "JobFactory" to let tae notify itself to perform a gc

func NewDiskCleaner 

func NewDiskCleaner(
	ctx context.Context,
	fs *objectio.ObjectFS,
	ckpClient checkpoint.RunnerReader,
	catalog *catalog.Catalog,
) *DiskCleaner

func (*DiskCleaner) AddChecker 

func (cleaner *DiskCleaner) AddChecker(checker func(item any) bool)

func (*DiskCleaner) CheckGC 

func (cleaner *DiskCleaner) CheckGC() error

func (*DiskCleaner) GC 

func (cleaner *DiskCleaner) GC(ctx context.Context) (err error)

func (*DiskCleaner) GetAndClearOutputs 

func (cleaner *DiskCleaner) GetAndClearOutputs() []string

func (*DiskCleaner) GetInputs 

func (cleaner *DiskCleaner) GetInputs() *GCTable

func (*DiskCleaner) GetMaxConsumed 

func (cleaner *DiskCleaner) GetMaxConsumed() *checkpoint.CheckpointEntry

func (*DiskCleaner) GetMinMerged 

func (cleaner *DiskCleaner) GetMinMerged() *checkpoint.CheckpointEntry

func (*DiskCleaner) SetMinMergeCountForTest 

func (cleaner *DiskCleaner) SetMinMergeCountForTest(count int)

func (*DiskCleaner) Start 

func (cleaner *DiskCleaner) Start()

func (*DiskCleaner) Stop 

func (cleaner *DiskCleaner) Stop()

type GCTable 

type GCTable struct {
	sync.Mutex
	// contains filtered or unexported fields
}

GCTable is a data structure in memory after consuming checkpoint

func NewGCTable 

func NewGCTable() *GCTable

func (*GCTable) Compare 

func (t *GCTable) Compare(table *GCTable) bool

For test

func (*GCTable) Merge 

func (t *GCTable) Merge(GCTable *GCTable)

Merge can merge two GCTables

func (*GCTable) Prefetch added in v0.8.0 

func (t *GCTable) Prefetch(ctx context.Context, name string, size int64, fs *objectio.ObjectFS) error

func (*GCTable) ReadTable 

func (t *GCTable) ReadTable(ctx context.Context, name string, size int64, fs *objectio.ObjectFS) error

ReadTable reads an s3 file and replays a GCTable in memory

func (*GCTable) SaveFullTable 

func (t *GCTable) SaveFullTable(start, end types.TS, fs *objectio.ObjectFS, files []string) ([]objectio.BlockObject, error)

SaveFullTable is to write data to s3

func (*GCTable) SaveTable 

func (t *GCTable) SaveTable(start, end types.TS, fs *objectio.ObjectFS, files []string) ([]objectio.BlockObject, error)

SaveTable is to write data to s3

func (*GCTable) SoftGC 

func (t *GCTable) SoftGC() []string

SoftGC is to remove objectentry that can be deleted from GCTable

func (*GCTable) String 

func (t *GCTable) String() string

func (*GCTable) UpdateTable 

func (t *GCTable) UpdateTable(data *logtail.CheckpointData)

type GCWorker 

type GCWorker struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewGCWorker 

func NewGCWorker(fs *objectio.ObjectFS, cleaner *DiskCleaner) *GCWorker

func (*GCWorker) ExecDelete 

func (g *GCWorker) ExecDelete(ctx context.Context, names []string) error

func (*GCWorker) Start 

func (g *GCWorker) Start() bool

type ObjectEntry 

type ObjectEntry struct {
	// contains filtered or unexported fields
}

func NewObjectEntry 

func NewObjectEntry() *ObjectEntry

func (*ObjectEntry) AddBlock 

func (o *ObjectEntry) AddBlock(block common.ID)

func (*ObjectEntry) AllowGC 

func (o *ObjectEntry) AllowGC() bool

func (*ObjectEntry) Compare 

func (o *ObjectEntry) Compare(object *ObjectEntry) bool

func (*ObjectEntry) DelBlock 

func (o *ObjectEntry) DelBlock(block common.ID)

func (*ObjectEntry) MergeEntry 

func (o *ObjectEntry) MergeEntry(entry *ObjectEntry)

func (*ObjectEntry) Refs 

func (o *ObjectEntry) Refs(n int)

func (*ObjectEntry) String 

func (o *ObjectEntry) String() string

func (*ObjectEntry) UnRefs 

func (o *ObjectEntry) UnRefs(n int)

type TableEntry 

type TableEntry struct {
	// contains filtered or unexported fields
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.