bootstrap

Documentation

Index

• Variables
• func NewPeerDataNodeServer(metaStore common.MetaStore, diskStore diskstore.DiskStore) pb.PeerDataNodeServer
• type BootstrapDetails
  • func NewBootstrapDetails() BootstrapDetails
• type BootstrapStage
• type BootstrapState
• type Bootstrapable
• type Options
  • func NewOptions() Options
• type PeerDataNodeServerImpl
  • func (p *PeerDataNodeServerImpl) AcquireToken(tableName string, shardID uint32) bool
  • func (p *PeerDataNodeServerImpl) BenchmarkFileTransfer(req *pb.BenchmarkRequest, stream pb.PeerDataNode_BenchmarkFileTransferServer) error
  • func (p *PeerDataNodeServerImpl) FetchTableShardMetaData(ctx context.Context, req *pb.TableShardMetaDataRequest) (*pb.TableShardMetaData, error)
  • func (p *PeerDataNodeServerImpl) FetchVectorPartyRawData(req *pb.VectorPartyRawDataRequest, ...) error
  • func (p *PeerDataNodeServerImpl) Health(ctx context.Context, req *pb.HealthCheckRequest) (*pb.HealthCheckResponse, error)
  • func (p *PeerDataNodeServerImpl) KeepAlive(stream pb.PeerDataNode_KeepAliveServer) error
  • func (p *PeerDataNodeServerImpl) ReleaseToken(tableName string, shardID uint32)
  • func (p *PeerDataNodeServerImpl) StartSession(ctx context.Context, req *pb.StartSessionRequest) (*pb.Session, error)

Variables

var (
	// ErrDatanodeIsBootstrapping raised when trying to bootstrap a datanode that's being bootstrapped.
	ErrDatanodeIsBootstrapping = errors.New("datanode is bootstrapping")

	// ErrDatanodeNotBootstrapped raised when trying to flush/snapshot data for a namespace that's not yet bootstrapped.
	ErrDatanodeNotBootstrapped = errors.New("datanode is not yet bootstrapped")

	// ErrTableShardIsBootstrapping raised when trying to bootstrap a shard that's being bootstrapped.
	ErrTableShardIsBootstrapping = errors.New("table shard is bootstrapping")

	// ErrTableShardNotBootstrappedToFlush raised when trying to flush data for a shard that's not yet bootstrapped.
	ErrTableShardNotBootstrappedToFlush = errors.New("table shard is not yet bootstrapped to flush")

	// ErrTableShardNotBootstrappedToSnapshot raised when trying to snapshot data for a shard that's not yet bootstrapped.
	ErrTableShardNotBootstrappedToSnapshot = errors.New("table shard is not yet bootstrapped to snapshot")

	// ErrTableShardNotBootstrappedToRead raised when trying to read data for a shard that's not yet bootstrapped.
	ErrTableShardNotBootstrappedToRead = errors.New("table shard is not yet bootstrapped to read")

	// ErrBootstrapEnqueued raised when trying to bootstrap and bootstrap becomes enqueued.
	ErrBootstrapEnqueued = errors.New("database bootstrapping enqueued bootstrap")
)

Functions

func NewPeerDataNodeServer

func NewPeerDataNodeServer(metaStore common.MetaStore, diskStore diskstore.DiskStore) pb.PeerDataNodeServer

Types

type BootstrapDetails

type BootstrapDetails interface {
	json.Marshaler

	SetSource(source string)
	SetNumColumns(numColumns int)
	SetBootstrapStage(stage BootstrapStage)
	AddVPToCopy(batch int32, columnID uint32)
	MarkVPFinished(batch int32, columnID uint32)
	Clear()
}

BootstrapDetail describes details for bootstrap

func NewBootstrapDetails

func NewBootstrapDetails() BootstrapDetails

type BootstrapStage

type BootstrapStage string

BootstrapStage represents stages of bootstrap

const (
	Waiting  BootstrapStage = "waiting"
	PeerCopy BootstrapStage = "peercopy"
	Preload  BootstrapStage = "preload"
	Recovery BootstrapStage = "recovery"
	Finished BootstrapStage = "finished"
)

type BootstrapState

type BootstrapState int

BootstrapState is an enum representing the possible bootstrap states for a shard.

const (
	// BootstrapNotStarted indicates bootstrap has not been started yet.
	BootstrapNotStarted BootstrapState = iota
	// Bootstrapping indicates bootstrap process is in progress.
	Bootstrapping
	// Bootstrapped indicates a bootstrap process has completed.
	Bootstrapped
)

type Bootstrapable

type Bootstrapable interface {
	Bootstrap(peerSource client.PeerSource, origin string, topo topology.Topology, topoState *topology.StateSnapshot, options Options) error
}

Bootstrapable defines bootstrapable interface

type Options

type Options interface {
	// MaxConcurrentTableShards returns the max number of concurrent bootstrapping table shards
	MaxConcurrentTableShards() int
	// SetMaxConcurrentShards sets the max number of concurrent bootstrapping table shards
	SetMaxConcurrentShards(numShards int) Options
	// MaxConcurrentStreamsPerTableShards returns the max number of current data streams per bootstrapping table shard
	MaxConcurrentStreamsPerTableShards() int
	// SetMaxConcurrentStreamsPerTableShards sets the max number of current data streams per bootstrapping table shard
	SetMaxConcurrentStreamsPerTableShards(numStreams int) Options
	// BootstrapSessionTTL returns the ttl for bootstrap session
	BootstrapSessionTTL() int64
	// SetBootstrapSessionTTL sets the session ttl for bootstrap session
	SetBootstrapSessionTTL(ttl int64) Options
}

Options defines options for bootstrap

func NewOptions

func NewOptions() Options

NewOptions returns bootstrap default options

type PeerDataNodeServerImpl

type PeerDataNodeServerImpl struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func (*PeerDataNodeServerImpl) AcquireToken

func (p *PeerDataNodeServerImpl) AcquireToken(tableName string, shardID uint32) bool

AcquireToken is to check if any bootstrap is running in the table/shard if no bootstrap session is running on the table/shard, it will increase the token count, and return true the caller need to release the usage by calling ReleaseToken

func (*PeerDataNodeServerImpl) BenchmarkFileTransfer

func (p *PeerDataNodeServerImpl) BenchmarkFileTransfer(req *pb.BenchmarkRequest, stream pb.PeerDataNode_BenchmarkFileTransferServer) error

BenchmarkFileTransfer is used to benchmark testing, we can remove later TODO

func (*PeerDataNodeServerImpl) FetchTableShardMetaData

func (p *PeerDataNodeServerImpl) FetchTableShardMetaData(ctx context.Context, req *pb.TableShardMetaDataRequest) (*pb.TableShardMetaData, error)

FetchTableShardMetaData to retrieve all metadata for one table/shard

func (*PeerDataNodeServerImpl) FetchVectorPartyRawData

func (p *PeerDataNodeServerImpl) FetchVectorPartyRawData(req *pb.VectorPartyRawDataRequest, stream pb.PeerDataNode_FetchVectorPartyRawDataServer) error

func (*PeerDataNodeServerImpl) Health

func (p *PeerDataNodeServerImpl) Health(ctx context.Context, req *pb.HealthCheckRequest) (*pb.HealthCheckResponse, error)

Health return the healthiness status of the data server

func (*PeerDataNodeServerImpl) KeepAlive

func (p *PeerDataNodeServerImpl) KeepAlive(stream pb.PeerDataNode_KeepAliveServer) error

KeepAlive is like client/server ping process, to notify health about each other

func (*PeerDataNodeServerImpl) ReleaseToken

func (p *PeerDataNodeServerImpl) ReleaseToken(tableName string, shardID uint32)

AcquireToken release the token count, must call this when call AcquireToken success

func (*PeerDataNodeServerImpl) StartSession

func (p *PeerDataNodeServerImpl) StartSession(ctx context.Context, req *pb.StartSessionRequest) (*pb.Session, error)

StartSession create new session for one table/shard/node, only One session can be established on one table/shard from one node