master

Documentation

Index

• Constants
• func GetKey(c *clientv3.Client, key string, timeout time.Duration) (string, error)
• type DataSet
  • func NewDataSet(fileList []string) (*DataSet, error)
  • func (d *DataSet) GetFile() (string, error)
  • func (d *DataSet) StartNewEpoch()
• type Error
  • func DuplicateInitDataSet(detail string) *Error
  • func (v *Error) Error() string
  • func (v *Error) ToRPCRet() *pb.RPCRet
• type ErrorType
  • func (t ErrorType) String() string
• type EtcdClient
  • func NewEtcdClient(endpoints []string, addr string, lockPath, addrPath, statePath string, ...) (*EtcdClient, error)
  • func (e *EtcdClient) Load() ([]byte, error)
  • func (e *EtcdClient) Save(state []byte) error
  • func (e *EtcdClient) Shutdown() error
• type InMemStore
  • func (m *InMemStore) Load() ([]byte, error)
  • func (m *InMemStore) Save(state []byte) error
  • func (m *InMemStore) Shutdown() error
• type Service
  • func NewService(etcd *EtcdClient, timeoutDur time.Duration, failureMax int) (*Service, error)
  • func (s *Service) AddDataSet(ctx context.Context, in *pb.DataSet) (*pb.RPCRet, error)
  • func (s *Service) Barrier(ctx context.Context, in *pb.BarrierRequest) (*pb.ClusterResponse, error)
  • func (s *Service) GetCluster(ctx context.Context, in *pb.ClusterRequest) (*pb.ClusterResponse, error)
  • func (s *Service) GetSubDataSet(context.Context, *pb.SubDataSetRequest) (*pb.SubDataSetResponse, error)
  • func (s *Service) GetTask(context.Context, *pb.TaskRequest) (*pb.TaskResponse, error)
  • func (s *Service) NewEpoch(ctx context.Context, in *pb.NewEpochRequest) (*pb.RPCRet, error)
  • func (s *Service) ReportChunks(ctx context.Context, in *pb.DataServerChunk) (*pb.RPCRet, error)
  • func (s *Service) SetDataSet(globPaths []string, _ *int) error
  • func (s *Service) TaskErrored(ctx context.Context, in *pb.Tasks) (*pb.RPCRet, error)
  • func (s *Service) TaskFailed(meta pb.TaskMeta, dummy *int) error
  • func (s *Service) TaskFinished(ctx context.Context, in *pb.Tasks) (*pb.RPCRet, error)
• type Store

Constants

const (
	// DefaultLockPath is the default etcd master lock path.
	DefaultLockPath = "/master/lock"
	// DefaultStatePath is the default etcd key for master state.
	DefaultStatePath = "/master/state"
	// DefaultAddrPath is the default etcd key for master address.
	DefaultAddrPath = "/master/addr"
)

Functions

func GetKey

func GetKey(c *clientv3.Client, key string, timeout time.Duration) (string, error)

GetKey gets the value by the specify key.

Types

type DataSet

type DataSet struct {
	Files []fileDataSet
}

DataSet represents a file list dataset.

func NewDataSet

func NewDataSet(fileList []string) (*DataSet, error)

NewDataSet constructs a new DataSet.

func (*DataSet) GetFile

func (d *DataSet) GetFile() (string, error)

GetFile gets one file from the file list

func (*DataSet) StartNewEpoch

func (d *DataSet) StartNewEpoch()

StartNewEpoch starts a new epoch

type Error

type Error struct {
	Type   ErrorType
	Detail string
}

Error implements Error interface

func DuplicateInitDataSet

func DuplicateInitDataSet(detail string) *Error

DuplicateInitDataSet make the correspond error.

func (*Error) Error

func (v *Error) Error() string

Error implements the error interface.

func (*Error) ToRPCRet

func (v *Error) ToRPCRet() *pb.RPCRet

ToRPCRet converts Error to RPCRet

type ErrorType

type ErrorType string

ErrorType is the typei name of error.

const (
	// ErrorTypeDuplicateInitDataSet is used to reported dataset error.
	ErrorTypeDuplicateInitDataSet ErrorType = "DuplicateInitDataSet"
)

func (ErrorType) String

func (t ErrorType) String() string

String converts a ErrorType into its corresponding canonical error message.

type EtcdClient

type EtcdClient struct {
	// contains filtered or unexported fields
}

EtcdClient is the etcd client that the master uses for fault tolerance and service registry.

func NewEtcdClient

func NewEtcdClient(endpoints []string, addr string, lockPath, addrPath, statePath string, ttlSec int) (*EtcdClient, error)

NewEtcdClient creates a new EtcdClient.

func (*EtcdClient) Load

func (e *EtcdClient) Load() ([]byte, error)

Load loads the state from etcd.

func (*EtcdClient) Save

func (e *EtcdClient) Save(state []byte) error

Save saves the state into the etcd.

func (*EtcdClient) Shutdown

func (e *EtcdClient) Shutdown() error

Shutdown shuts down the etcd client gracefully.

type InMemStore

type InMemStore struct {
	// contains filtered or unexported fields
}

InMemStore is an in memory implementation of Store interface.

It does not tolerate the fault that causes the program to crash.

func (*InMemStore) Load

func (m *InMemStore) Load() ([]byte, error)

Load loads the state from the in-memory store.

func (*InMemStore) Save

func (m *InMemStore) Save(state []byte) error

Save saves the state into the in-memory store.

func (*InMemStore) Shutdown

func (m *InMemStore) Shutdown() error

Shutdown shuts down the in mem store.

type Service

type Service struct {
	Chunks map[string][]pb.Chunk // DataServerID->ChunksArray
	// contains filtered or unexported fields
}

Service is the master server service.

func NewService

func NewService(etcd *EtcdClient, timeoutDur time.Duration, failureMax int) (*Service, error)

NewService creates a new service.

func (*Service) AddDataSet

func (s *Service) AddDataSet(ctx context.Context, in *pb.DataSet) (*pb.RPCRet, error)

AddDataSet adds a initial dataset to service.

func (*Service) Barrier

func (s *Service) Barrier(ctx context.Context, in *pb.BarrierRequest) (*pb.ClusterResponse, error)

Barrier implementes the proto interface.

func (*Service) GetCluster

func (s *Service) GetCluster(ctx context.Context, in *pb.ClusterRequest) (*pb.ClusterResponse, error)

GetCluster gets cluster elements from the service.

func (*Service) GetSubDataSet

func (s *Service) GetSubDataSet(context.Context, *pb.SubDataSetRequest) (*pb.SubDataSetResponse, error)

GetSubDataSet implements the proto interface.

func (*Service) GetTask

func (s *Service) GetTask(context.Context, *pb.TaskRequest) (*pb.TaskResponse, error)

GetTask gets a new task from the service. passID is the client side pass count

func (*Service) NewEpoch

func (s *Service) NewEpoch(ctx context.Context, in *pb.NewEpochRequest) (*pb.RPCRet, error)

NewEpoch starts a new epoch of the service.

func (*Service) ReportChunks

func (s *Service) ReportChunks(ctx context.Context, in *pb.DataServerChunk) (*pb.RPCRet, error)

ReportChunks implementes the proto interface.

func (*Service) SetDataSet

func (s *Service) SetDataSet(globPaths []string, _ *int) error

SetDataSet implements the proto interface.

func (*Service) TaskErrored

func (s *Service) TaskErrored(ctx context.Context, in *pb.Tasks) (*pb.RPCRet, error)

TaskErrored reports a new tasks error to the service.

func (*Service) TaskFailed

func (s *Service) TaskFailed(meta pb.TaskMeta, dummy *int) error

TaskFailed tells the service that a task is failed.

func (*Service) TaskFinished

func (s *Service) TaskFinished(ctx context.Context, in *pb.Tasks) (*pb.RPCRet, error)

TaskFinished tell the service that a task is finished.

type Store

type Store interface {
	Save([]byte) error
	Load() ([]byte, error)
	Shutdown() error
}

Store is the interface for save and load the master state.