master

Documentation

Index

• Constants
• Variables
• func GetKey(c *clientv3.Client, key string, timeout time.Duration) (string, error)
• func WithAddr(addr string) func(c *Client) error
• func WithBuffer(bufSize int) func(*Client) error
• func WithEtcd(endpoints []string, timeout time.Duration) func(*Client) error
• type Chunk
• type Client
  • func NewClient(opts ...func(*Client) error) (*Client, error)
  • func (c *Client) NextRecord() ([]byte, error)
  • func (c *Client) RequestSaveModel(trainerID string, blockDur time.Duration) (bool, error)
  • func (c *Client) SetDataset(globPaths []string) error
  • func (c *Client) StartGetRecords(passID int)
• type EtcdClient
  • func NewEtcdClient(endpoints []string, addr string, lockPath, addrPath, statePath string, ...) (*EtcdClient, error)
  • func (e *EtcdClient) Load() ([]byte, error)
  • func (e *EtcdClient) Save(state []byte) error
  • func (e *EtcdClient) Shutdown() error
• type InMemStore
  • func (m *InMemStore) Load() ([]byte, error)
  • func (m *InMemStore) Save(state []byte) error
  • func (m *InMemStore) Shutdown() error
• type SaveModelRequest
• type Service
  • func NewService(store Store, chunksPerTask int, timeoutDur time.Duration, failureMax int) (*Service, error)
  • func (s *Service) GetTask(passID int, task *Task) error
  • func (s *Service) RequestSaveModel(req SaveModelRequest, need *bool) error
  • func (s *Service) SetDataset(globPaths []string, _ *int) error
  • func (s *Service) TaskFailed(meta TaskMeta, dummy *int) error
  • func (s *Service) TaskFinished(taskID int, dummy *int) error
• type Store
• type Task
• type TaskMeta

Constants

const (
	// DefaultLockPath is the default etcd master lock path.
	DefaultLockPath = "/master/lock"
	// DefaultStatePath is the default etcd key for master state.
	DefaultStatePath = "/master/state"
	// DefaultAddrPath is the default etcd key for master address.
	DefaultAddrPath = "/master/addr"
)

Variables

var ErrAllTaskFailed = errors.New("all task finished")

ErrAllTaskFailed occur when tasks are in done or failed state.

var ErrNoMoreAvailable = errors.New("no more available task")

ErrNoMoreAvailable occur when no task in todo and yet not all done or fail.

var ErrPassAfter = errors.New("pass number larger than master")

ErrPassAfter client side pass number does not match with master counter.

var ErrPassBefore = errors.New("pass number smaller than master")

ErrPassBefore client side pass number does not match with master counter.

Functions

func GetKey

func GetKey(c *clientv3.Client, key string, timeout time.Duration) (string, error)

GetKey gets the value by the specify key.

func WithAddr

func WithAddr(addr string) func(c *Client) error

WithAddr sets the client to use fixed master address.

func WithBuffer

func WithBuffer(bufSize int) func(*Client) error

WithBuffer sets the client to buffer the training record.

bufSize is the record buffer size. NextRecord will read from this buffer.

func WithEtcd

func WithEtcd(endpoints []string, timeout time.Duration) func(*Client) error

WithEtcd sets the client to use etcd for master discovery.

Types

type Chunk

type Chunk struct {
	Path  string
	Index recordio.Index // chunk index
}

Chunk is a chunk of data consisted of several data instances.

type Client

type Client struct {
	// contains filtered or unexported fields
}

Client is the client of the master server.

func NewClient

func NewClient(opts ...func(*Client) error) (*Client, error)

NewClient creates a new Client.

func (*Client) NextRecord

func (c *Client) NextRecord() ([]byte, error)

NextRecord returns next record in the dataset.

NextRecord will block until the next record is available. It is thread-safe.

func (*Client) RequestSaveModel

func (c *Client) RequestSaveModel(trainerID string, blockDur time.Duration) (bool, error)

RequestSaveModel requests the master server to approve the caller to save the model.

func (*Client) SetDataset

func (c *Client) SetDataset(globPaths []string) error

SetDataset sets dataset to dispatch for the master server.

SetDataset can be call multiple times at one pass. But only the first call will be honored.

After all tasks are done, another call of SetDataset will start another pass.

func (*Client) StartGetRecords

func (c *Client) StartGetRecords(passID int)

StartGetRecords must be called at beginning of each pass

type EtcdClient

type EtcdClient struct {
	// contains filtered or unexported fields
}

EtcdClient is the etcd client that the master uses for fault tolerance and service registry.

func NewEtcdClient

func NewEtcdClient(endpoints []string, addr string, lockPath, addrPath, statePath string, ttlSec int) (*EtcdClient, error)

NewEtcdClient creates a new EtcdClient.

func (*EtcdClient) Load

func (e *EtcdClient) Load() ([]byte, error)

Load loads the state from etcd.

func (*EtcdClient) Save

func (e *EtcdClient) Save(state []byte) error

Save saves the state into the etcd.

func (*EtcdClient) Shutdown

func (e *EtcdClient) Shutdown() error

Shutdown shuts down the etcd client gracefully.

type InMemStore

type InMemStore struct {
	// contains filtered or unexported fields
}

InMemStore is an in memory implementation of Store interface.

It does not tolerate the fault that causes the program to crash.

func (*InMemStore) Load

func (m *InMemStore) Load() ([]byte, error)

Load loads the state from the in-memory store.

func (*InMemStore) Save

func (m *InMemStore) Save(state []byte) error

Save saves the state into the in-memory store.

func (*InMemStore) Shutdown

func (m *InMemStore) Shutdown() error

Shutdown shuts down the in mem store.

type SaveModelRequest

type SaveModelRequest struct {
	TrainerID string
	BlockDur  time.Duration
}

SaveModelRequest is the request for saving model

type Service

type Service struct {
	// contains filtered or unexported fields
}

Service is the master server service.

func NewService

func NewService(store Store, chunksPerTask int, timeoutDur time.Duration, failureMax int) (*Service, error)

NewService creates a new service.

func (*Service) GetTask

func (s *Service) GetTask(passID int, task *Task) error

GetTask gets a new task from the service. passID is the client side pass count

func (*Service) RequestSaveModel

func (s *Service) RequestSaveModel(req SaveModelRequest, need *bool) error

RequestSaveModel requests the master server to approve the caller to save the model.

func (*Service) SetDataset

func (s *Service) SetDataset(globPaths []string, _ *int) error

SetDataset sets dataset to dispatch for the master server.

SetDataset can be call multiple times. But only the first call will be honored.

func (*Service) TaskFailed

func (s *Service) TaskFailed(meta TaskMeta, dummy *int) error

TaskFailed tells the service that a task is failed.

func (*Service) TaskFinished

func (s *Service) TaskFinished(taskID int, dummy *int) error

TaskFinished tell the service that a task is finished.

type Store

type Store interface {
	Save([]byte) error
	Load() ([]byte, error)
	Shutdown() error
}

Store is the interface for save and load the master state.

type Task

type Task struct {
	Meta   TaskMeta
	Chunks []Chunk
}

Task is the basic unit of data instances assigned to trainers.

type TaskMeta

type TaskMeta struct {
	ID    int
	Epoch int
}

TaskMeta is a struct which stores task's meta info.