storage

Documentation

Index

• type IndexFile
• type KeyValue
• type LocalFilePersister
  • func NewLocalFilePersister(path string) *LocalFilePersister
  • func (f *LocalFilePersister) Abort(file IndexFile) error
  • func (f *LocalFilePersister) Commit(file IndexFile) (string, error)
  • func (f *LocalFilePersister) Delete(file IndexFile) error
  • func (f *LocalFilePersister) Destroy() error
  • func (f *LocalFilePersister) Init() error
  • func (f *LocalFilePersister) Readable(file IndexFile) (string, error)
  • func (f *LocalFilePersister) Request(file IndexFile) (string, error)
  • func (f *LocalFilePersister) RetrieveMRIdx(path string) (int32, int32)
  • func (f *LocalFilePersister) Writable(file IndexFile) (string, error)
• type Persister
• type S3Config
• type S3Persister
  • func NewS3Persister(conf *S3Config) (*S3Persister, error)
  • func (s *S3Persister) Abort(file IndexFile) error
  • func (s *S3Persister) Commit(file IndexFile) (string, error)
  • func (s *S3Persister) Delete(file IndexFile) error
  • func (s *S3Persister) Destroy() error
  • func (s *S3Persister) Init() error
  • func (s *S3Persister) Readable(file IndexFile) (string, error)
  • func (s *S3Persister) Request(file IndexFile) (string, error)
  • func (s *S3Persister) RetrieveMRIdx(path string) (int32, int32)
  • func (s *S3Persister) Writable(file IndexFile) (string, error)

Types

type IndexFile

type IndexFile struct {
	Temporary bool
	MapIdx    int32
	ReduceIdx int32
	Body      *[]KeyValue
}

IndexFile defines mapreduce persist file.

type KeyValue

type KeyValue struct {
	Key   string
	Value string
}

type LocalFilePersister

type LocalFilePersister struct {
	// contains filtered or unexported fields
}

LocalFilePersister provides local persist service.

func NewLocalFilePersister

func NewLocalFilePersister(path string) *LocalFilePersister

NewLocalFilePersister returns local persist service instance. For unix-like user, your provided path must meet unix naming rules for path.

func (*LocalFilePersister) Abort

func (f *LocalFilePersister) Abort(file IndexFile) error

Abort aborts the index file right now. Not implemented for LocalFilePersister.

func (*LocalFilePersister) Commit

func (f *LocalFilePersister) Commit(file IndexFile) (string, error)

Commit writes the index file right now.

func (*LocalFilePersister) Delete

func (f *LocalFilePersister) Delete(file IndexFile) error

Delete deletes the index file right now.

func (*LocalFilePersister) Destroy

func (f *LocalFilePersister) Destroy() error

Destroy destroys local persist service. Not implemented for LocalFilePersister.

func (*LocalFilePersister) Init

func (f *LocalFilePersister) Init() error

Init inits local persist service.

func (*LocalFilePersister) Readable

func (f *LocalFilePersister) Readable(file IndexFile) (string, error)

Readable checks whether the index file is readable right now or not.

func (*LocalFilePersister) Request

func (f *LocalFilePersister) Request(file IndexFile) (string, error)

Request reads the index file right now.

func (*LocalFilePersister) RetrieveMRIdx

func (f *LocalFilePersister) RetrieveMRIdx(path string) (int32, int32)

RetrieveMRIdx retrieves MapIdx and ReduceIdx from given path. Path must meet naming rule "/path/to/MR-Map-%d-Reduce-%d.tmp".

func (*LocalFilePersister) Writable

func (f *LocalFilePersister) Writable(file IndexFile) (string, error)

Writable checks whether the index file is writable right now or not.

type Persister

type Persister interface {
	Init() (err error)
	Destroy() (err error)
	RetrieveMRIdx(path string) (mIdx int32, rIdx int32)
	Writable(file IndexFile) (path string, err error)
	Commit(file IndexFile) (path string, err error)
	Readable(file IndexFile) (path string, err error)
	Request(file IndexFile) (path string, err error)
	Abort(file IndexFile) (err error)
	Delete(file IndexFile) (err error)
}

Persister defines persist interface methods.

type S3Config

type S3Config struct {
	Endpoint          string `json:"endpoint"`
	AccessKeyID       string `json:"access_key_id"`
	SecretAccessKeyID string `json:"secret_access_key_id"`
	UseSSL            bool   `json:"use_ssl"`
	Bucket            string `json:"bucket"`
	Root              string `json:"root"`
}

S3Config defines s3 persist service config.

type S3Persister

type S3Persister struct {
	// contains filtered or unexported fields
}

S3Persister provides s3 persist service.

func NewS3Persister

func NewS3Persister(conf *S3Config) (*S3Persister, error)

NewS3Persister returns s3 persist service instance.

func (*S3Persister) Abort

func (s *S3Persister) Abort(file IndexFile) error

Abort aborts the local index file right now.

func (*S3Persister) Commit

func (s *S3Persister) Commit(file IndexFile) (string, error)

Commit writes the index file right now. Upload local index file to s3.

func (*S3Persister) Delete

func (s *S3Persister) Delete(file IndexFile) error

Delete deletes the remote index file right now.

func (*S3Persister) Destroy

func (s *S3Persister) Destroy() error

Destroy destroys s3 persist service. Remove all local index files.

func (*S3Persister) Init

func (s *S3Persister) Init() error

Init inits s3 persist service.

func (*S3Persister) Readable

func (s *S3Persister) Readable(file IndexFile) (string, error)

Readable checks whether the index file is readable right now or not. Download remote index file at s3 to local.

func (*S3Persister) Request

func (s *S3Persister) Request(file IndexFile) (string, error)

Request reads the local index file right now.

func (*S3Persister) RetrieveMRIdx

func (s *S3Persister) RetrieveMRIdx(path string) (int32, int32)

RetrieveMRIdx retrieves MapIdx and ReduceIdx from given path. Path must meet naming rule "/path/to/MR-Map-%d-Reduce-%d.tmp".

func (*S3Persister) Writable

func (s *S3Persister) Writable(file IndexFile) (string, error)

Writable checks whether the index file is writable right now or not. Create local index file.