corfs

Documentation

Index

• type FileInfo
• type FileSystem
  • func InferFilesystem(location string) FileSystem
  • func InitFilesystem(fsType FileSystemType) FileSystem
• type FileSystemType
• type LocalFileSystem
  • func (l *LocalFileSystem) Delete(filePath string) error
  • func (l *LocalFileSystem) Init() error
  • func (l *LocalFileSystem) Join(elem ...string) string
  • func (l *LocalFileSystem) ListFiles(pathGlob string) ([]FileInfo, error)
  • func (l *LocalFileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
  • func (l *LocalFileSystem) OpenWriter(filePath string) (io.WriteCloser, error)
  • func (l *LocalFileSystem) Stat(filePath string) (FileInfo, error)
• type S3FileSystem
  • func (s *S3FileSystem) Delete(filePath string) error
  • func (s *S3FileSystem) Init() error
  • func (s *S3FileSystem) Join(elem ...string) string
  • func (s *S3FileSystem) ListFiles(pathGlob string) ([]FileInfo, error)
  • func (s *S3FileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
  • func (s *S3FileSystem) OpenWriter(filePath string) (io.WriteCloser, error)
  • func (s *S3FileSystem) Stat(filePath string) (FileInfo, error)

Types

type FileInfo

type FileInfo struct {
	Name string // file path
	Size int64  // file size in bytes
}

FileInfo provides information about a file

type FileSystem

type FileSystem interface {
	ListFiles(pathGlob string) ([]FileInfo, error)
	Stat(filePath string) (FileInfo, error)
	OpenReader(filePath string, startAt int64) (io.ReadCloser, error)
	OpenWriter(filePath string) (io.WriteCloser, error)
	Delete(filePath string) error
	Join(elem ...string) string
	Init() error
}

FileSystem provides the file backend for MapReduce jobs. Input data is read from a file system. Intermediate and output data is written to a file system. This is abstracted to allow remote filesystems like S3 to be supported.

func InferFilesystem

func InferFilesystem(location string) FileSystem

InferFilesystem initializes a filesystem by inferring its type from a file address. For example, locations starting with "s3://" will resolve to an S3 filesystem.

func InitFilesystem

func InitFilesystem(fsType FileSystemType) FileSystem

InitFilesystem intializes a filesystem of the given type

type FileSystemType

type FileSystemType int

FileSystemType is an identifier for supported FileSystems

const (
	Local FileSystemType = iota
	S3
)

Identifiers for supported FileSystemTypes

type LocalFileSystem

type LocalFileSystem struct{}

LocalFileSystem wraps "os" to provide access to the local filesystem.

func (*LocalFileSystem) Delete

func (l *LocalFileSystem) Delete(filePath string) error

Delete deletes the file at filePath.

func (*LocalFileSystem) Init

func (l *LocalFileSystem) Init() error

Init initializes the filesystem.

func (*LocalFileSystem) Join

func (l *LocalFileSystem) Join(elem ...string) string

Join joins file path elements

func (*LocalFileSystem) ListFiles

func (l *LocalFileSystem) ListFiles(pathGlob string) ([]FileInfo, error)

ListFiles lists files that match pathGlob.

func (*LocalFileSystem) OpenReader

func (l *LocalFileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)

OpenReader opens a reader to the file at filePath. The reader is initially seeked to "startAt" bytes into the file.

func (*LocalFileSystem) OpenWriter

func (l *LocalFileSystem) OpenWriter(filePath string) (io.WriteCloser, error)

OpenWriter opens a writer to the file at filePath.

func (*LocalFileSystem) Stat

func (l *LocalFileSystem) Stat(filePath string) (FileInfo, error)

Stat returns information about the file at filePath.

type S3FileSystem

type S3FileSystem struct {
	// contains filtered or unexported fields
}

S3FileSystem abstracts AWS S3 as a filesystem

func (*S3FileSystem) Delete

func (s *S3FileSystem) Delete(filePath string) error

Delete deletes the file at filePath.

func (*S3FileSystem) Init

func (s *S3FileSystem) Init() error

Init initializes the filesystem.

func (*S3FileSystem) Join

func (s *S3FileSystem) Join(elem ...string) string

Join joins file path elements

func (*S3FileSystem) ListFiles

func (s *S3FileSystem) ListFiles(pathGlob string) ([]FileInfo, error)

ListFiles lists files that match pathGlob.

func (*S3FileSystem) OpenReader

func (s *S3FileSystem) OpenReader(filePath string, startAt int64) (io.ReadCloser, error)

OpenReader opens a reader to the file at filePath. The reader is initially seeked to "startAt" bytes into the file.

func (*S3FileSystem) OpenWriter

func (s *S3FileSystem) OpenWriter(filePath string) (io.WriteCloser, error)

OpenWriter opens a writer to the file at filePath.

func (*S3FileSystem) Stat

func (s *S3FileSystem) Stat(filePath string) (FileInfo, error)

Stat returns information about the file at filePath.