filesystem

package
v0.0.0-...-ca59f27 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 4, 2017 License: Apache-2.0 Imports: 11 Imported by: 0

Documentation

Index

Constants

View Source
const (
	AWS_ACCESS_KEY = OptionName("aws_access_key")
	AWS_SECRET_KEY = OptionName("aws_secret_key")
)

Variables

View Source
var (
	Option = make(map[OptionName]string)
)

Functions

func IsDir

func IsDir(filepath string) bool

func Set

func Set(name OptionName, value string)

Types

type FileLocation

type FileLocation struct {
	Location string
}

func List

func List(filepath string) ([]*FileLocation, error)

type HdfsFileSystem

type HdfsFileSystem struct {
}

Get the namenode from 1) the hdfs://namenode/... string 2) Or from env HADOOP_NAMENODE. 3) Or based on env HADOOP_CONF_DIR or HADOOP_HOME to locate hdfs-site.xml and core-site.xml

func (*HdfsFileSystem) Accept

func (fs *HdfsFileSystem) Accept(fl *FileLocation) bool

func (*HdfsFileSystem) IsDir

func (fs *HdfsFileSystem) IsDir(fl *FileLocation) bool

func (*HdfsFileSystem) List

func (fs *HdfsFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)

List generates a full list of file locations under the given location, which should have a prefix of hdfs://

func (*HdfsFileSystem) Open

func (fs *HdfsFileSystem) Open(fl *FileLocation) (VirtualFile, error)

type LocalFileSystem

type LocalFileSystem struct {
}

func (*LocalFileSystem) Accept

func (fs *LocalFileSystem) Accept(fl *FileLocation) bool

func (*LocalFileSystem) IsDir

func (fs *LocalFileSystem) IsDir(fl *FileLocation) bool

func (*LocalFileSystem) List

func (fs *LocalFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)

func (*LocalFileSystem) Open

func (fs *LocalFileSystem) Open(fl *FileLocation) (VirtualFile, error)

type OptionName

type OptionName string

type S3FileSystem

type S3FileSystem struct {
}

func (*S3FileSystem) Accept

func (fs *S3FileSystem) Accept(fl *FileLocation) bool

func (*S3FileSystem) IsDir

func (fs *S3FileSystem) IsDir(fl *FileLocation) bool

func (*S3FileSystem) List

func (fs *S3FileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)

func (*S3FileSystem) Open

func (fs *S3FileSystem) Open(fl *FileLocation) (VirtualFile, error)

type VirtualFile

type VirtualFile interface {
	// io.ReaderAt
	io.ReadCloser
}

func Open

func Open(filepath string) (VirtualFile, error)

type VirtualFileSystem

type VirtualFileSystem interface {
	Accept(*FileLocation) bool
	Open(*FileLocation) (VirtualFile, error)
	List(*FileLocation) ([]*FileLocation, error)
	IsDir(*FileLocation) bool
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL