filesystem

Documentation

Index

• Constants
• Variables
• func IsDir(filepath string) bool
• func Set(name OptionName, value string)
• type FileLocation
  • func List(filepath string) ([]*FileLocation, error)
• type GoogleStorageFileSystem
  • func (fs *GoogleStorageFileSystem) Accept(fl *FileLocation) bool
  • func (fs *GoogleStorageFileSystem) IsDir(fl *FileLocation) bool
  • func (fs *GoogleStorageFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)
  • func (fs *GoogleStorageFileSystem) Open(fl *FileLocation) (v VirtualFile, err error)
• type HdfsFileSystem
  • func (fs *HdfsFileSystem) Accept(fl *FileLocation) bool
  • func (fs *HdfsFileSystem) IsDir(fl *FileLocation) bool
  • func (fs *HdfsFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)
  • func (fs *HdfsFileSystem) Open(fl *FileLocation) (VirtualFile, error)
• type LocalFileSystem
  • func (fs *LocalFileSystem) Accept(fl *FileLocation) bool
  • func (fs *LocalFileSystem) IsDir(fl *FileLocation) bool
  • func (fs *LocalFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)
  • func (fs *LocalFileSystem) Open(fl *FileLocation) (VirtualFile, error)
• type OptionName
• type S3FileSystem
  • func (fs *S3FileSystem) Accept(fl *FileLocation) bool
  • func (fs *S3FileSystem) IsDir(fl *FileLocation) bool
  • func (fs *S3FileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)
  • func (fs *S3FileSystem) Open(fl *FileLocation) (VirtualFile, error)
• type VirtualFile
  • func Open(filepath string) (VirtualFile, error)
• type VirtualFileGS
  • func (vf *VirtualFileGS) Close() error
  • func (vf *VirtualFileGS) Size() int64
• type VirtualFileHdfs
  • func (vf *VirtualFileHdfs) Size() int64
• type VirtualFileLocal
  • func (vf *VirtualFileLocal) Size() int64
• type VirtualFileS3
  • func (vf *VirtualFileS3) Close() error
  • func (vf *VirtualFileS3) Size() int64
• type VirtualFileSystem

Constants

const (
	AWS_ACCESS_KEY = OptionName("aws_access_key")
	AWS_SECRET_KEY = OptionName("aws_secret_key")
)

Variables

var (
	Option = make(map[OptionName]string)
)

Functions

func IsDir

func IsDir(filepath string) bool

func Set

func Set(name OptionName, value string)

Types

type FileLocation

type FileLocation struct {
	Location string
}

func List

func List(filepath string) ([]*FileLocation, error)

type GoogleStorageFileSystem

type GoogleStorageFileSystem struct{}

GoogleStorageFileSystem type

func (*GoogleStorageFileSystem) Accept

func (fs *GoogleStorageFileSystem) Accept(fl *FileLocation) bool

Accept - criteria for accepting path as google storage location

func (*GoogleStorageFileSystem) IsDir

func (fs *GoogleStorageFileSystem) IsDir(fl *FileLocation) bool

IsDir - returns true if directory detected This function assumes that if the prefix ends with "/" then the intent of the user is to represent a Dir

func (*GoogleStorageFileSystem) List

func (fs *GoogleStorageFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)

List - list items in google storage directory

func (*GoogleStorageFileSystem) Open

func (fs *GoogleStorageFileSystem) Open(fl *FileLocation) (v VirtualFile, err error)

Open - open given file location and return virtual file

type HdfsFileSystem

type HdfsFileSystem struct {
}

Get the namenode from 1) the hdfs://namenode/... string 2) Or from env HADOOP_NAMENODE. 3) Or based on env HADOOP_CONF_DIR or HADOOP_HOME to locate hdfs-site.xml and core-site.xml

func (*HdfsFileSystem) Accept

func (fs *HdfsFileSystem) Accept(fl *FileLocation) bool

func (*HdfsFileSystem) IsDir

func (fs *HdfsFileSystem) IsDir(fl *FileLocation) bool

func (*HdfsFileSystem) List

func (fs *HdfsFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)

List generates a full list of file locations under the given location, which should have a prefix of hdfs://

func (*HdfsFileSystem) Open

func (fs *HdfsFileSystem) Open(fl *FileLocation) (VirtualFile, error)

type LocalFileSystem

type LocalFileSystem struct {
}

func (*LocalFileSystem) Accept

func (fs *LocalFileSystem) Accept(fl *FileLocation) bool

func (*LocalFileSystem) IsDir

func (fs *LocalFileSystem) IsDir(fl *FileLocation) bool

func (*LocalFileSystem) List

func (fs *LocalFileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)

func (*LocalFileSystem) Open

func (fs *LocalFileSystem) Open(fl *FileLocation) (VirtualFile, error)

type OptionName

type OptionName string

type S3FileSystem

type S3FileSystem struct {
}

func (*S3FileSystem) Accept

func (fs *S3FileSystem) Accept(fl *FileLocation) bool

func (*S3FileSystem) IsDir

func (fs *S3FileSystem) IsDir(fl *FileLocation) bool

func (*S3FileSystem) List

func (fs *S3FileSystem) List(fl *FileLocation) (fileLocations []*FileLocation, err error)

func (*S3FileSystem) Open

func (fs *S3FileSystem) Open(fl *FileLocation) (VirtualFile, error)

type VirtualFile

type VirtualFile interface {
	io.ReaderAt
	io.ReadCloser
	io.Seeker
	Size() int64
}

func Open

func Open(filepath string) (VirtualFile, error)

type VirtualFileGS

type VirtualFileGS struct {
	*os.File
	// contains filtered or unexported fields
}

VirtualFileGS - Virtual File implementation for Google Storage

func (*VirtualFileGS) Close

func (vf *VirtualFileGS) Close() error

Close - virtual file Closeer function

func (*VirtualFileGS) Size

func (vf *VirtualFileGS) Size() int64

Size - returns virtual file size

type VirtualFileHdfs

type VirtualFileHdfs struct {
	*hdfs.FileReader
}

func (*VirtualFileHdfs) Size

func (vf *VirtualFileHdfs) Size() int64

type VirtualFileLocal

type VirtualFileLocal struct {
	*os.File
}

func (*VirtualFileLocal) Size

func (vf *VirtualFileLocal) Size() int64

type VirtualFileS3

type VirtualFileS3 struct {
	*os.File
	// contains filtered or unexported fields
}

func (*VirtualFileS3) Close

func (vf *VirtualFileS3) Close() error

func (*VirtualFileS3) Size

func (vf *VirtualFileS3) Size() int64

type VirtualFileSystem

type VirtualFileSystem interface {
	Accept(*FileLocation) bool
	Open(*FileLocation) (VirtualFile, error)
	List(*FileLocation) ([]*FileLocation, error)
	IsDir(*FileLocation) bool
}