ParquetFile

Documentation

Index

• func ConvertToThriftReader(file ParquetFile, offset int64, size int64) *thrift.TBufferedTransport
• func GetMemFileFs() afero.Fs
• func SetInMemFileFs(fs *afero.Fs)
• type BufferFile
  • func (bf BufferFile) Close() error
  • func (bf BufferFile) Create(name string) (ParquetFile, error)
  • func (bf BufferFile) Open(name string) (ParquetFile, error)
  • func (bf BufferFile) Read(p []byte) (cnt int, err error)
  • func (bf BufferFile) Seek(offset int64, pos int) (int64, error)
  • func (bf BufferFile) Write(p []byte) (int, error)
• type LocalFile
  • func (self *LocalFile) Close() error
  • func (self *LocalFile) Create(name string) (ParquetFile, error)
  • func (self *LocalFile) Open(name string) (ParquetFile, error)
  • func (self *LocalFile) Read(b []byte) (cnt int, err error)
  • func (self *LocalFile) Seek(offset int64, pos int) (int64, error)
  • func (self *LocalFile) Write(b []byte) (n int, err error)
• type MemFile
  • func (fs *MemFile) Close() error
  • func (fs *MemFile) Create(name string) (ParquetFile, error)
  • func (fs *MemFile) Open(name string) (ParquetFile, error)
  • func (fs *MemFile) Read(b []byte) (cnt int, err error)
  • func (fs *MemFile) Seek(offset int64, pos int) (int64, error)
  • func (fs *MemFile) Write(b []byte) (n int, err error)
• type OnCloseFunc
• type ParquetFile
  • func NewBufferFile(b []byte) (ParquetFile, error)
  • func NewLocalFileReader(name string) (ParquetFile, error)
  • func NewLocalFileWriter(name string) (ParquetFile, error)
  • func NewMemFileWriter(name string, f OnCloseFunc) (ParquetFile, error)
  • func NewWriterFile(writer io.Writer) ParquetFile
• type WriterFile
  • func (self *WriterFile) Close() error
  • func (self *WriterFile) Create(name string) (ParquetFile, error)
  • func (self *WriterFile) Open(name string) (ParquetFile, error)
  • func (self *WriterFile) Read(b []byte) (int, error)
  • func (self *WriterFile) Seek(offset int64, pos int) (int64, error)
  • func (self *WriterFile) Write(b []byte) (int, error)

Functions

func ConvertToThriftReader

func ConvertToThriftReader(file ParquetFile, offset int64, size int64) *thrift.TBufferedTransport

Convert a file reater to Thrift reader

func GetMemFileFs

func GetMemFileFs() afero.Fs

GetMemFileFs - returns the current memory file-system being used by ParquetFile

func SetInMemFileFs

func SetInMemFileFs(fs *afero.Fs)

SetInMemFileFs - overrides local in-memory fileSystem NOTE: this is set by NewMemFileWriter is created and memFs is still nil

Types

type BufferFile

type BufferFile struct {
	Reader *bytes.Reader
	// contains filtered or unexported fields
}

BufferFile allows reading parquet messages from a memory buffer.

func (BufferFile) Close

func (bf BufferFile) Close() error

Close is a no-op for a memory buffer.

func (BufferFile) Create

func (bf BufferFile) Create(name string) (ParquetFile, error)

func (BufferFile) Open

func (bf BufferFile) Open(name string) (ParquetFile, error)

func (BufferFile) Read

func (bf BufferFile) Read(p []byte) (cnt int, err error)

Read reads data form BufferFile into p.

func (BufferFile) Seek

func (bf BufferFile) Seek(offset int64, pos int) (int64, error)

Seek seeks in the underlying memory buffer.

func (BufferFile) Write

func (bf BufferFile) Write(p []byte) (int, error)

Write writes data from p into BufferFile.

type LocalFile

type LocalFile struct {
	FilePath string
	File     *os.File
}

func (*LocalFile) Close

func (self *LocalFile) Close() error

func (*LocalFile) Create

func (self *LocalFile) Create(name string) (ParquetFile, error)

func (*LocalFile) Open

func (self *LocalFile) Open(name string) (ParquetFile, error)

func (*LocalFile) Read

func (self *LocalFile) Read(b []byte) (cnt int, err error)

func (*LocalFile) Seek

func (self *LocalFile) Seek(offset int64, pos int) (int64, error)

func (*LocalFile) Write

func (self *LocalFile) Write(b []byte) (n int, err error)

type MemFile

type MemFile struct {
	FilePath string
	File     afero.File
	OnClose  OnCloseFunc
}

MemFile - ParquetFile type for in-memory file operations

func (*MemFile) Close

func (fs *MemFile) Close() error

Close - close file and execute OnCloseFunc

func (*MemFile) Create

func (fs *MemFile) Create(name string) (ParquetFile, error)

Create - create in-memory file

func (*MemFile) Open

func (fs *MemFile) Open(name string) (ParquetFile, error)

Open - open file in-memory

func (*MemFile) Read

func (fs *MemFile) Read(b []byte) (cnt int, err error)

Read - read file

func (*MemFile) Seek

func (fs *MemFile) Seek(offset int64, pos int) (int64, error)

Seek - seek function

func (*MemFile) Write

func (fs *MemFile) Write(b []byte) (n int, err error)

Write - write file in-memory

type OnCloseFunc

type OnCloseFunc func(string, io.Reader) error

OnCloseFunc function type, handles what to do after converted file is closed in-memory. Close() will pass the filename string and data as io.reader

type ParquetFile

type ParquetFile interface {
	io.Seeker
	io.Reader
	io.Writer
	io.Closer
	Open(name string) (ParquetFile, error)
	Create(name string) (ParquetFile, error)
}

func NewBufferFile

func NewBufferFile(b []byte) (ParquetFile, error)

NewBufferFile creates new in memory parquet buffer.

func NewLocalFileReader

func NewLocalFileReader(name string) (ParquetFile, error)

func NewLocalFileWriter

func NewLocalFileWriter(name string) (ParquetFile, error)

func NewMemFileWriter

func NewMemFileWriter(name string, f OnCloseFunc) (ParquetFile, error)

NewMemFileWriter - intiates and creates an instance of MemFiles NOTE: there is no NewMemFileReader as this particular type was written to handle in-memory converstions and offloading. The results of conversion can then be stored and read via HDFS, LocalFS, etc without the need for loading the file back into memory directly

func NewWriterFile

func NewWriterFile(writer io.Writer) ParquetFile

type WriterFile

type WriterFile struct {
	Writer io.Writer
}

func (*WriterFile) Close

func (self *WriterFile) Close() error

func (*WriterFile) Create

func (self *WriterFile) Create(name string) (ParquetFile, error)

func (*WriterFile) Open

func (self *WriterFile) Open(name string) (ParquetFile, error)

func (*WriterFile) Read

func (self *WriterFile) Read(b []byte) (int, error)

func (*WriterFile) Seek

func (self *WriterFile) Seek(offset int64, pos int) (int64, error)

func (*WriterFile) Write

func (self *WriterFile) Write(b []byte) (int, error)