s3inventory

package
v0.109.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 10, 2023 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source
const (
	OrcFormatName     = "ORC"
	ParquetFormatName = "Parquet"
)

Variables

View Source
var (
	ErrUnsupportedInventoryFormat = errors.New("unsupported inventory type. supported types: parquet, orc")
	ErrRequiredFieldNotFound      = errors.New("required field not found in inventory")
	ErrUnknownField               = errors.New("unknown field")
)

Functions

This section is empty.

Types

type FileReader

type FileReader interface {
	MetadataReader
	Read(n int) ([]*InventoryObject, error)
}

type IReader

type IReader interface {
	GetFileReader(format string, bucket string, key string) (FileReader, error)
	GetMetadataReader(format string, bucket string, key string) (MetadataReader, error)
}

func NewReader

func NewReader(ctx context.Context, svc s3iface.S3API, logger logging.Logger) IReader

type InventoryObject

type InventoryObject struct {
	Bucket         string
	Key            string
	IsLatest       bool
	IsDeleteMarker bool
	Size           int64
	LastModified   *time.Time
	Checksum       string
}

func NewInventoryObject

func NewInventoryObject() *InventoryObject

func (*InventoryObject) GetPhysicalAddress

func (o *InventoryObject) GetPhysicalAddress() string

type MetadataReader

type MetadataReader interface {
	GetNumRows() int64
	Close() error
	FirstObjectKey() string
	LastObjectKey() string
}

type OrcField

type OrcField struct {
	IndexInFile   int
	IndexInSelect int
}

type OrcFile

type OrcFile struct {
	*os.File
}

func DownloadOrc

func DownloadOrc(ctx context.Context, svc s3iface.S3API, logger logging.Logger, bucket string, key string, tailOnly bool) (*OrcFile, error)

DownloadOrc downloads a file from s3 and returns a ReaderSeeker to it. If tailOnly is set to true, download only the tail (metadata+footer) by trying the last `orcInitialReadSize` bytes of the file. Then, check the last byte to see if the whole tail was downloaded. If not, download again with the actual tail length.

func (*OrcFile) Close

func (or *OrcFile) Close() error

func (*OrcFile) Size

func (or *OrcFile) Size() int64

type OrcInventoryFileReader

type OrcInventoryFileReader struct {
	// contains filtered or unexported fields
}

func (*OrcInventoryFileReader) Close

func (r *OrcInventoryFileReader) Close() error

func (*OrcInventoryFileReader) FirstObjectKey

func (r *OrcInventoryFileReader) FirstObjectKey() string

func (*OrcInventoryFileReader) GetNumRows

func (r *OrcInventoryFileReader) GetNumRows() int64

func (*OrcInventoryFileReader) LastObjectKey

func (r *OrcInventoryFileReader) LastObjectKey() string

func (*OrcInventoryFileReader) Read

type OrcSelect

type OrcSelect struct {
	SelectFields  []string       // the list of fields to select from the file
	IndexInSelect map[string]int // for each field, its index in the select query
	IndexInFile   map[string]int // for each field, its index in the original file
}

type ParquetInventoryFileReader

type ParquetInventoryFileReader struct {
	*reader.ParquetReader
	// contains filtered or unexported fields
}

func NewParquetInventoryFileReader

func NewParquetInventoryFileReader(parquetReader *reader.ParquetReader) (*ParquetInventoryFileReader, error)

func (*ParquetInventoryFileReader) Close

func (p *ParquetInventoryFileReader) Close() error

func (*ParquetInventoryFileReader) FirstObjectKey

func (p *ParquetInventoryFileReader) FirstObjectKey() string

func (*ParquetInventoryFileReader) LastObjectKey

func (p *ParquetInventoryFileReader) LastObjectKey() string

func (*ParquetInventoryFileReader) Read

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

func (*Reader) GetFileReader

func (o *Reader) GetFileReader(format string, bucket string, key string) (FileReader, error)

func (*Reader) GetMetadataReader

func (o *Reader) GetMetadataReader(format string, bucket string, key string) (MetadataReader, error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL