s3inventory

Documentation

Index

• Constants
• Variables
• type FileReader
• type IReader
  • func NewReader(ctx context.Context, svc s3iface.S3API, logger logging.Logger) IReader
• type InventoryObject
  • func NewInventoryObject() *InventoryObject
  • func (o *InventoryObject) GetPhysicalAddress() string
• type MetadataReader
• type OrcField
• type OrcFile
  • func DownloadOrc(ctx context.Context, svc s3iface.S3API, logger logging.Logger, bucket string, ...) (*OrcFile, error)
  • func (or *OrcFile) Close() error
  • func (or *OrcFile) Size() int64
• type OrcInventoryFileReader
  • func (r *OrcInventoryFileReader) Close() error
  • func (r *OrcInventoryFileReader) FirstObjectKey() string
  • func (r *OrcInventoryFileReader) GetNumRows() int64
  • func (r *OrcInventoryFileReader) LastObjectKey() string
  • func (r *OrcInventoryFileReader) Read(n int) ([]*InventoryObject, error)
• type OrcSelect
• type ParquetInventoryFileReader
  • func NewParquetInventoryFileReader(parquetReader *reader.ParquetReader) (*ParquetInventoryFileReader, error)
  • func (p *ParquetInventoryFileReader) Close() error
  • func (p *ParquetInventoryFileReader) FirstObjectKey() string
  • func (p *ParquetInventoryFileReader) LastObjectKey() string
  • func (p *ParquetInventoryFileReader) Read(n int) ([]*InventoryObject, error)
• type Reader
  • func (o *Reader) GetFileReader(format string, bucket string, key string) (FileReader, error)
  • func (o *Reader) GetMetadataReader(format string, bucket string, key string) (MetadataReader, error)

Constants

const (
	OrcFormatName     = "ORC"
	ParquetFormatName = "Parquet"
)

Variables

var (
	ErrUnsupportedInventoryFormat = errors.New("unsupported inventory type. supported types: parquet, orc")
	ErrRequiredFieldNotFound      = errors.New("required field not found in inventory")
	ErrUnknownField               = errors.New("unknown field")
)

Types

type FileReader

type FileReader interface {
	MetadataReader
	Read(n int) ([]*InventoryObject, error)
}

type IReader

type IReader interface {
	GetFileReader(format string, bucket string, key string) (FileReader, error)
	GetMetadataReader(format string, bucket string, key string) (MetadataReader, error)
}

func NewReader

func NewReader(ctx context.Context, svc s3iface.S3API, logger logging.Logger) IReader

type InventoryObject

type InventoryObject struct {
	Bucket         string
	Key            string
	IsLatest       bool
	IsDeleteMarker bool
	Size           int64
	LastModified   *time.Time
	Checksum       string
}

func NewInventoryObject

func NewInventoryObject() *InventoryObject

func (*InventoryObject) GetPhysicalAddress

func (o *InventoryObject) GetPhysicalAddress() string

type MetadataReader

type MetadataReader interface {
	GetNumRows() int64
	Close() error
	FirstObjectKey() string
	LastObjectKey() string
}

type OrcField

type OrcField struct {
	IndexInFile   int
	IndexInSelect int
}

type OrcFile

type OrcFile struct {
	*os.File
}

func DownloadOrc

func DownloadOrc(ctx context.Context, svc s3iface.S3API, logger logging.Logger, bucket string, key string, tailOnly bool) (*OrcFile, error)

DownloadOrc downloads a file from s3 and returns a ReaderSeeker to it. If tailOnly is set to true, download only the tail (metadata+footer) by trying the last `orcInitialReadSize` bytes of the file. Then, check the last byte to see if the whole tail was downloaded. If not, download again with the actual tail length.

func (*OrcFile) Close

func (or *OrcFile) Close() error

func (*OrcFile) Size

func (or *OrcFile) Size() int64

type OrcInventoryFileReader

type OrcInventoryFileReader struct {
	// contains filtered or unexported fields
}

func (*OrcInventoryFileReader) Close

func (r *OrcInventoryFileReader) Close() error

func (*OrcInventoryFileReader) FirstObjectKey

func (r *OrcInventoryFileReader) FirstObjectKey() string

func (*OrcInventoryFileReader) GetNumRows

func (r *OrcInventoryFileReader) GetNumRows() int64

func (*OrcInventoryFileReader) LastObjectKey

func (r *OrcInventoryFileReader) LastObjectKey() string

func (*OrcInventoryFileReader) Read

func (r *OrcInventoryFileReader) Read(n int) ([]*InventoryObject, error)

type OrcSelect

type OrcSelect struct {
	SelectFields  []string       // the list of fields to select from the file
	IndexInSelect map[string]int // for each field, its index in the select query
	IndexInFile   map[string]int // for each field, its index in the original file
}

type ParquetInventoryFileReader

type ParquetInventoryFileReader struct {
	*reader.ParquetReader
	// contains filtered or unexported fields
}

func NewParquetInventoryFileReader

func NewParquetInventoryFileReader(parquetReader *reader.ParquetReader) (*ParquetInventoryFileReader, error)

func (*ParquetInventoryFileReader) Close

func (p *ParquetInventoryFileReader) Close() error

func (*ParquetInventoryFileReader) FirstObjectKey

func (p *ParquetInventoryFileReader) FirstObjectKey() string

func (*ParquetInventoryFileReader) LastObjectKey

func (p *ParquetInventoryFileReader) LastObjectKey() string

func (*ParquetInventoryFileReader) Read

func (p *ParquetInventoryFileReader) Read(n int) ([]*InventoryObject, error)

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

func (*Reader) GetFileReader

func (o *Reader) GetFileReader(format string, bucket string, key string) (FileReader, error)

func (*Reader) GetMetadataReader

func (o *Reader) GetMetadataReader(format string, bucket string, key string) (MetadataReader, error)